amadan

carlos / platform Public

moves-phase1-7

Squashed from moves-phase1-7 into main.
claude 0715faca664041152e2a437a70debd06c7177669 parent 1b5802f
10 files changed, +3730 −10
  • cmd/moveprobe/README.md +233 −0
  • cmd/moveprobe/detach_unix.go +9 −0
  • cmd/moveprobe/detach_windows.go +9 −0
  • cmd/moveprobe/main.go +1021 −0
  • cmd/moveprobe/main_test.go +1352 −0
  • docs/superpowers/plans/2026-10-06-moves-phase1-7-rehearsal-runbook.md +4 −4
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +4 −4
  • docs/superpowers/plans/2026-10-06-moves-phase1-rehearsal.md +1085 −0
  • internal/console/apiinstancemove_test.go +6 −0
  • internal/provision/provision_test.go +7 −2

CI passed — run details

CI log 
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	322.246s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.145s
ok  	github.com/carlosframework/platform/cmd/moveprobe	93.175s
ok  	github.com/carlosframework/platform/internal/activator	211.287s
ok  	github.com/carlosframework/platform/internal/agentsock	0.016s
ok  	github.com/carlosframework/platform/internal/alerting	0.003s
ok  	github.com/carlosframework/platform/internal/analytics	0.018s
ok  	github.com/carlosframework/platform/internal/appconfig	2.213s
ok  	github.com/carlosframework/platform/internal/apperr	0.070s
ok  	github.com/carlosframework/platform/internal/applog	0.128s
ok  	github.com/carlosframework/platform/internal/awsinv	0.003s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.683s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.009s
ok  	github.com/carlosframework/platform/internal/caps	0.012s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.185s
ok  	github.com/carlosframework/platform/internal/changelog	0.240s
ok  	github.com/carlosframework/platform/internal/clamd	0.057s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	353.677s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.426s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.014s
ok  	github.com/carlosframework/platform/internal/dbverify	0.014s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.014s
ok  	github.com/carlosframework/platform/internal/docsite	0.014s
ok  	github.com/carlosframework/platform/internal/domains	1.986s
ok  	github.com/carlosframework/platform/internal/doorkey	0.014s
ok  	github.com/carlosframework/platform/internal/economics	0.014s
ok  	github.com/carlosframework/platform/internal/edge	25.723s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.014s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.137s
ok  	github.com/carlosframework/platform/internal/gate	1.655s
ok  	github.com/carlosframework/platform/internal/genselect	0.020s
ok  	github.com/carlosframework/platform/internal/geo	0.017s
ok  	github.com/carlosframework/platform/internal/geoip	0.031s
ok  	github.com/carlosframework/platform/internal/host	51.482s
ok  	github.com/carlosframework/platform/internal/hostactors	0.644s
ok  	github.com/carlosframework/platform/internal/hostbox	1.245s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.239s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.014s
ok  	github.com/carlosframework/platform/internal/hostexec	0.310s
ok  	github.com/carlosframework/platform/internal/hostfence	0.027s
ok  	github.com/carlosframework/platform/internal/hostfiles	4.119s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.022s
ok  	github.com/carlosframework/platform/internal/hostguard	0.133s
ok  	github.com/carlosframework/platform/internal/hostops	1.309s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.002s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.002s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.124s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.004s
ok  	github.com/carlosframework/platform/internal/infraview	0.002s
ok  	github.com/carlosframework/platform/internal/instancectl	1.700s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.677s
ok  	github.com/carlosframework/platform/internal/instancefs	0.004s
ok  	github.com/carlosframework/platform/internal/instancekey	0.003s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.009s
ok  	github.com/carlosframework/platform/internal/layout	1.326s
ok  	github.com/carlosframework/platform/internal/lease	9.425s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.006s
ok  	github.com/carlosframework/platform/internal/ledger	0.013s
ok  	github.com/carlosframework/platform/internal/mail	0.149s
ok  	github.com/carlosframework/platform/internal/maintenance	0.077s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.515s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.067s
ok  	github.com/carlosframework/platform/internal/metrics	0.004s
ok  	github.com/carlosframework/platform/internal/moves	20.971s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.720s
ok  	github.com/carlosframework/platform/internal/objstore	0.153s
ok  	github.com/carlosframework/platform/internal/ops	0.150s
ok  	github.com/carlosframework/platform/internal/park	0.175s
ok  	github.com/carlosframework/platform/internal/privatefile	0.004s
ok  	github.com/carlosframework/platform/internal/provision	3.173s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.006s
ok  	github.com/carlosframework/platform/internal/registry	3.665s
ok  	github.com/carlosframework/platform/internal/relay	7.226s
ok  	github.com/carlosframework/platform/internal/release	18.353s
ok  	github.com/carlosframework/platform/internal/rollout	1.487s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.003s
ok  	github.com/carlosframework/platform/internal/rootaws	0.006s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.003s
ok  	github.com/carlosframework/platform/internal/schedule	2.232s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	2.289s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.544s
ok  	github.com/carlosframework/platform/internal/status	0.038s
ok  	github.com/carlosframework/platform/internal/steering	0.005s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.127s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.003s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.002s
ok  	github.com/carlosframework/platform/internal/updatewords	0.002s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.002s
ok  	github.com/carlosframework/platform/internal/vet	1.517s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.022s
=== RUN   TestT4StorageProbeMinio
2026/10/12 05:21:08 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DDB0A969877046, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.014s
minio contract: PASS
diff --git a/cmd/moveprobe/README.md b/cmd/moveprobe/README.md
new file mode 100644
index 000000000..8f386216a
--- /dev/null
+++ b/cmd/moveprobe/README.md
@@ -0,0 +1,233 @@
+# moveprobe
+
+moveprobe is a small app for rehearsing instance moves. It keeps state in
+every place a real app can, so that after a move you can check that each kind
+of state arrived. It is not part of the platform release. You build it and
+ship it like any other app.
+
+## What it keeps
+
+- **Database rows.** A background loop writes a row every half second. A
+ separate worker process writes one every second. Each timer fire writes one.
+ Every row records the region it was written in.
+- **A key file.** On first boot it writes `instance.key` into the instance's
+ files directory and records the same key in the database. Every boot checks
+ that the two still match.
+- **The platform's instance key.** On first boot it records a fingerprint of
+ `CARLOS_INSTANCE_KEY`. Every boot checks that the key is still set and
+ unchanged.
+- **An object in its store.** On every boot it writes `probe/<boot id>.txt`
+ to the store bound with the prefix `MOVEPROBE`, and reads it back on every
+ readiness check.
+- **A sealed secret.** It needs `MOVEPROBE_SECRET` to be set.
+- **Timers.** A once-timer you arm through the app, and a schedule you set
+ with `carlos schedule set`.
+
+## Build
+
+The rehearsal boxes are arm64:
+
+```sh
+CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o moveprobe ./cmd/moveprobe
+```
+
+## Deploy
+
+Run these signed in to the ops account, unless the rehearsal names another.
+Use the deployment's instance domain where these say `example.com`, and pick
+a host name no earlier rehearsal has used.
+
+```sh
+carlos ship --app moveprobe --label "moveprobe" ./moveprobe
+carlos instances enable --app moveprobe
+carlos store create --app moveprobe --env-prefix MOVEPROBE
+carlos secrets set --app moveprobe MOVEPROBE_SECRET=<a long random value>
+carlos schedule set --app moveprobe --name every-minute --every 1m --path /tick
+carlos instances create --app moveprobe --host mp1.example.com --region ie
+```
+
+Leave private storage off when you enable instances, and create the store
+without `--environment`. A move only carries the app's own store. It refuses
+an instance whose store belongs to an environment or to the instance itself.
+The account must be approved for object storage before `store create`
+provisions anything.
+
+Keep the value of `MOVEPROBE_SECRET` in your shell. It is also the password
+for `/arm-once` and `/report`.
+
+## Use it
+
+```sh
+curl https://mp1.example.com/ready
+curl -X POST -H "Authorization: Bearer $MOVEPROBE_SECRET" "https://mp1.example.com/arm-once?in=600"
+curl -H "Authorization: Bearer $MOVEPROBE_SECRET" https://mp1.example.com/report
+```
+
+| path | who can call it | what it does |
+|---|---|---|
+| `/` and `/ready` | anyone | 200 when the key file, the platform key, the stored object and the secret all check out, and the worker's writes are landing. Otherwise 500 with every failed check. The store check runs at most once every 5 seconds, and requests in between share its result. A move probes `/`. |
+| `GET /report` | `MOVEPROBE_SECRET` | JSON: row counts by kind and region with their lowest and highest sequence numbers, the timer fires by name and due time, both key fingerprints, the stored object, whether the secret is set, the worker's pid and failed writes, and the region the app saw. Any failed database read makes it a 500, never a partial report. |
+| `POST /arm-once?in=<seconds>` | `MOVEPROBE_SECRET` | Arms a once-timer named `once-<n>` that calls `/once` after that many seconds. Answers 503 when the instance has no control socket. |
+| `POST /tick`, `POST /once` | the platform | Timer fires. Each one writes a row with the timer's name and due time. |
+
+The background rows are numbered in one sequence that carries on across
+boots and regions. If no row is missing, the count of `bg` rows equals the
+highest `bg` sequence number.
+
+The region comes from `CARLOS_REGION`. On a box that has no region, the app
+uses `home`, and `/report` says so.
+
+## Make readiness fail on purpose
+
+`MOVEPROBE_FAIL_READY_IN=<region>` makes the app fail its readiness check in
+that region. The app reads it when it starts, so set it before you request
+the move, and remove it once the run's checks are recorded:
+
+```sh
+carlos env set --app moveprobe MOVEPROBE_FAIL_READY_IN=ie
+carlos env unset --app moveprobe MOVEPROBE_FAIL_READY_IN
+```
+
+There are two modes:
+
+- `MOVEPROBE_FAIL_READY_IN=ie` writes nothing at all in `ie`. It does not
+ open the database, touch the files directory, start the worker or call the
+ control socket. Every path answers 503. A rollback from here should be
+ clean.
+- `MOVEPROBE_FAIL_READY_IN=ie,after-write` writes one background row in
+ `ie`, starts the worker, and then answers 503 on `/` and `/ready`. A
+ rollback from here is dirty, and the source should end up holding that
+ row.
+
+In any other region the variable has no effect.
+
+## The worker
+
+The app starts a worker process in its own session, with no connection to
+the app's output. Stopping the app does not stop the worker. That is the
+point: it shows whether a move's stop really stops every process the
+instance started. Once the app has gone, the worker keeps writing for 60
+seconds and then exits by itself.
+
+The worker tells the app whether each write landed. `/report` shows how many
+committed and how many failed since the app started, how many of the latest
+failed in a row, the last error, when it last heard from the worker, and
+why the worker is gone if it is. `/ready` fails until a worker row has
+committed since the app started, whenever the worker's latest write failed,
+if the worker has sent nothing for 3 seconds, and once the worker has
+exited.
+
+## Strings
+
+Every body the app can send, and when. Text in angle brackets is filled in
+when the app answers. `/ready` joins its failed checks with `; `.
+
+On `/` and `/ready`:
+
+- `ready`: every check passed (200).
+- `before-write mode: this box writes nothing`: the failure switch names
+ this region with no suffix. Every path gives this answer (503).
+- `after-write mode: not ready here`: the failure switch names this region
+ with `,after-write` (503).
+- `key mismatch: instance.key exists but the database has no key`: the
+ files directory has a key file, but the database never recorded one
+ (500).
+- `key mismatch: instance.key is missing`: the database has a key, but the
+ file is gone (500).
+- `key mismatch: instance.key differs from the database`: the file and the
+ database hold different keys (500).
+- `key mismatch: instance.key: <error>`: the file is there but cannot be
+ read (500).
+- `CARLOS_INSTANCE_KEY is not set`: the platform key is missing (500).
+- `CARLOS_INSTANCE_KEY differs from the one seen at first boot`: the
+ platform key changed (500).
+- `MOVEPROBE_SECRET is not set`: the sealed secret is missing (500).
+- `store: <names> not set`: one or more of the store variables is missing
+ (500).
+- `store: put <key>: <error>`: the boot's object could not be written
+ (500).
+- `store: get <key>: <error>`: the boot's object could not be read back
+ (500).
+- `store: <key> reads back different bytes`: the object came back changed
+ (500).
+- `store: <error>`: the request gave up while waiting for a store check
+ that another request had started (500).
+- `worker: exited (<how>)`: the worker has exited. `<how>` is how it
+ ended, such as `exit status 1` or `signal: killed`, or `its status pipe
+ closed` when the app saw it go before learning how (500).
+- `worker: did not start: <error>`: the app could not start the worker
+ (500).
+- `worker: no status for <n>s`: the worker has not reported a write for 3
+ seconds or more, for example because it is stuck (500).
+- `worker: no row committed since start`: the worker has not written a row
+ since the app started, for example in its first moments or if it did not
+ start (500).
+- `worker: no row committed since start; last error: <error>`: as above,
+ and its writes are failing (500).
+- `worker: the last write failed: <error>`: the worker's latest write
+ failed (500).
+
+On `/report`:
+
+- The JSON report (200). Its fields are `host`, `region`, `region_from`,
+ `mode`, `boot_id`, `rows` (each with `kind`, `region`, `count`,
+ `min_seq`, `max_seq`), `fired` (each with `kind`, `name`, `due_at`,
+ `count`), `file_key_sha256`, `platform_key_sha256`, `object` (with
+ `key`, `body` and, when it could not be read, `error`), `has_secret`,
+ `worker_pid`, `worker_writes_committed`, `worker_writes_failed`,
+ `worker_failed_in_a_row`, `worker_last_error`, `worker_exit` and
+ `worker_last_status_at` (unix seconds, 0 before the first).
+- `report: <error>`: a database read failed (500).
+
+The values the app writes into the report itself:
+
+- `region`: the value of `CARLOS_REGION`, or `home` when the box has no
+ region.
+- `region_from`: `CARLOS_REGION` when the variable is set, and
+ `unset, treated as home` when it is not.
+- `mode`: `normal`, or `after-write` when the failure switch names this
+ region with `,after-write`. Before-write mode sends no report.
+- `kind` in `rows`: `bg` for the background loop, `worker` for the
+ worker, `tick` and `once` for timer fires. `kind` in `fired`: `tick` or
+ `once`.
+- `object.key`: `probe/<boot id>.txt`, the object this boot wrote.
+- `object.body`: the object's contents as read back. The app writes
+ `moveprobe boot <boot id> host <host> region <region> at <time>`,
+ followed by a new line, where the time is UTC in RFC 3339 form.
+- `object.error`: `store: <names> not set` when the store variables are
+ missing, or `get <key>: <error>` when the object cannot be read back.
+- `worker_last_error`: the database's own error text for the worker's
+ latest failed write. The app adds no words of its own.
+- `worker_exit`: empty while the worker runs; otherwise
+ `exited (<how>)` or `did not start: <error>`, as on `/ready` without the
+ `worker: ` prefix.
+
+On `/arm-once`:
+
+- `{"name": "once-<n>", "at": "<time>", "due_at": <unix seconds>}`: the
+ timer is armed (200).
+- `in: want whole seconds, at most 400 days`: `in` is missing or out of
+ range (400).
+- `no control socket: CARLOS_CONTROL_SOCKET is not set`: the instance has
+ no control socket (503).
+- `CARLOS_ADMIN_TOKEN is not set`: the instance has no admin token, so it
+ cannot arm timers (503).
+- `database: <error>`: the timer number could not be taken (500).
+- `control socket: <error>`: the request to the control socket could not be
+ made or sent (500 or 502).
+- `control socket: <status> <reply>`: the control socket refused the timer
+ (502).
+
+On `/tick` and `/once`:
+
+- `ok`: the fire is recorded (200).
+- `want X-Carlos-Schedule and X-Carlos-Schedule-At`: a fire without the
+ timer's name or due time (400).
+- `database: <error>`: the row could not be written (500).
+
+On `/report`, `/arm-once`, `/tick` and `/once`:
+
+- `unauthorized`: the bearer is missing or wrong (401).
+
+Anywhere else, Go's own answers: `404 page not found` for an unknown path,
+and `Method Not Allowed` for a known path with the wrong method (405).
diff --git a/cmd/moveprobe/detach_unix.go b/cmd/moveprobe/detach_unix.go
new file mode 100644
index 000000000..596f3cc16
--- /dev/null
+++ b/cmd/moveprobe/detach_unix.go
@@ -0,0 +1,9 @@
+//go:build !windows
+
+package main
+
+import "syscall"
+
+// detachAttr puts the worker in its own session, so a signal to the main's
+// process group never reaches it.
+func detachAttr() *syscall.SysProcAttr { return &syscall.SysProcAttr{Setsid: true} }
diff --git a/cmd/moveprobe/detach_windows.go b/cmd/moveprobe/detach_windows.go
new file mode 100644
index 000000000..4aaa9a771
--- /dev/null
+++ b/cmd/moveprobe/detach_windows.go
@@ -0,0 +1,9 @@
+//go:build windows
+
+package main
+
+import "syscall"
+
+// detachAttr is nil on Windows, which has no sessions to leave; moveprobe
+// only runs on the platform's Linux boxes, and builds here so the tree does.
+func detachAttr() *syscall.SysProcAttr { return nil }
diff --git a/cmd/moveprobe/main.go b/cmd/moveprobe/main.go
new file mode 100644
index 000000000..9d07ecea8
--- /dev/null
+++ b/cmd/moveprobe/main.go
@@ -0,0 +1,1021 @@
+// Command moveprobe is the moves rehearsal probe (moves spec §14.4): a small
+// app shaped like a customer app, deployed by the rehearsal runbook so that a
+// move can be shown to carry every kind of state an instance has.
+//
+// It holds state in each place an instance can:
+//
+// - the database: a background loop writes a row every 500 ms, a forked
+// worker writes one every second, and timer fires write one each;
+// - the files directory: instance.key, written at first boot and checked
+// against the database at every boot;
+// - the platform's instance key (CARLOS_INSTANCE_KEY), fingerprinted at
+// first boot and checked at every boot;
+// - its store binding (MOVEPROBE_*): an object written at boot and read
+// back on every readiness check;
+// - a sealed secret (MOVEPROBE_SECRET);
+// - timers: once-timers armed through the instance control socket, and a
+// declared schedule set by the operator with `carlos schedule set`.
+//
+// The exec backend starts it as `moveprobe --socket <path> --db <path>`
+// (internal/activator/backend_exec.go, spawnChildFrom). The files directory
+// is the database path without ".db" (the flat layout). "/" answers what
+// "/ready" answers, because a move probes "/" when moves.ready_path is unset.
+//
+// MOVEPROBE_FAIL_READY_IN=<region>[,after-write], read at startup, makes the
+// probe fail its readiness in the named region. With no suffix (before-write)
+// it writes nothing at all: no database open, no file, no worker, no socket
+// call. With ",after-write" it writes one background row, lets its worker
+// write, and then fails readiness. A box with no CARLOS_REGION is region
+// "home".
+//
+// The README beside this file has the build and deploy lines.
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/subtle"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/fs"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/exec"
+ "os/signal"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ _ "modernc.org/sqlite"
+)
+
+const (
+ bgEvery = 500 * time.Millisecond
+ workerEvery = time.Second
+ // workerGrace is how long the worker keeps writing after its parent has
+ // gone. Long enough to outlive the main through a cut's stop and stop
+ // evidence (seconds); short enough that an ordinary sleep, which stops
+ // only the main, does not leave it writing for ever.
+ workerGrace = 60 * time.Second
+ // workerStale is how long the main waits for the worker's next status
+ // line (one a second) before readiness fails.
+ workerStale = 3 * time.Second
+ storeWait = 10 * time.Second
+ // storeCacheFor is how long one readiness store check answers for:
+ // anonymous /ready traffic costs at most one check per window.
+ storeCacheFor = 5 * time.Second
+
+ modeNormal = "normal"
+ modeBeforeWrite = "before-write"
+ modeAfterWrite = "after-write"
+)
+
+func main() {
+ if len(os.Args) > 1 && os.Args[1] == "worker" {
+ os.Exit(runWorker(os.Args[2:]))
+ }
+ os.Exit(run(os.Args[1:]))
+}
+
+func logf(format string, args ...any) {
+ fmt.Fprintf(os.Stderr, "moveprobe: "+format+"\n", args...)
+}
+
+// region is CARLOS_REGION, or "home" when the box has no placement code and
+// the agent leaves the variable out.
+func region() (name, from string) {
+ if r := strings.TrimSpace(os.Getenv("CARLOS_REGION")); r != "" {
+ return r, "CARLOS_REGION"
+ }
+ return "home", "unset, treated as home"
+}
+
+// failMode reads MOVEPROBE_FAIL_READY_IN for this region. A suffix other
+// than after-write is read as before-write, the mode that writes nothing.
+func failMode(spec, here string) string {
+ spec = strings.TrimSpace(spec)
+ if spec == "" {
+ return modeNormal
+ }
+ where, suffix, hasSuffix := strings.Cut(spec, ",")
+ if strings.TrimSpace(where) != here {
+ return modeNormal
+ }
+ if hasSuffix && strings.TrimSpace(suffix) == modeAfterWrite {
+ return modeAfterWrite
+ }
+ if hasSuffix {
+ logf("MOVEPROBE_FAIL_READY_IN suffix %q is not after-write; using before-write", suffix)
+ }
+ return modeBeforeWrite
+}
+
+func run(args []string) int {
+ fl := flag.NewFlagSet("moveprobe", flag.ContinueOnError)
+ sock := fl.String("socket", "", "unix socket to listen on")
+ dbPath := fl.String("db", "", "the instance database")
+ if err := fl.Parse(args); err != nil {
+ return 2
+ }
+ if *sock == "" || *dbPath == "" {
+ logf("--socket and --db are required")
+ return 2
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ here, from := region()
+ mode := failMode(os.Getenv("MOVEPROBE_FAIL_READY_IN"), here)
+ if mode == modeBeforeWrite {
+ logf("before-write mode in region %s: nothing is written", here)
+ return serve(ctx, *sock, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, "before-write mode: this box writes nothing", http.StatusServiceUnavailable)
+ }))
+ }
+
+ a, err := boot(ctx, *dbPath, here, from, mode)
+ if err != nil {
+ logf("%v", err)
+ return 1
+ }
+ defer a.db.Close()
+
+ loopCtx, stopLoop := context.WithCancel(ctx)
+ defer stopLoop()
+ if mode == modeAfterWrite {
+ // The mode's whole promise is this row; without it the rollback
+ // would have nothing to find, so the app does not start.
+ if err := a.insertSeq("bg"); err != nil {
+ logf("after-write: the bg row did not commit: %v", err)
+ return 1
+ }
+ } else {
+ go a.bgLoop(loopCtx)
+ }
+ a.forkWorker()
+ return serve(ctx, *sock, a.routes())
+}
+
+// serve listens on sock until ctx ends, then shuts down.
+func serve(ctx context.Context, sock string, h http.Handler) int {
+ if fi, err := os.Lstat(sock); err == nil && fi.Mode()&fs.ModeSocket != 0 {
+ os.Remove(sock)
+ }
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ logf("listen: %v", err)
+ return 1
+ }
+ srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second}
+ errc := make(chan error, 1)
+ go func() { errc <- srv.Serve(ln) }()
+ select {
+ case err := <-errc:
+ logf("serve: %v", err)
+ return 1
+ case <-ctx.Done():
+ }
+ shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ srv.Shutdown(shutCtx)
+ return 0
+}
+
+type app struct {
+ db *sql.DB
+ dbPath string
+ filesDir string
+ host string
+ region string
+ regionFrom string
+ mode string
+ bootID string
+
+ fileKey string // meta instance_key; empty when the boot could not settle it
+ fileKeyErr error // set at boot when the file and the database disagree
+ platformFP string // SHA-256 of CARLOS_INSTANCE_KEY
+ platErr error
+
+ store *s3.Client
+ storeErr error
+ bucket string
+ objKey string
+ objBody string
+
+ // The readiness store check: one in flight at a time, its result kept
+ // for storeCacheFor.
+ storeMu sync.Mutex
+ storeRunning chan struct{} // closed when the running check finishes
+ storeAt time.Time // when the cached result was taken
+ storeResult error
+
+ mu sync.Mutex
+ objPut bool
+ workerPID int
+ // The worker reports every write over a pipe: how many committed and
+ // how many failed since this start, how many of the latest failed in a
+ // row, and the last error.
+ workerCommitted int64
+ workerFailed int64
+ workerInARow int64
+ workerLastErr string
+ workerLastAt time.Time // the latest status line
+ workerExit string // why the worker is gone; empty while it runs
+
+ // control is the one client for the instance control socket, nil when
+ // the agent delivered none.
+ control *http.Client
+}
+
+func openDB(path string) (*sql.DB, error) {
+ db, err := sql.Open("sqlite", "file:"+url.PathEscape(path)+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
+ if err != nil {
+ return nil, err
+ }
+ db.SetMaxOpenConns(1)
+ return db, nil
+}
+
+func boot(ctx context.Context, dbPath, here, from, mode string) (*app, error) {
+ id := make([]byte, 8)
+ if _, err := rand.Read(id); err != nil {
+ return nil, err
+ }
+ a := &app{
+ dbPath: dbPath,
+ filesDir: strings.TrimSuffix(dbPath, ".db"),
+ host: os.Getenv("CARLOS_HOST"),
+ region: here,
+ regionFrom: from,
+ mode: mode,
+ bootID: hex.EncodeToString(id),
+ }
+ db, err := openDB(dbPath)
+ if err != nil {
+ return nil, fmt.Errorf("open %s: %w", dbPath, err)
+ }
+ a.db = db
+ for _, q := range []string{
+ `CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
+ `CREATE TABLE IF NOT EXISTS rows (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ kind TEXT NOT NULL,
+ name TEXT NOT NULL DEFAULT '',
+ seq INTEGER NOT NULL DEFAULT 0,
+ due_at INTEGER NOT NULL DEFAULT 0,
+ fired_at INTEGER NOT NULL,
+ region TEXT NOT NULL)`,
+ `CREATE INDEX IF NOT EXISTS rows_kind_seq ON rows (kind, seq)`,
+ } {
+ if _, err := db.Exec(q); err != nil {
+ db.Close()
+ return nil, fmt.Errorf("schema: %w", err)
+ }
+ }
+ if err := os.MkdirAll(a.filesDir, 0o700); err != nil {
+ db.Close()
+ return nil, fmt.Errorf("files directory: %w", err)
+ }
+ if err := a.settleFileKey(); err != nil {
+ db.Close()
+ return nil, err
+ }
+ if err := a.settlePlatformKey(); err != nil {
+ db.Close()
+ return nil, err
+ }
+ a.setupStore()
+ if sock := os.Getenv("CARLOS_CONTROL_SOCKET"); sock != "" {
+ a.control = &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, "unix", sock)
+ },
+ }}
+ }
+ if a.storeErr == nil {
+ putCtx, cancel := context.WithTimeout(ctx, storeWait)
+ if err := a.putObject(putCtx); err != nil {
+ logf("store: %v", err)
+ }
+ cancel()
+ }
+ return a, nil
+}
+
+func (a *app) meta(key string) (string, bool, error) {
+ var v string
+ err := a.db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&v)
+ if errors.Is(err, sql.ErrNoRows) {
+ return "", false, nil
+ }
+ return v, err == nil, err
+}
+
+func (a *app) keyPath() string { return filepath.Join(a.filesDir, "instance.key") }
+
+// settleFileKey writes instance.key at first boot and records it in meta;
+// on a later boot it remembers meta's value for the readiness check. A file
+// with no database key is a mismatch, never a reason to adopt the file.
+func (a *app) settleFileKey() error {
+ v, found, err := a.meta("instance_key")
+ if err != nil {
+ return fmt.Errorf("meta: %w", err)
+ }
+ if found {
+ a.fileKey = v
+ return nil
+ }
+ if _, err := os.Lstat(a.keyPath()); err == nil {
+ a.fileKeyErr = errors.New("key mismatch: instance.key exists but the database has no key")
+ return nil
+ }
+ raw := make([]byte, 32)
+ if _, err := rand.Read(raw); err != nil {
+ return err
+ }
+ key := hex.EncodeToString(raw)
+ f, err := os.OpenFile(a.keyPath(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return fmt.Errorf("instance.key: %w", err)
+ }
+ if _, err := f.WriteString(key + "\n"); err != nil {
+ f.Close()
+ return fmt.Errorf("instance.key: %w", err)
+ }
+ if err := f.Sync(); err != nil {
+ f.Close()
+ return fmt.Errorf("instance.key: %w", err)
+ }
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("instance.key: %w", err)
+ }
+ if _, err := a.db.Exec(`INSERT INTO meta (key, value) VALUES ('instance_key', ?)`, key); err != nil {
+ return fmt.Errorf("meta: %w", err)
+ }
+ a.fileKey = key
+ return nil
+}
+
+// settlePlatformKey records CARLOS_INSTANCE_KEY's fingerprint the first time
+// it is seen and holds every later boot to it.
+func (a *app) settlePlatformKey() error {
+ k := os.Getenv("CARLOS_INSTANCE_KEY")
+ if k == "" {
+ a.platErr = errors.New("CARLOS_INSTANCE_KEY is not set")
+ return nil
+ }
+ a.platformFP = fingerprint(k)
+ v, found, err := a.meta("platform_key_sha256")
+ if err != nil {
+ return fmt.Errorf("meta: %w", err)
+ }
+ if !found {
+ if _, err := a.db.Exec(`INSERT INTO meta (key, value) VALUES ('platform_key_sha256', ?)`, a.platformFP); err != nil {
+ return fmt.Errorf("meta: %w", err)
+ }
+ return nil
+ }
+ if v != a.platformFP {
+ a.platErr = errors.New("CARLOS_INSTANCE_KEY differs from the one seen at first boot")
+ }
+ return nil
+}
+
+func fingerprint(s string) string {
+ sum := sha256.Sum256([]byte(s))
+ return hex.EncodeToString(sum[:])
+}
+
+// fileKeyNow is instance.key as it is on disk now, trimmed.
+func (a *app) fileKeyNow() (string, error) {
+ b, err := os.ReadFile(a.keyPath())
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimSpace(string(b)), nil
+}
+
+func (a *app) checkFileKey() error {
+ if a.fileKeyErr != nil {
+ return a.fileKeyErr
+ }
+ got, err := a.fileKeyNow()
+ switch {
+ case errors.Is(err, fs.ErrNotExist):
+ return errors.New("key mismatch: instance.key is missing")
+ case err != nil:
+ return fmt.Errorf("key mismatch: instance.key: %v", err)
+ case subtle.ConstantTimeCompare([]byte(got), []byte(a.fileKey)) != 1:
+ return errors.New("key mismatch: instance.key differs from the database")
+ }
+ return nil
+}
+
+// ---- the store ----------------------------------------------------------------
+
+type storeConfig struct {
+ bucket, region, endpoint, accessKey, secretKey string
+}
+
+// storeFromEnv reads the MOVEPROBE binding. The endpoint is optional: the
+// platform delivers it only when the deployment has one, and never on AWS.
+func storeFromEnv() (storeConfig, error) {
+ c := storeConfig{
+ bucket: os.Getenv("MOVEPROBE_BUCKET"),
+ region: os.Getenv("MOVEPROBE_REGION"),
+ endpoint: os.Getenv("MOVEPROBE_ENDPOINT"),
+ accessKey: os.Getenv("MOVEPROBE_ACCESS_KEY"),
+ secretKey: os.Getenv("MOVEPROBE_SECRET_KEY"),
+ }
+ var missing []string
+ for _, v := range []struct{ name, val string }{
+ {"MOVEPROBE_BUCKET", c.bucket},
+ {"MOVEPROBE_REGION", c.region},
+ {"MOVEPROBE_ACCESS_KEY", c.accessKey},
+ {"MOVEPROBE_SECRET_KEY", c.secretKey},
+ } {
+ if v.val == "" {
+ missing = append(missing, v.name)
+ }
+ }
+ if len(missing) > 0 {
+ return c, fmt.Errorf("store: %s not set", strings.Join(missing, ", "))
+ }
+ return c, nil
+}
+
+// newStoreClient builds the S3 client from the binding's keys alone. It
+// never loads the default credential chain: the app shares the agent's uid
+// on a flat box, so the chain could find the box's own role and read the
+// object back without the binding. Path-style only with an endpoint, as
+// objstore.S3ClientFor does.
+func newStoreClient(c storeConfig) *s3.Client {
+ return s3.New(s3.Options{
+ Region: c.region,
+ Credentials: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
+ return aws.Credentials{AccessKeyID: c.accessKey, SecretAccessKey: c.secretKey, Source: "MOVEPROBE"}, nil
+ }),
+ BaseEndpoint: func() *string {
+ if c.endpoint == "" {
+ return nil
+ }
+ return aws.String(c.endpoint)
+ }(),
+ UsePathStyle: c.endpoint != "",
+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+ })
+}
+
+func (a *app) setupStore() {
+ c, err := storeFromEnv()
+ if err != nil {
+ a.storeErr = err
+ return
+ }
+ a.store = newStoreClient(c)
+ a.bucket = c.bucket
+ a.objKey = "probe/" + a.bootID + ".txt"
+ a.objBody = fmt.Sprintf("moveprobe boot %s host %s region %s at %s\n", a.bootID, a.host, a.region, time.Now().UTC().Format(time.RFC3339))
+}
+
+func (a *app) putObject(ctx context.Context) error {
+ _, err := a.store.PutObject(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(a.bucket),
+ Key: aws.String(a.objKey),
+ Body: bytes.NewReader([]byte(a.objBody)),
+ })
+ if err != nil {
+ return fmt.Errorf("put %s: %w", a.objKey, err)
+ }
+ a.mu.Lock()
+ a.objPut = true
+ a.mu.Unlock()
+ return nil
+}
+
+func (a *app) getObject(ctx context.Context) (string, error) {
+ out, err := a.store.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(a.bucket), Key: aws.String(a.objKey)})
+ if err != nil {
+ return "", fmt.Errorf("get %s: %w", a.objKey, err)
+ }
+ defer out.Body.Close()
+ b, err := io.ReadAll(io.LimitReader(out.Body, 1<<16))
+ if err != nil {
+ return "", fmt.Errorf("get %s: %w", a.objKey, err)
+ }
+ return string(b), nil
+}
+
+// checkStore puts the boot's object if the boot could not, then reads it
+// back.
+func (a *app) checkStore(ctx context.Context) error {
+ if a.storeErr != nil {
+ return a.storeErr
+ }
+ ctx, cancel := context.WithTimeout(ctx, storeWait)
+ defer cancel()
+ a.mu.Lock()
+ put := a.objPut
+ a.mu.Unlock()
+ if !put {
+ if err := a.putObject(ctx); err != nil {
+ return fmt.Errorf("store: %v", err)
+ }
+ }
+ got, err := a.getObject(ctx)
+ if err != nil {
+ return fmt.Errorf("store: %v", err)
+ }
+ if got != a.objBody {
+ return fmt.Errorf("store: %s reads back different bytes", a.objKey)
+ }
+ return nil
+}
+
+// storeVerdict is checkStore for readiness, coalesced: a result younger than
+// storeCacheFor is reused, and a request that arrives while a check runs
+// waits for that check instead of starting another. The check itself runs
+// on its own deadline, so one caller giving up does not cut it short for the
+// others.
+func (a *app) storeVerdict(ctx context.Context) error {
+ a.storeMu.Lock()
+ if !a.storeAt.IsZero() && time.Since(a.storeAt) < storeCacheFor {
+ err := a.storeResult
+ a.storeMu.Unlock()
+ return err
+ }
+ if running := a.storeRunning; running != nil {
+ a.storeMu.Unlock()
+ select {
+ case <-running:
+ case <-ctx.Done():
+ return fmt.Errorf("store: %v", ctx.Err())
+ }
+ a.storeMu.Lock()
+ err := a.storeResult
+ a.storeMu.Unlock()
+ return err
+ }
+ done := make(chan struct{})
+ a.storeRunning = done
+ a.storeMu.Unlock()
+
+ err := a.checkStore(context.Background())
+
+ a.storeMu.Lock()
+ a.storeResult, a.storeAt, a.storeRunning = err, time.Now(), nil
+ a.storeMu.Unlock()
+ close(done)
+ return err
+}
+
+// ---- rows ---------------------------------------------------------------------
+
+// insertSeq writes the next row of a sequenced kind. The sequence is taken
+// inside the insert, so two writers (a main and a worker from an earlier
+// boot, or two generations) never take the same number.
+func (a *app) insertSeq(kind string) error {
+ return insertSeq(a.db, kind, a.region)
+}
+
+func insertSeq(db *sql.DB, kind, region string) error {
+ _, err := db.Exec(`INSERT INTO rows (kind, seq, fired_at, region)
+ SELECT ?, coalesce(max(seq), 0) + 1, ?, ? FROM rows WHERE kind = ?`,
+ kind, time.Now().Unix(), region, kind)
+ return err
+}
+
+func (a *app) bgLoop(ctx context.Context) {
+ t := time.NewTicker(bgEvery)
+ defer t.Stop()
+ for {
+ if err := a.insertSeq("bg"); err != nil {
+ logf("bg: %v", err)
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ }
+ }
+}
+
+// ---- the worker ---------------------------------------------------------------
+
+// forkWorker starts the worker in its own session, with its stdio on
+// /dev/null, so it neither shares the main's signals nor holds the pipes the
+// backend reads the main's output from. A pipe of the main's own (the
+// worker's fd 3) carries the outcome of each of the worker's writes.
+func (a *app) forkWorker() {
+ status, statusW, err := os.Pipe()
+ if err != nil {
+ logf("worker: %v", err)
+ return
+ }
+ cmd := exec.Command(os.Args[0], "worker", "--db", a.dbPath, "--parent", strconv.Itoa(os.Getpid()), "--status-fd", "3")
+ cmd.SysProcAttr = detachAttr()
+ cmd.ExtraFiles = []*os.File{statusW}
+ err = cmd.Start()
+ statusW.Close()
+ if err != nil {
+ status.Close()
+ logf("worker: %v", err)
+ a.mu.Lock()
+ a.workerExit = "did not start: " + err.Error()
+ a.mu.Unlock()
+ return
+ }
+ a.mu.Lock()
+ a.workerPID = cmd.Process.Pid
+ a.mu.Unlock()
+ go func() {
+ err := cmd.Wait()
+ how := "exit status 0"
+ if err != nil {
+ how = err.Error()
+ }
+ a.mu.Lock()
+ a.workerExit = "exited (" + how + ")"
+ a.mu.Unlock()
+ }()
+ go a.readWorkerStatus(status)
+}
+
+// readWorkerStatus reads the worker's lines: "ok", or "fail <error>".
+func (a *app) readWorkerStatus(r *os.File) {
+ defer r.Close()
+ sc := bufio.NewScanner(r)
+ for sc.Scan() {
+ line := sc.Text()
+ a.mu.Lock()
+ a.workerLastAt = time.Now()
+ if msg, failed := strings.CutPrefix(line, "fail "); failed {
+ a.workerFailed++
+ a.workerInARow++
+ a.workerLastErr = msg
+ } else {
+ a.workerCommitted++
+ a.workerInARow = 0
+ }
+ a.mu.Unlock()
+ }
+ // The pipe closes when the worker exits; Wait's own account, when it
+ // comes, replaces this one.
+ a.mu.Lock()
+ if a.workerExit == "" {
+ a.workerExit = "exited (its status pipe closed)"
+ }
+ a.mu.Unlock()
+}
+
+// checkWorker holds readiness to the worker: it is still running and has
+// sent a status line within workerStale, at least one of its rows has
+// committed since this start, and its latest write succeeded.
+func (a *app) checkWorker() error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ switch {
+ case a.workerExit != "":
+ return errors.New("worker: " + a.workerExit)
+ case !a.workerLastAt.IsZero() && time.Since(a.workerLastAt) > workerStale:
+ return fmt.Errorf("worker: no status for %ds", int(time.Since(a.workerLastAt).Seconds()))
+ case a.workerCommitted == 0 && a.workerInARow > 0:
+ return fmt.Errorf("worker: no row committed since start; last error: %s", a.workerLastErr)
+ case a.workerCommitted == 0:
+ return errors.New("worker: no row committed since start")
+ case a.workerInARow > 0:
+ return fmt.Errorf("worker: the last write failed: %s", a.workerLastErr)
+ }
+ return nil
+}
+
+// runWorker writes a worker row every second. Once its parent has gone it
+// keeps writing for workerGrace and then exits.
+func runWorker(args []string) int {
+ fl := flag.NewFlagSet("moveprobe worker", flag.ContinueOnError)
+ dbPath := fl.String("db", "", "the instance database")
+ parent := fl.Int("parent", 0, "the main's pid")
+ statusFD := fl.Int("status-fd", 0, "a pipe to report each write on (0: none)")
+ if err := fl.Parse(args); err != nil || *dbPath == "" || *parent <= 0 {
+ return 2
+ }
+ here, _ := region()
+ // Writes to the pipe fail once the main has gone; they are ignored.
+ report := func(err error) {}
+ if *statusFD > 0 {
+ status := os.NewFile(uintptr(*statusFD), "status")
+ defer status.Close()
+ report = func(err error) {
+ if err != nil {
+ fmt.Fprintf(status, "fail %s\n", strings.ReplaceAll(err.Error(), "\n", " "))
+ } else {
+ fmt.Fprintln(status, "ok")
+ }
+ }
+ }
+ db, err := openDB(*dbPath)
+ if err != nil {
+ report(err)
+ return 1
+ }
+ defer db.Close()
+ var orphaned time.Time
+ t := time.NewTicker(workerEvery)
+ defer t.Stop()
+ for {
+ if orphaned.IsZero() && os.Getppid() != *parent {
+ orphaned = time.Now()
+ }
+ if !orphaned.IsZero() && time.Since(orphaned) >= workerGrace {
+ return 0
+ }
+ report(insertSeq(db, "worker", here))
+ <-t.C
+ }
+}
+
+// ---- HTTP ---------------------------------------------------------------------
+
+func (a *app) routes() http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/{$}", a.handleReady)
+ mux.HandleFunc("/ready", a.handleReady)
+ mux.HandleFunc("GET /report", a.bearer(os.Getenv("MOVEPROBE_SECRET"), a.handleReport))
+ mux.HandleFunc("POST /arm-once", a.bearer(os.Getenv("MOVEPROBE_SECRET"), a.handleArmOnce))
+ // The platform's runner signs each fire with the instance's admin token.
+ mux.HandleFunc("POST /tick", a.bearer(os.Getenv("CARLOS_ADMIN_TOKEN"), a.fired("tick")))
+ mux.HandleFunc("POST /once", a.bearer(os.Getenv("CARLOS_ADMIN_TOKEN"), a.fired("once")))
+ return mux
+}
+
+// bearer admits a request carrying "Authorization: Bearer <want>". An empty
+// want admits nothing.
+func (a *app) bearer(want string, next http.HandlerFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ got := sha256.Sum256([]byte(r.Header.Get("Authorization")))
+ exp := sha256.Sum256([]byte("Bearer " + want))
+ if want == "" || subtle.ConstantTimeCompare(got[:], exp[:]) != 1 {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ next(w, r)
+ }
+}
+
+// readiness is /ready's answer: every check that fails, or ready.
+func (a *app) readiness(ctx context.Context) (int, string) {
+ if a.mode == modeAfterWrite {
+ return http.StatusServiceUnavailable, "after-write mode: not ready here"
+ }
+ var fails []string
+ if err := a.checkFileKey(); err != nil {
+ fails = append(fails, err.Error())
+ }
+ if a.platErr != nil {
+ fails = append(fails, a.platErr.Error())
+ }
+ if os.Getenv("MOVEPROBE_SECRET") == "" {
+ fails = append(fails, "MOVEPROBE_SECRET is not set")
+ }
+ if err := a.storeVerdict(ctx); err != nil {
+ fails = append(fails, err.Error())
+ }
+ if err := a.checkWorker(); err != nil {
+ fails = append(fails, err.Error())
+ }
+ if len(fails) > 0 {
+ return http.StatusInternalServerError, strings.Join(fails, "; ")
+ }
+ return http.StatusOK, "ready"
+}
+
+func (a *app) handleReady(w http.ResponseWriter, r *http.Request) {
+ code, msg := a.readiness(r.Context())
+ if code != http.StatusOK {
+ http.Error(w, msg, code)
+ return
+ }
+ fmt.Fprintln(w, msg)
+}
+
+func (a *app) fired(kind string) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ name := r.Header.Get("X-Carlos-Schedule")
+ due, err := strconv.ParseInt(r.Header.Get("X-Carlos-Schedule-At"), 10, 64)
+ if name == "" || err != nil {
+ http.Error(w, "want X-Carlos-Schedule and X-Carlos-Schedule-At", http.StatusBadRequest)
+ return
+ }
+ if _, err := a.db.Exec(`INSERT INTO rows (kind, name, due_at, fired_at, region) VALUES (?, ?, ?, ?, ?)`,
+ kind, name, due, time.Now().Unix(), a.region); err != nil {
+ http.Error(w, "database: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+ fmt.Fprintln(w, "ok")
+ }
+}
+
+// handleArmOnce registers once-<n> for /once, due in the given seconds,
+// through the instance control socket.
+func (a *app) handleArmOnce(w http.ResponseWriter, r *http.Request) {
+ in, err := strconv.ParseInt(r.URL.Query().Get("in"), 10, 64)
+ if err != nil || in < 0 || in > 400*24*3600 {
+ http.Error(w, "in: want whole seconds, at most 400 days", http.StatusBadRequest)
+ return
+ }
+ if a.control == nil {
+ http.Error(w, "no control socket: CARLOS_CONTROL_SOCKET is not set", http.StatusServiceUnavailable)
+ return
+ }
+ token := os.Getenv("CARLOS_ADMIN_TOKEN")
+ if token == "" {
+ http.Error(w, "CARLOS_ADMIN_TOKEN is not set", http.StatusServiceUnavailable)
+ return
+ }
+ var n int64
+ if err := a.db.QueryRow(`INSERT INTO meta (key, value) VALUES ('once_seq', '1')
+ ON CONFLICT (key) DO UPDATE SET value = CAST(value AS INTEGER) + 1
+ RETURNING CAST(value AS INTEGER)`).Scan(&n); err != nil {
+ http.Error(w, "database: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+ name := fmt.Sprintf("once-%d", n)
+ at := time.Now().Add(time.Duration(in) * time.Second).UTC().Truncate(time.Second)
+ body, _ := json.Marshal(map[string]string{"at": at.Format(time.RFC3339), "path": "/once"})
+
+ ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://carlos/schedule/"+name, bytes.NewReader(body))
+ if err != nil {
+ http.Error(w, "control socket: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := a.control.Do(req)
+ if err != nil {
+ http.Error(w, "control socket: "+err.Error(), http.StatusBadGateway)
+ return
+ }
+ // Drained before close, so the connection goes back for the next arm.
+ defer func() {
+ io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
+ resp.Body.Close()
+ }()
+ reply, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode/100 != 2 {
+ http.Error(w, fmt.Sprintf("control socket: %d %s", resp.StatusCode, strings.TrimSpace(string(reply))), http.StatusBadGateway)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"name": name, "at": at.Format(time.RFC3339), "due_at": at.Unix()})
+}
+
+type rowCount struct {
+ Kind string `json:"kind"`
+ Region string `json:"region"`
+ Count int64 `json:"count"`
+ MinSeq int64 `json:"min_seq"`
+ MaxSeq int64 `json:"max_seq"`
+}
+
+type firedCount struct {
+ Kind string `json:"kind"`
+ Name string `json:"name"`
+ DueAt int64 `json:"due_at"`
+ Count int64 `json:"count"`
+}
+
+type objectReport struct {
+ Key string `json:"key"`
+ Body string `json:"body"`
+ Error string `json:"error,omitempty"`
+}
+
+type report struct {
+ Host string `json:"host"`
+ Region string `json:"region"`
+ RegionFrom string `json:"region_from"`
+ Mode string `json:"mode"`
+ BootID string `json:"boot_id"`
+ Rows []rowCount `json:"rows"`
+ Fired []firedCount `json:"fired"`
+ FileKeySHA256 string `json:"file_key_sha256"`
+ PlatformKeySHA256 string `json:"platform_key_sha256"`
+ Object objectReport `json:"object"`
+ HasSecret bool `json:"has_secret"`
+ WorkerPID int `json:"worker_pid"`
+ WorkerCommitted int64 `json:"worker_writes_committed"`
+ WorkerFailed int64 `json:"worker_writes_failed"`
+ WorkerInARow int64 `json:"worker_failed_in_a_row"`
+ WorkerLastError string `json:"worker_last_error"`
+ WorkerExit string `json:"worker_exit"`
+ WorkerLastStatus int64 `json:"worker_last_status_at"`
+}
+
+// counts reads the row counts and the timer fires. Any failed query, scan
+// or read part way through is an error, never a shorter list.
+func (a *app) counts(ctx context.Context) ([]rowCount, []firedCount, error) {
+ rowsOut := []rowCount{}
+ err := a.query(ctx, `SELECT kind, region, count(*), min(seq), max(seq) FROM rows GROUP BY kind, region ORDER BY kind, region`,
+ func(rows *sql.Rows) error {
+ var c rowCount
+ if err := rows.Scan(&c.Kind, &c.Region, &c.Count, &c.MinSeq, &c.MaxSeq); err != nil {
+ return err
+ }
+ rowsOut = append(rowsOut, c)
+ return nil
+ })
+ if err != nil {
+ return nil, nil, err
+ }
+ fired := []firedCount{}
+ err = a.query(ctx, `SELECT kind, name, due_at, count(*) FROM rows WHERE kind IN ('tick', 'once') GROUP BY kind, name, due_at ORDER BY kind, name, due_at`,
+ func(rows *sql.Rows) error {
+ var c firedCount
+ if err := rows.Scan(&c.Kind, &c.Name, &c.DueAt, &c.Count); err != nil {
+ return err
+ }
+ fired = append(fired, c)
+ return nil
+ })
+ if err != nil {
+ return nil, nil, err
+ }
+ return rowsOut, fired, nil
+}
+
+func (a *app) query(ctx context.Context, q string, each func(*sql.Rows) error) error {
+ rows, err := a.db.QueryContext(ctx, q)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+ for rows.Next() {
+ if err := each(rows); err != nil {
+ return err
+ }
+ }
+ return rows.Err()
+}
+
+func (a *app) handleReport(w http.ResponseWriter, r *http.Request) {
+ rep := report{
+ Host: a.host, Region: a.region, RegionFrom: a.regionFrom, Mode: a.mode, BootID: a.bootID,
+ PlatformKeySHA256: a.platformFP,
+ HasSecret: os.Getenv("MOVEPROBE_SECRET") != "",
+ }
+ a.mu.Lock()
+ rep.WorkerPID = a.workerPID
+ rep.WorkerCommitted, rep.WorkerFailed = a.workerCommitted, a.workerFailed
+ rep.WorkerInARow, rep.WorkerLastError = a.workerInARow, a.workerLastErr
+ rep.WorkerExit = a.workerExit
+ if !a.workerLastAt.IsZero() {
+ rep.WorkerLastStatus = a.workerLastAt.Unix()
+ }
+ a.mu.Unlock()
+ if k, err := a.fileKeyNow(); err == nil {
+ rep.FileKeySHA256 = fingerprint(k)
+ }
+
+ var err error
+ if rep.Rows, rep.Fired, err = a.counts(r.Context()); err != nil {
+ http.Error(w, "report: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ rep.Object.Key = a.objKey
+ if a.storeErr != nil {
+ rep.Object.Error = a.storeErr.Error()
+ } else {
+ ctx, cancel := context.WithTimeout(r.Context(), storeWait)
+ body, err := a.getObject(ctx)
+ cancel()
+ rep.Object.Body = body
+ if err != nil {
+ rep.Object.Error = err.Error()
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ enc.Encode(rep)
+}
diff --git a/cmd/moveprobe/main_test.go b/cmd/moveprobe/main_test.go
new file mode 100644
index 000000000..693ecda3c
--- /dev/null
+++ b/cmd/moveprobe/main_test.go
@@ -0,0 +1,1352 @@
+//go:build linux
+
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "database/sql"
+ "database/sql/driver"
+ "encoding/hex"
+ "encoding/json"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "math/big"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/schedule"
+
+ _ "modernc.org/sqlite"
+)
+
+const (
+ testHost = "h1.example.com"
+ adminToken = "admin-token-for-tests"
+ probeSecret = "probe-secret-for-tests"
+ platformKey = "platform-instance-key-for-tests"
+ accessKey = "AKIDMOVEPROBETEST"
+ bucketName = "probe-bucket"
+ storeRegion = "eu-west-1"
+)
+
+// binPath is the moveprobe binary the tests start, built once in TestMain,
+// so every test starts the app the way the exec backend does: a separate
+// process given --socket and --db.
+var binPath string
+
+func TestMain(m *testing.M) {
+ dir, err := os.MkdirTemp("", "mpbin")
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+ binPath = filepath.Join(dir, "moveprobe")
+ if out, err := exec.Command("go", "build", "-o", binPath, ".").CombinedOutput(); err != nil {
+ fmt.Fprintf(os.Stderr, "build moveprobe: %v\n%s", err, out)
+ os.RemoveAll(dir)
+ os.Exit(1)
+ }
+ code := m.Run()
+ os.RemoveAll(dir)
+ os.Exit(code)
+}
+
+// shortTempDir keeps unix socket paths under the kernel's length limit
+// (as internal/activator/rolelaunch_root_linux_test.go does); t.TempDir's
+// names carry the test's name and can pass 107 bytes.
+func shortTempDir(t *testing.T) string {
+ t.Helper()
+ dir, err := os.MkdirTemp("", "mp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.RemoveAll(dir) })
+ return dir
+}
+
+// fixture is one instance's world: its data directory (the flat layout,
+// <dir>/h1.db and <dir>/h1/), a test S3 holding its store, and a control
+// socket served by the platform's own schedule verbs over a registry.
+type fixture struct {
+ t *testing.T
+ sockDir string
+ dataDir string
+ db string
+ bucket *leasetest.VersionedBucket
+ store http.Handler
+ storeURL string
+ reg *registry.Registry
+ control string
+ env map[string]string
+
+ mu sync.Mutex
+ auths []string
+ storeReq atomic.Int64
+ // storeDelay slows every store request, in milliseconds.
+ storeDelay atomic.Int64
+ // controlConns counts the connections the control socket accepted.
+ controlConns atomic.Int64
+ starts int
+}
+
+func newFixture(t *testing.T) *fixture {
+ t.Helper()
+ f := &fixture{t: t, sockDir: shortTempDir(t), dataDir: t.TempDir()}
+ f.db = filepath.Join(f.dataDir, "h1.db")
+
+ f.bucket = leasetest.NewVersionedBucket()
+ f.bucket.AddBucket(bucketName)
+ s3h, _ := leasetest.NewS3Server(f.bucket)
+ f.store = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ f.storeReq.Add(1)
+ time.Sleep(time.Duration(f.storeDelay.Load()) * time.Millisecond)
+ f.mu.Lock()
+ f.auths = append(f.auths, r.Header.Get("Authorization"))
+ f.mu.Unlock()
+ s3h.ServeHTTP(w, r)
+ })
+ srv := httptest.NewServer(f.store)
+ t.Cleanup(srv.Close)
+ f.storeURL = srv.URL
+
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ f.reg = reg
+ f.control = filepath.Join(f.sockDir, "c.sock")
+ ln, err := net.Listen("unix", f.control)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mux := http.NewServeMux()
+ schedule.Mount(mux, testHost, reg, func() (string, error) { return adminToken, nil }, time.Now)
+ cs := &http.Server{Handler: mux, ConnState: func(_ net.Conn, s http.ConnState) {
+ if s == http.StateNew {
+ f.controlConns.Add(1)
+ }
+ }}
+ go cs.Serve(ln)
+ t.Cleanup(func() { cs.Close() })
+
+ // The environment agentOwnedVars composes over <host>.env: the control
+ // socket (when the listener bound), the purge channel, the region (when
+ // the box has one) and the host; then the env file's own variables.
+ f.env = map[string]string{
+ "CARLOS_CONTROL_SOCKET": f.control,
+ "CARLOS_PURGE": "header",
+ "CARLOS_REGION": "uk",
+ "CARLOS_HOST": testHost,
+ "CARLOS_ADMIN_TOKEN": adminToken,
+ "CARLOS_INSTANCE_KEY": platformKey,
+ "MOVEPROBE_BUCKET": bucketName,
+ "MOVEPROBE_REGION": storeRegion,
+ "MOVEPROBE_ENDPOINT": f.storeURL,
+ "MOVEPROBE_ACCESS_KEY": accessKey,
+ "MOVEPROBE_SECRET_KEY": "secret-key-for-tests",
+ "MOVEPROBE_SECRET": probeSecret,
+ }
+ // Registered first, so it runs last: every probe's main is gone by then.
+ t.Cleanup(func() { killWorkers(f.db) })
+ return f
+}
+
+func (f *fixture) seenAuth() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.auths...)
+}
+
+// envWith is the fixture's environment with changes applied; an empty
+// value removes the variable.
+func (f *fixture) envWith(changes map[string]string) map[string]string {
+ out := map[string]string{}
+ for k, v := range f.env {
+ out[k] = v
+ }
+ for k, v := range changes {
+ if v == "" {
+ delete(out, k)
+ } else {
+ out[k] = v
+ }
+ }
+ return out
+}
+
+// childEnv mirrors the backend's: the parent's environment without any
+// CARLOS_ variable, then the instance's variables. The test also drops the
+// probe's and AWS's own names and every proxy setting, so nothing from the
+// shell running the tests reaches the app.
+func childEnv(vars map[string]string) []string {
+ var out []string
+ for _, kv := range os.Environ() {
+ k, _, _ := strings.Cut(kv, "=")
+ switch {
+ case strings.HasPrefix(k, "CARLOS_"), strings.HasPrefix(k, "MOVEPROBE_"), strings.HasPrefix(k, "AWS_"):
+ continue
+ case strings.EqualFold(k, "HTTP_PROXY"), strings.EqualFold(k, "HTTPS_PROXY"), strings.EqualFold(k, "NO_PROXY"), k == "SSL_CERT_FILE", k == "SSL_CERT_DIR":
+ continue
+ }
+ out = append(out, kv)
+ }
+ for k, v := range vars {
+ out = append(out, k+"="+v)
+ }
+ return out
+}
+
+type lockedBuffer struct {
+ mu sync.Mutex
+ b bytes.Buffer
+}
+
+func (l *lockedBuffer) Write(p []byte) (int, error) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.b.Write(p)
+}
+
+func (l *lockedBuffer) String() string {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.b.String()
+}
+
+type probe struct {
+ t *testing.T
+ cmd *exec.Cmd
+ sock string
+ out *lockedBuffer
+ done chan struct{}
+}
+
+// start runs the binary exactly as spawnChildFrom does:
+// exec.Command(bin, "--socket", sock, "--db", dbPath) with the composed env.
+// Its stdout and stderr are pipes, as the backend's tee makes them.
+func (f *fixture) start(env map[string]string) *probe {
+ f.t.Helper()
+ f.starts++
+ p := &probe{t: f.t, sock: filepath.Join(f.sockDir, fmt.Sprintf("a%d.sock", f.starts)), out: &lockedBuffer{}, done: make(chan struct{})}
+ p.cmd = exec.Command(binPath, "--socket", p.sock, "--db", f.db)
+ p.cmd.Env = childEnv(env)
+ p.cmd.Stdout = p.out
+ p.cmd.Stderr = p.out
+ if err := p.cmd.Start(); err != nil {
+ f.t.Fatal(err)
+ }
+ go func() { p.cmd.Wait(); close(p.done) }()
+ f.t.Cleanup(p.kill)
+ return p
+}
+
+func (p *probe) exited() bool {
+ select {
+ case <-p.done:
+ return true
+ default:
+ return false
+ }
+}
+
+func (p *probe) kill() {
+ if p.exited() {
+ return
+ }
+ p.cmd.Process.Kill()
+ <-p.done
+}
+
+// stop is the backend's Stop: SIGTERM to the leader only.
+func (p *probe) stop() {
+ p.t.Helper()
+ if p.exited() {
+ return
+ }
+ p.cmd.Process.Signal(syscall.SIGTERM)
+ select {
+ case <-p.done:
+ case <-time.After(10 * time.Second):
+ p.t.Errorf("moveprobe did not exit on SIGTERM within 10s; output:\n%s", p.out)
+ p.kill()
+ }
+}
+
+func unixClient(sock string) *http.Client {
+ return &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, "unix", sock)
+ },
+ }}
+}
+
+func (p *probe) do(method, path, bearer string, hdr map[string]string) (int, string, error) {
+ req, err := http.NewRequest(method, "http://"+testHost+path, nil)
+ if err != nil {
+ return 0, "", err
+ }
+ if bearer != "" {
+ req.Header.Set("Authorization", "Bearer "+bearer)
+ }
+ for k, v := range hdr {
+ req.Header.Set(k, v)
+ }
+ resp, err := unixClient(p.sock).Do(req)
+ if err != nil {
+ return 0, "", err
+ }
+ defer resp.Body.Close()
+ b, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(b), nil
+}
+
+// wait polls path until it answers want, and fails the test with the last
+// answer and the app's output otherwise.
+func (p *probe) wait(path string, want int) string {
+ p.t.Helper()
+ deadline := time.Now().Add(20 * time.Second)
+ var last string
+ for time.Now().Before(deadline) {
+ code, body, err := p.do(http.MethodGet, path, "", nil)
+ if err == nil && code == want {
+ return body
+ }
+ last = fmt.Sprintf("code %d body %q err %v", code, body, err)
+ if p.exited() {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ p.t.Fatalf("GET %s never answered %d; last: %s; output:\n%s", path, want, last, p.out)
+ return ""
+}
+
+type reportDoc struct {
+ Region string `json:"region"`
+ RegionFrom string `json:"region_from"`
+ Mode string `json:"mode"`
+ Rows []struct {
+ Kind string `json:"kind"`
+ Region string `json:"region"`
+ Count int64 `json:"count"`
+ MinSeq int64 `json:"min_seq"`
+ MaxSeq int64 `json:"max_seq"`
+ } `json:"rows"`
+ Fired []struct {
+ Kind string `json:"kind"`
+ Name string `json:"name"`
+ DueAt int64 `json:"due_at"`
+ Count int64 `json:"count"`
+ } `json:"fired"`
+ FileKeySHA256 string `json:"file_key_sha256"`
+ PlatformKeySHA256 string `json:"platform_key_sha256"`
+ Object struct {
+ Key string `json:"key"`
+ Body string `json:"body"`
+ Error string `json:"error"`
+ } `json:"object"`
+ HasSecret bool `json:"has_secret"`
+ WorkerPID int `json:"worker_pid"`
+ WorkerWritesFailed int64 `json:"worker_writes_failed"`
+ WorkerWritesCommitted int64 `json:"worker_writes_committed"`
+ WorkerFailedInARow int64 `json:"worker_failed_in_a_row"`
+ WorkerLastError string `json:"worker_last_error"`
+ WorkerExit string `json:"worker_exit"`
+ WorkerLastStatusAt int64 `json:"worker_last_status_at"`
+}
+
+func (r reportDoc) count(kind, region string) int64 {
+ for _, row := range r.Rows {
+ if row.Kind == kind && row.Region == region {
+ return row.Count
+ }
+ }
+ return 0
+}
+
+func (p *probe) report() reportDoc {
+ p.t.Helper()
+ code, body, err := p.do(http.MethodGet, "/report", probeSecret, nil)
+ if err != nil || code != http.StatusOK {
+ p.t.Fatalf("/report: code %d err %v body %q; output:\n%s", code, err, body, p.out)
+ }
+ var r reportDoc
+ if err := json.Unmarshal([]byte(body), &r); err != nil {
+ p.t.Fatalf("/report: %v in %q", err, body)
+ }
+ return r
+}
+
+// workerPIDs lists the pids whose command line starts with the test binary
+// and "worker" and names db.
+func workerPIDs(db string) []int {
+ var pids []int
+ ents, _ := os.ReadDir("/proc")
+ for _, e := range ents {
+ pid, err := strconv.Atoi(e.Name())
+ if err != nil {
+ continue
+ }
+ raw, err := os.ReadFile(filepath.Join("/proc", e.Name(), "cmdline"))
+ if err != nil {
+ continue
+ }
+ args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
+ if len(args) < 2 || args[0] != binPath || args[1] != "worker" || !strings.Contains(string(raw), db) {
+ continue
+ }
+ if alive(pid) {
+ pids = append(pids, pid)
+ }
+ }
+ return pids
+}
+
+// alive is true while pid exists and is neither a zombie nor dead.
+func alive(pid int) bool {
+ raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
+ if err != nil {
+ return false
+ }
+ i := strings.LastIndexByte(string(raw), ')')
+ if i < 0 || i+2 >= len(raw) {
+ return false
+ }
+ state := raw[i+2]
+ return state != 'Z' && state != 'X'
+}
+
+func killWorkers(db string) {
+ for _, pid := range workerPIDs(db) {
+ syscall.Kill(pid, syscall.SIGKILL)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for len(workerPIDs(db)) > 0 && time.Now().Before(deadline) {
+ time.Sleep(50 * time.Millisecond)
+ }
+}
+
+func testDB(t *testing.T, db string) *sql.DB {
+ t.Helper()
+ d, err := sql.Open("sqlite", "file:"+db+"?_pragma=busy_timeout(5000)")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return d
+}
+
+func countRows(t *testing.T, db, kind, region string) int64 {
+ t.Helper()
+ d := testDB(t, db)
+ defer d.Close()
+ var n int64
+ if err := d.QueryRow(`SELECT count(*) FROM rows WHERE kind=? AND region=?`, kind, region).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+}
+
+// checkpoint folds the WAL into the database file and closes, so the file
+// is what a move would install.
+func checkpoint(t *testing.T, db string) {
+ t.Helper()
+ d := testDB(t, db)
+ if _, err := d.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+ t.Fatal(err)
+ }
+ d.Close()
+}
+
+// snapshot is every entry under dir with its size and SHA-256: the
+// database file's digest and the files directory's listing, and anything
+// else that appears beside them (a -wal or -shm file included).
+func snapshot(t *testing.T, dir string) string {
+ t.Helper()
+ var lines []string
+ err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ rel, _ := filepath.Rel(dir, p)
+ if d.IsDir() {
+ lines = append(lines, rel+"/")
+ return nil
+ }
+ b, err := os.ReadFile(p)
+ if err != nil {
+ return err
+ }
+ sum := sha256.Sum256(b)
+ lines = append(lines, fmt.Sprintf("%s %d %x", rel, len(b), sum))
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ sort.Strings(lines)
+ return strings.Join(lines, "\n")
+}
+
+func sha(s string) string {
+ sum := sha256.Sum256([]byte(s))
+ return hex.EncodeToString(sum[:])
+}
+
+// fire delivers a timer the way the platform's runner does: the request
+// schedule.BuildRequest builds, sent over the instance's socket.
+func (p *probe) fire(tm registry.Timer, token string) (int, string) {
+ p.t.Helper()
+ req, err := schedule.BuildRequest(registry.Route{Host: testHost}, tm, token, "test")
+ if err != nil {
+ p.t.Fatal(err)
+ }
+ resp, err := unixClient(p.sock).Do(req)
+ if err != nil {
+ p.t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ b, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(b)
+}
+
+// ---- the launch contract --------------------------------------------------
+
+func TestMoveprobeLaunchContract(t *testing.T) {
+ t.Run("the agent's variables and a store endpoint", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ if body := p.wait("/ready", http.StatusOK); !strings.Contains(body, "ready") {
+ t.Errorf("/ready body %q", body)
+ }
+ // The platform probes "/" when moves.ready_path is unset.
+ p.wait("/", http.StatusOK)
+
+ vs, err := f.bucket.ListVersions(context.Background(), bucketName, "probe/")
+ if err != nil || len(vs) != 1 {
+ t.Fatalf("probe objects: %v %v", vs, err)
+ }
+ for _, a := range f.seenAuth() {
+ if !strings.Contains(a, "Credential="+accessKey+"/") {
+ t.Errorf("a store request was not signed with MOVEPROBE_ACCESS_KEY: %q", a)
+ }
+ }
+
+ // /arm-once and /report need the probe's bearer.
+ if code, _, _ := p.do(http.MethodGet, "/report", "", nil); code != http.StatusUnauthorized {
+ t.Errorf("GET /report without a bearer: %d, want 401", code)
+ }
+ if code, _, _ := p.do(http.MethodGet, "/report", "wrong", nil); code != http.StatusUnauthorized {
+ t.Errorf("GET /report with the wrong bearer: %d, want 401", code)
+ }
+ if code, _, _ := p.do(http.MethodPost, "/arm-once?in=600", "", nil); code != http.StatusUnauthorized {
+ t.Errorf("POST /arm-once without a bearer: %d, want 401", code)
+ }
+ before := time.Now().Unix()
+ code, body, err := p.do(http.MethodPost, "/arm-once?in=600", probeSecret, nil)
+ if err != nil || code != http.StatusOK {
+ t.Fatalf("POST /arm-once: %d %q %v", code, body, err)
+ }
+ tm, err := f.reg.GetTimer(testHost, "once-1")
+ if err != nil {
+ t.Fatalf("once-1 not registered through the control socket: %v (arm answered %q)", err, body)
+ }
+ if tm.Kind != schedule.KindOnce || tm.Path != "/once" || tm.DueAt < before+600 || tm.DueAt > time.Now().Unix()+601 {
+ t.Errorf("once-1 = %+v, want a once-timer for /once due in 600s", tm)
+ }
+ if code, body, _ := p.do(http.MethodPost, "/arm-once?in=600", probeSecret, nil); code != http.StatusOK {
+ t.Errorf("the second arm: %d %q", code, body)
+ }
+ if _, err := f.reg.GetTimer(testHost, "once-2"); err != nil {
+ t.Errorf("the second arm is not once-2: %v", err)
+ }
+
+ r := p.report()
+ if r.Region != "uk" || r.RegionFrom != "CARLOS_REGION" {
+ t.Errorf("report region %q from %q, want uk from CARLOS_REGION", r.Region, r.RegionFrom)
+ }
+ if r.PlatformKeySHA256 != sha(platformKey) || r.FileKeySHA256 == "" || !r.HasSecret || r.Object.Body == "" || r.Object.Error != "" {
+ t.Errorf("report = %+v", r)
+ }
+ p.stop()
+ })
+
+ t.Run("no CARLOS_REGION and no control socket", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.envWith(map[string]string{"CARLOS_REGION": "", "CARLOS_CONTROL_SOCKET": ""}))
+ p.wait("/ready", http.StatusOK)
+ code, body, _ := p.do(http.MethodPost, "/arm-once?in=60", probeSecret, nil)
+ if code != http.StatusServiceUnavailable || !strings.Contains(body, "CARLOS_CONTROL_SOCKET") {
+ t.Errorf("/arm-once with no control socket: %d %q, want 503 naming CARLOS_CONTROL_SOCKET", code, body)
+ }
+ p.wait("/ready", http.StatusOK) // unaffected
+ r := p.report()
+ if r.Region != "home" || !strings.Contains(r.RegionFrom, "unset") {
+ t.Errorf("report region %q from %q, want home, with CARLOS_REGION unset", r.Region, r.RegionFrom)
+ }
+ time.Sleep(700 * time.Millisecond)
+ if n := countRows(t, f.db, "bg", "home"); n == 0 {
+ t.Errorf("no bg rows written as region home")
+ }
+ p.stop()
+ })
+
+ t.Run("no MOVEPROBE_ENDPOINT: AWS's own host, the delivered keys", func(t *testing.T) {
+ f := newFixture(t)
+ proxy, caFile, connects := awsLookalike(t, f.store)
+ p := f.start(f.envWith(map[string]string{
+ "MOVEPROBE_ENDPOINT": "",
+ "HTTPS_PROXY": proxy,
+ "SSL_CERT_FILE": caFile,
+ }))
+ p.wait("/ready", http.StatusOK)
+ want := bucketName + ".s3." + storeRegion + ".amazonaws.com:443"
+ got := connects()
+ if len(got) == 0 {
+ t.Fatalf("no store request went to AWS's host")
+ }
+ for _, h := range got {
+ if h != want {
+ t.Errorf("store request to %q, want %q (virtual-hosted, no endpoint)", h, want)
+ }
+ }
+ for _, a := range f.seenAuth() {
+ if !strings.Contains(a, "Credential="+accessKey+"/") {
+ t.Errorf("a store request was not signed with MOVEPROBE_ACCESS_KEY: %q", a)
+ }
+ }
+ p.stop()
+ })
+
+ t.Run("the store keys come only from MOVEPROBE_", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.envWith(map[string]string{
+ "MOVEPROBE_ACCESS_KEY": "",
+ "AWS_ACCESS_KEY_ID": accessKey,
+ "AWS_SECRET_ACCESS_KEY": "x",
+ "AWS_ENDPOINT_URL": f.storeURL,
+ "AWS_ENDPOINT_URL_S3": f.storeURL,
+ }))
+ body := p.wait("/ready", http.StatusInternalServerError)
+ if !strings.Contains(body, "MOVEPROBE_ACCESS_KEY") {
+ t.Errorf("/ready body %q does not name MOVEPROBE_ACCESS_KEY", body)
+ }
+ if n := f.storeReq.Load(); n != 0 {
+ t.Errorf("%d store requests made without the binding's keys", n)
+ }
+ p.stop()
+ })
+
+ t.Run("the sealed secret is required", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.envWith(map[string]string{"MOVEPROBE_SECRET": ""}))
+ body := p.wait("/ready", http.StatusInternalServerError)
+ if !strings.Contains(body, "MOVEPROBE_SECRET") {
+ t.Errorf("/ready body %q does not name MOVEPROBE_SECRET", body)
+ }
+ p.stop()
+ })
+}
+
+// awsLookalike stands in for S3's regional host: an HTTPS proxy whose
+// CONNECT tunnels end at a TLS server holding a certificate for
+// *.s3.<region>.amazonaws.com, signed by a CA written to the returned file.
+// The tunnelled requests are virtual-hosted; they reach store path-style.
+func awsLookalike(t *testing.T, store http.Handler) (proxyURL, caFile string, connects func() []string) {
+ t.Helper()
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tmpl := &x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{CommonName: "moveprobe test s3"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ DNSNames: []string{"*.s3." + storeRegion + ".amazonaws.com", "s3." + storeRegion + ".amazonaws.com"},
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ BasicConstraintsValid: true,
+ IsCA: true,
+ }
+ der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ caFile = filepath.Join(t.TempDir(), "ca.pem")
+ if err := os.WriteFile(caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ tlsCfg := &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}}
+
+ vhost := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ bucket, _, _ := strings.Cut(r.Host, ".")
+ r.URL.Path = "/" + bucket + r.URL.Path
+ r.URL.RawPath = ""
+ store.ServeHTTP(w, r)
+ })
+ var mu sync.Mutex
+ var hosts []string
+ proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodConnect {
+ http.Error(w, "CONNECT only", http.StatusMethodNotAllowed)
+ return
+ }
+ mu.Lock()
+ hosts = append(hosts, r.Host)
+ mu.Unlock()
+ conn, brw, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ return
+ }
+ brw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n")
+ brw.Flush()
+ srv := &http.Server{Handler: vhost}
+ srv.Serve(&oneConn{c: tls.Server(&bufferedConn{Conn: conn, r: brw.Reader}, tlsCfg)})
+ }))
+ t.Cleanup(proxy.Close)
+ return proxy.URL, caFile, func() []string {
+ mu.Lock()
+ defer mu.Unlock()
+ return append([]string(nil), hosts...)
+ }
+}
+
+type bufferedConn struct {
+ net.Conn
+ r *bufio.Reader
+}
+
+func (b *bufferedConn) Read(p []byte) (int, error) { return b.r.Read(p) }
+
+// oneConn is a listener that yields one connection; Serve returns once it
+// has been handed out and the connection is served on its own goroutine.
+type oneConn struct {
+ c net.Conn
+ used bool
+}
+
+func (o *oneConn) Accept() (net.Conn, error) {
+ if o.used {
+ return nil, io.EOF
+ }
+ o.used = true
+ return o.c, nil
+}
+func (o *oneConn) Close() error { return nil }
+func (o *oneConn) Addr() net.Addr { return o.c.LocalAddr() }
+
+// ---- the forked worker ------------------------------------------------------
+
+func TestMoveprobeWorkerOutlivesMain(t *testing.T) {
+ t.Parallel() // a minute of waiting for the worker's own exit
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+ r := p.report()
+ if r.WorkerPID <= 0 || !alive(r.WorkerPID) {
+ t.Fatalf("no live worker in the report: %d", r.WorkerPID)
+ }
+ if got := workerPIDs(f.db); len(got) != 1 || got[0] != r.WorkerPID {
+ t.Fatalf("worker processes %v, report says %d", got, r.WorkerPID)
+ }
+
+ // The main dies outright. Its stdout and stderr are pipes; a worker
+ // still holding them would keep Wait from returning (the backend's tee
+ // waits up to its drain grace for exactly that).
+ killed := time.Now()
+ p.cmd.Process.Kill()
+ select {
+ case <-p.done:
+ case <-time.After(2 * time.Second):
+ t.Fatalf("the main's Wait did not return within 2s of its death: the worker holds its stdio")
+ }
+
+ n0 := countRows(t, f.db, "worker", "uk")
+ time.Sleep(3 * time.Second)
+ if !alive(r.WorkerPID) {
+ t.Fatalf("the worker died with the main")
+ }
+ if n1 := countRows(t, f.db, "worker", "uk"); n1 < n0+2 {
+ t.Fatalf("the worker stopped writing after the main died: %d then %d rows", n0, n1)
+ }
+
+ // It exits on its own about a minute after its parent has gone.
+ for alive(r.WorkerPID) && time.Since(killed) < 80*time.Second {
+ time.Sleep(250 * time.Millisecond)
+ }
+ gone := time.Since(killed)
+ if alive(r.WorkerPID) {
+ t.Fatalf("the worker is still alive %s after the main died", gone.Round(time.Second))
+ }
+ if gone < 59*time.Second || gone > 70*time.Second {
+ t.Errorf("the worker exited %s after the main died, want about 60s", gone.Round(time.Second))
+ }
+}
+
+// ---- the failure switch ---------------------------------------------------
+
+func TestReadyFailsWhereTold(t *testing.T) {
+ f := newFixture(t)
+
+ // The installed state: a normal boot in uk, stopped, its worker killed
+ // (the stop evidence does that on a real box) and the WAL folded in.
+ seed := f.start(f.env)
+ seed.wait("/ready", http.StatusOK)
+ time.Sleep(time.Second)
+ seed.stop()
+ killWorkers(f.db)
+ checkpoint(t, f.db)
+ installed := snapshot(t, f.dataDir)
+ if !strings.Contains(installed, "h1/instance.key") || !strings.Contains(installed, "h1.db ") {
+ t.Fatalf("the seed boot left no database or key file:\n%s", installed)
+ }
+ timers, _ := f.reg.TimersFor(testHost)
+
+ t.Run("before-write writes nothing", func(t *testing.T) {
+ p := f.start(f.envWith(map[string]string{"CARLOS_REGION": "ie", "MOVEPROBE_FAIL_READY_IN": "ie"}))
+ p.wait("/ready", http.StatusServiceUnavailable)
+ p.wait("/", http.StatusServiceUnavailable)
+ if code, _, _ := p.do(http.MethodPost, "/arm-once?in=60", probeSecret, nil); code != http.StatusServiceUnavailable {
+ t.Errorf("/arm-once in before-write: %d, want 503", code)
+ }
+ code, _ := p.fire(registry.Timer{Host: testHost, Name: "every-minute", Kind: schedule.KindSchedule, Path: "/tick", DueAt: time.Now().Unix()}, adminToken)
+ if code != http.StatusServiceUnavailable {
+ t.Errorf("/tick in before-write: %d, want 503", code)
+ }
+ code, _ = p.fire(registry.Timer{Host: testHost, Name: "once-9", Kind: schedule.KindOnce, Path: "/once", DueAt: time.Now().Unix()}, adminToken)
+ if code != http.StatusServiceUnavailable {
+ t.Errorf("/once in before-write: %d, want 503", code)
+ }
+ time.Sleep(2 * time.Second)
+ if w := workerPIDs(f.db); len(w) != 0 {
+ t.Errorf("before-write forked a worker: %v", w)
+ }
+ p.stop()
+ if got := snapshot(t, f.dataDir); got != installed {
+ t.Errorf("before-write changed the data directory\ninstalled:\n%s\nafter:\n%s", installed, got)
+ }
+ if got, _ := f.reg.TimersFor(testHost); len(got) != len(timers) {
+ t.Errorf("before-write changed the timers: %v then %v", timers, got)
+ }
+ })
+
+ t.Run("after-write writes one bg row, then fails", func(t *testing.T) {
+ p := f.start(f.envWith(map[string]string{"CARLOS_REGION": "ie", "MOVEPROBE_FAIL_READY_IN": "ie,after-write"}))
+ p.wait("/ready", http.StatusServiceUnavailable)
+ p.wait("/", http.StatusServiceUnavailable)
+ time.Sleep(2 * time.Second)
+ r := p.report()
+ if r.Mode != "after-write" {
+ t.Errorf("mode %q, want after-write", r.Mode)
+ }
+ if n := r.count("bg", "ie"); n != 1 {
+ t.Errorf("after-write wrote %d bg rows in ie, want exactly 1", n)
+ }
+ if n := r.count("worker", "ie"); n < 1 {
+ t.Errorf("after-write's worker wrote no rows in ie")
+ }
+ p.stop()
+ killWorkers(f.db)
+ })
+
+ t.Run("another region's switch does not apply", func(t *testing.T) {
+ p := f.start(f.envWith(map[string]string{"MOVEPROBE_FAIL_READY_IN": "ie"}))
+ p.wait("/ready", http.StatusOK)
+ p.stop()
+ killWorkers(f.db)
+ })
+
+ t.Run("a box with no region is home", func(t *testing.T) {
+ checkpoint(t, f.db)
+ before := snapshot(t, f.dataDir)
+ p := f.start(f.envWith(map[string]string{"CARLOS_REGION": "", "MOVEPROBE_FAIL_READY_IN": "home"}))
+ p.wait("/ready", http.StatusServiceUnavailable)
+ time.Sleep(time.Second)
+ p.stop()
+ if got := snapshot(t, f.dataDir); got != before {
+ t.Errorf("before-write at home changed the data directory")
+ }
+ })
+}
+
+// ---- the report -----------------------------------------------------------
+
+func TestReportCountsRowsByRegion(t *testing.T) {
+ f := newFixture(t)
+ uk := f.start(f.env)
+ uk.wait("/ready", http.StatusOK)
+
+ code, body, _ := uk.do(http.MethodPost, "/arm-once?in=1", probeSecret, nil)
+ if code != http.StatusOK {
+ t.Fatalf("/arm-once: %d %q", code, body)
+ }
+ once, err := f.reg.GetTimer(testHost, "once-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A tick and the once-timer, delivered as the runner delivers them.
+ tick := registry.Timer{Host: testHost, Name: "every-minute", Kind: schedule.KindSchedule, Path: "/tick", DueAt: 1_700_000_040}
+ if code, body := uk.fire(tick, adminToken); code != http.StatusOK {
+ t.Fatalf("/tick: %d %q", code, body)
+ }
+ if code, body := uk.fire(*once, adminToken); code != http.StatusOK {
+ t.Fatalf("/once: %d %q", code, body)
+ }
+ // Neither fires without the platform's token.
+ if code, _ := uk.fire(tick, "wrong"); code != http.StatusUnauthorized {
+ t.Errorf("/tick with the wrong token: %d, want 401", code)
+ }
+ if code, _ := uk.fire(*once, ""); code != http.StatusUnauthorized {
+ t.Errorf("/once with no token: %d, want 401", code)
+ }
+ time.Sleep(1500 * time.Millisecond)
+ ukReport := uk.report()
+ uk.stop()
+ killWorkers(f.db)
+
+ // The same database, now in ie.
+ ie := f.start(f.envWith(map[string]string{"CARLOS_REGION": "ie"}))
+ ie.wait("/ready", http.StatusOK)
+ time.Sleep(1500 * time.Millisecond)
+ r := ie.report()
+ ie.stop()
+
+ if r.Region != "ie" {
+ t.Errorf("region %q, want ie", r.Region)
+ }
+ for _, c := range []struct {
+ kind, region string
+ min int64
+ }{{"bg", "uk", 2}, {"bg", "ie", 2}, {"worker", "uk", 1}, {"worker", "ie", 1}, {"tick", "uk", 1}, {"once", "uk", 1}} {
+ if n := r.count(c.kind, c.region); n < c.min {
+ t.Errorf("%s rows in %s: %d, want at least %d (rows %+v)", c.kind, c.region, n, c.min, r.Rows)
+ }
+ }
+ if n := r.count("tick", "ie") + r.count("once", "ie"); n != 0 {
+ t.Errorf("%d timer rows in ie, none fired there", n)
+ }
+ // No background row is missing: the sequence runs on across boots.
+ var bgCount, bgMax int64
+ for _, row := range r.Rows {
+ if row.Kind == "bg" {
+ bgCount += row.Count
+ bgMax = max(bgMax, row.MaxSeq)
+ }
+ }
+ if bgCount != bgMax {
+ t.Errorf("bg rows %d, highest seq %d: a row is missing or repeated", bgCount, bgMax)
+ }
+
+ var gotOnce, gotTick bool
+ for _, fr := range r.Fired {
+ switch {
+ case fr.Kind == "once" && fr.Name == "once-1":
+ gotOnce = fr.Count == 1 && fr.DueAt == once.DueAt
+ case fr.Kind == "tick" && fr.Name == "every-minute":
+ gotTick = fr.Count == 1 && fr.DueAt == 1_700_000_040
+ }
+ }
+ if !gotOnce || !gotTick {
+ t.Errorf("fired = %+v, want once-1 once at %d and every-minute once at 1700000040", r.Fired, once.DueAt)
+ }
+ if r.FileKeySHA256 == "" || r.FileKeySHA256 != ukReport.FileKeySHA256 {
+ t.Errorf("file key fingerprint %q in ie, %q in uk", r.FileKeySHA256, ukReport.FileKeySHA256)
+ }
+ if r.PlatformKeySHA256 != sha(platformKey) {
+ t.Errorf("platform key fingerprint %q, want %q", r.PlatformKeySHA256, sha(platformKey))
+ }
+ if !r.HasSecret || r.Object.Body == "" || !strings.HasPrefix(r.Object.Key, "probe/") {
+ t.Errorf("report = %+v", r)
+ }
+}
+
+// ---- the instance key -----------------------------------------------------
+
+func TestReadyChecksBothKeys(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+ p.stop()
+ killWorkers(f.db)
+
+ keyFile := filepath.Join(strings.TrimSuffix(f.db, ".db"), "instance.key")
+ orig, err := os.ReadFile(keyFile)
+ if err != nil {
+ t.Fatalf("first boot wrote no key file beside the database: %v", err)
+ }
+ if b, err := hex.DecodeString(strings.TrimSpace(string(orig))); err != nil || len(b) != 32 {
+ t.Errorf("instance.key is not 32 bytes of hex: %q", orig)
+ }
+
+ check := func(name string, env map[string]string, want int, says string) {
+ t.Helper()
+ p := f.start(env)
+ body := p.wait("/ready", want)
+ if says != "" && !strings.Contains(body, says) {
+ t.Errorf("%s: /ready body %q, want it to say %q", name, body, says)
+ }
+ p.stop()
+ killWorkers(f.db)
+ }
+
+ if err := os.WriteFile(keyFile, []byte(strings.Repeat("ab", 32)), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ check("changed file", f.env, http.StatusInternalServerError, "key mismatch")
+ if err := os.Remove(keyFile); err != nil {
+ t.Fatal(err)
+ }
+ check("missing file", f.env, http.StatusInternalServerError, "key mismatch")
+ if err := os.WriteFile(keyFile, orig, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ check("restored file", f.env, http.StatusOK, "")
+ check("platform key changed", f.envWith(map[string]string{"CARLOS_INSTANCE_KEY": "another-key"}), http.StatusInternalServerError, "CARLOS_INSTANCE_KEY")
+ check("platform key absent", f.envWith(map[string]string{"CARLOS_INSTANCE_KEY": ""}), http.StatusInternalServerError, "CARLOS_INSTANCE_KEY")
+ check("both restored", f.env, http.StatusOK, "")
+}
+
+// ---- write failures -------------------------------------------------------
+
+// failInserts makes every insert of kind into rows abort, through a trigger
+// the test adds to the instance's own database: a real write failure, with
+// nothing in the app to switch it on.
+func failInserts(t *testing.T, db, kind string) {
+ t.Helper()
+ d := testDB(t, db)
+ defer d.Close()
+ q := fmt.Sprintf(`CREATE TRIGGER fail_%[1]s BEFORE INSERT ON rows WHEN NEW.kind = '%[1]s' BEGIN SELECT RAISE(ABORT, 'forced %[1]s failure'); END`, kind)
+ if _, err := d.Exec(q); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func allowInserts(t *testing.T, db, kind string) {
+ t.Helper()
+ d := testDB(t, db)
+ defer d.Close()
+ if _, err := d.Exec(`DROP TRIGGER fail_` + kind); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestReadyFollowsTheWorkersLastWrite: /ready is 200 only once a worker row
+// has committed since this start and the worker's latest write succeeded.
+func TestReadyFollowsTheWorkersLastWrite(t *testing.T) {
+ f := newFixture(t)
+ seed := f.start(f.env)
+ seed.wait("/ready", http.StatusOK)
+ seed.stop()
+ killWorkers(f.db)
+
+ // Worker rows from the earlier boot are in the database; none can
+ // commit in this one.
+ failInserts(t, f.db, "worker")
+ p := f.start(f.env)
+ body := p.wait("/ready", http.StatusInternalServerError)
+ if !strings.Contains(body, "worker: no row committed since start") {
+ t.Errorf("/ready before the first worker row: %q", body)
+ }
+ time.Sleep(1500 * time.Millisecond)
+ if code, body, _ := p.do(http.MethodGet, "/ready", "", nil); code != http.StatusInternalServerError {
+ t.Errorf("/ready with no worker row yet: %d %q, want 500", code, body)
+ }
+ r := p.report()
+ if r.WorkerWritesFailed == 0 || r.WorkerWritesCommitted != 0 || !strings.Contains(r.WorkerLastError, "forced worker failure") {
+ t.Errorf("report: committed %d, failed %d, last %q", r.WorkerWritesCommitted, r.WorkerWritesFailed, r.WorkerLastError)
+ }
+
+ // The first good write makes it ready.
+ allowInserts(t, f.db, "worker")
+ p.wait("/ready", http.StatusOK)
+
+ // One failed write is enough to fail it.
+ failInserts(t, f.db, "worker")
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ r = p.report()
+ if r.WorkerFailedInARow > 0 || time.Now().After(deadline) {
+ break
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ code, body, _ := p.do(http.MethodGet, "/ready", "", nil)
+ if r.WorkerFailedInARow != 1 || code != http.StatusInternalServerError || !strings.Contains(body, "worker: the last write failed") {
+ t.Errorf("after one failed write (in a row %d): /ready %d %q, want 500", r.WorkerFailedInARow, code, body)
+ }
+
+ // The next good write makes it ready again; the counters stay.
+ allowInserts(t, f.db, "worker")
+ p.wait("/ready", http.StatusOK)
+ if r := p.report(); r.WorkerFailedInARow != 0 || r.WorkerWritesFailed == 0 || r.WorkerWritesCommitted == 0 {
+ t.Errorf("after recovery: committed %d, failed %d, in a row %d", r.WorkerWritesCommitted, r.WorkerWritesFailed, r.WorkerFailedInARow)
+ }
+ p.stop()
+}
+
+func TestAfterWriteExitsWhenItsRowFails(t *testing.T) {
+ f := newFixture(t)
+ seed := f.start(f.env)
+ seed.wait("/ready", http.StatusOK)
+ seed.stop()
+ killWorkers(f.db)
+
+ failInserts(t, f.db, "bg")
+ p := f.start(f.envWith(map[string]string{"CARLOS_REGION": "ie", "MOVEPROBE_FAIL_READY_IN": "ie,after-write"}))
+ select {
+ case <-p.done:
+ case <-time.After(15 * time.Second):
+ t.Fatalf("after-write kept running without its bg row; output:\n%s", p.out)
+ }
+ if code := p.cmd.ProcessState.ExitCode(); code == 0 {
+ t.Errorf("after-write exited 0 without its bg row; output:\n%s", p.out)
+ }
+ if !strings.Contains(p.out.String(), "forced bg failure") {
+ t.Errorf("the exit does not say why; output:\n%s", p.out)
+ }
+ if w := workerPIDs(f.db); len(w) != 0 {
+ t.Errorf("after-write forked a worker before its row committed: %v", w)
+ }
+ if n := countRows(t, f.db, "bg", "ie"); n != 0 {
+ t.Errorf("%d bg rows in ie", n)
+ }
+}
+
+// ---- report query failures ------------------------------------------------
+
+// failDriver is a database/sql driver whose query matching the DSN (a
+// substring) yields one good row and then fails, as a read that breaks part
+// way through would. Every other query yields no rows.
+type failDriver struct{}
+
+func (failDriver) Open(failOn string) (driver.Conn, error) { return failConn{failOn}, nil }
+
+type failConn struct{ failOn string }
+
+func (c failConn) Prepare(q string) (driver.Stmt, error) { return failStmt{c.failOn, q}, nil }
+func (failConn) Close() error { return nil }
+func (failConn) Begin() (driver.Tx, error) { return nil, errors.New("no transactions") }
+
+type failStmt struct{ failOn, q string }
+
+func (failStmt) Close() error { return nil }
+func (failStmt) NumInput() int { return -1 }
+func (failStmt) Exec([]driver.Value) (driver.Result, error) {
+ return nil, errors.New("no writes")
+}
+func (s failStmt) Query([]driver.Value) (driver.Rows, error) {
+ cols := []string{"kind", "region", "count", "min", "max"}
+ row := []driver.Value{"bg", "uk", int64(3), int64(1), int64(3)}
+ if strings.Contains(s.q, "IN ('tick', 'once')") {
+ cols = []string{"kind", "name", "due_at", "count"}
+ row = []driver.Value{"once", "once-1", int64(1_700_000_000), int64(1)}
+ }
+ return &failRows{cols: cols, row: row, fail: strings.Contains(s.q, s.failOn)}, nil
+}
+
+type failRows struct {
+ cols []string
+ row []driver.Value
+ fail bool
+ n int
+}
+
+func (r *failRows) Columns() []string { return r.cols }
+func (r *failRows) Close() error { return nil }
+func (r *failRows) Next(dest []driver.Value) error {
+ r.n++
+ switch {
+ case !r.fail:
+ return io.EOF
+ case r.n == 1:
+ copy(dest, r.row)
+ return nil
+ default:
+ return errors.New("forced read failure")
+ }
+}
+
+func init() { sql.Register("moveprobe-fail", failDriver{}) }
+
+func TestReportNeverAnswersPartial(t *testing.T) {
+ report := func(t *testing.T, db *sql.DB) (int, string) {
+ t.Helper()
+ a := &app{db: db, filesDir: t.TempDir(), region: "uk", mode: modeNormal, storeErr: errors.New("store: not under test")}
+ rec := httptest.NewRecorder()
+ a.handleReport(rec, httptest.NewRequest(http.MethodGet, "/report", nil))
+ return rec.Code, rec.Body.String()
+ }
+ for _, failOn := range []string{"GROUP BY kind, region", "IN ('tick', 'once')"} {
+ t.Run("rows fail part way: "+failOn, func(t *testing.T) {
+ db, err := sql.Open("moveprobe-fail", failOn)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ code, body := report(t, db)
+ if code != http.StatusInternalServerError || !strings.Contains(body, "forced read failure") {
+ t.Errorf("/report: %d %q, want 500 naming the read failure", code, body)
+ }
+ })
+ }
+ t.Run("a closed database", func(t *testing.T) {
+ db := testDB(t, filepath.Join(t.TempDir(), "closed.db"))
+ db.Close()
+ if code, body := report(t, db); code != http.StatusInternalServerError {
+ t.Errorf("/report on a closed database: %d %q, want 500", code, body)
+ }
+ })
+ t.Run("nothing fails", func(t *testing.T) {
+ db, err := sql.Open("moveprobe-fail", "matches no query")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ if code, body := report(t, db); code != http.StatusOK {
+ t.Errorf("/report: %d %q, want 200", code, body)
+ }
+ })
+}
+
+func TestArmOnceReusesOneControlConnection(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+ for i := 0; i < 3; i++ {
+ if code, body, _ := p.do(http.MethodPost, "/arm-once?in=600", probeSecret, nil); code != http.StatusOK {
+ t.Fatalf("arm %d: %d %q", i+1, code, body)
+ }
+ }
+ if n := f.controlConns.Load(); n != 1 {
+ t.Errorf("three arms opened %d connections to the control socket, want 1", n)
+ }
+ p.stop()
+}
+
+// TestReadyFailsWhenTheWorkerStops: a worker that has died, or has sent no
+// status for 3 s, fails readiness, and the report says so.
+func TestReadyFailsWhenTheWorkerStops(t *testing.T) {
+ t.Run("the worker dies after its first good write", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+ r := p.report()
+ if r.WorkerWritesCommitted == 0 || r.WorkerExit != "" || r.WorkerLastStatusAt == 0 {
+ t.Fatalf("before: committed %d, exit %q, last status %d", r.WorkerWritesCommitted, r.WorkerExit, r.WorkerLastStatusAt)
+ }
+ if err := syscall.Kill(r.WorkerPID, syscall.SIGKILL); err != nil {
+ t.Fatal(err)
+ }
+ body := p.wait("/ready", http.StatusInternalServerError)
+ if !strings.Contains(body, "worker: exited") {
+ t.Errorf("/ready after the worker died: %q", body)
+ }
+ if r := p.report(); !strings.Contains(r.WorkerExit, "exited") {
+ t.Errorf("report worker_exit %q after the worker died", r.WorkerExit)
+ }
+ p.stop()
+ })
+
+ t.Run("the worker sends no status for 3 s", func(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+ pid := p.report().WorkerPID
+ if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil {
+ t.Fatal(err)
+ }
+ stopped := time.Now()
+ body := p.wait("/ready", http.StatusInternalServerError)
+ if waited := time.Since(stopped); waited < 2*time.Second {
+ t.Errorf("/ready failed %s after the worker stopped, before 3 s without status", waited)
+ }
+ if !strings.Contains(body, "worker: no status for") {
+ t.Errorf("/ready with a stalled worker: %q", body)
+ }
+ syscall.Kill(pid, syscall.SIGCONT)
+ p.wait("/ready", http.StatusOK)
+ p.stop()
+ })
+}
+
+// TestReadyCoalescesTheStoreCheck: anonymous /ready traffic costs at most one
+// store check per 5 s, however many requests arrive, and requests that land
+// while a check is running share its result.
+func TestReadyCoalescesTheStoreCheck(t *testing.T) {
+ f := newFixture(t)
+ p := f.start(f.env)
+ p.wait("/ready", http.StatusOK)
+
+ burst := func() {
+ t.Helper()
+ var wg sync.WaitGroup
+ codes := make(chan int, 50)
+ for i := 0; i < 50; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ code, _, err := p.do(http.MethodGet, "/ready", "", nil)
+ if err != nil {
+ code = 0
+ }
+ codes <- code
+ }()
+ }
+ wg.Wait()
+ close(codes)
+ for c := range codes {
+ if c != http.StatusOK {
+ t.Errorf("a /ready in the burst answered %d", c)
+ }
+ }
+ }
+
+ // A slow store, so the burst lands while a check is in flight.
+ f.storeDelay.Store(300)
+ time.Sleep(5100 * time.Millisecond) // past any cached result
+ n0 := f.storeReq.Load()
+ burst()
+ n1 := f.storeReq.Load()
+ if n1-n0 > 1 {
+ t.Errorf("50 concurrent /ready calls made %d store requests, want at most 1", n1-n0)
+ }
+ burst()
+ if n2 := f.storeReq.Load(); n2 != n1 {
+ t.Errorf("a second burst inside the 5 s window made %d store requests, want 0", n2-n1)
+ }
+ time.Sleep(5100 * time.Millisecond)
+ n3 := f.storeReq.Load()
+ burst()
+ if n4 := f.storeReq.Load(); n4-n3 > 1 {
+ t.Errorf("the next window's burst made %d store requests, want at most 1", n4-n3)
+ }
+ p.stop()
+}
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-7-rehearsal-runbook.md b/docs/superpowers/plans/2026-10-06-moves-phase1-7-rehearsal-runbook.md
index 2436d880e..3a7afd9b1 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-7-rehearsal-runbook.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-7-rehearsal-runbook.md
@@ -8,11 +8,11 @@
**Tech Stack:** Go, `net/http`, `database/sql` with modernc sqlite, the platform's `carlos` CLI, `tofu` for nothing (no infra change in this plan beyond §15 step 5, which is applied as scoped work by Paul).
-**Spec:** rev 7.18 §14.3, §14.4, §15. Index: `2026-10-06-moves-phase1-0-index.md`. Plans 1 to 6 merged and rolled per the platform roll recipe.
+**Spec:** rev 7.23 §14.3, §14.4, §15. Index: `2026-10-06-moves-phase1-0-index.md`. Plans 1 to 6 are merged, not rolled, and not complete (preflight N1): no run in this plan executes on main until claim Plan 12 and its moves dependants are merged, released and rolled.
## Global Constraints
-- Gate, cross-compile, sandboxed `GOCACHE`; tests serial and niced.
+- Gate, cross-compile, the default Go build cache; tests serial and niced.
- Rollout order (§15): the dormant release first (consoles, then every box), M17's `sibling_staging_arns` and the versioned-object grants (§15 step 5) applied by Paul, then `control/moves-protocol.json` written create-only **before** any protected production rehearsal (§11, joint N12), then the rehearsals in order: dev IE to uk done; dev forced clean rollback; dev forced dirty rollback; dev declare-lost; the same on the flagship with scoped approval; a disposable ips instance in `bdn`; kpmg with Paul's approval.
- A run passes only if: the key verifies, the object reads back, the secret is present, every timer ran exactly once around the cut, no background row is missing, and no copy remains in a live namespace of the region that lost (§14.4).
- `moveprobe`'s readiness-failure switch is an environment variable (`MOVEPROBE_FAIL_READY_IN=<region>[,after-write]`) delivered through the config bundle, so a rehearsal flips it with `carlos env set`, never a rebuild.
@@ -43,12 +43,12 @@
**Files:** create `cmd/moveprobe/main.go`, `cmd/moveprobe/main_test.go`, `cmd/moveprobe/README.md`.
**Interfaces (the app's behaviour):**
-- The exec backend's launch contract is **flags, not variables** (`backend_exec.go:942`: `exec.Command(bin, "--socket", rt.Socket, "--db", dbPath)`): `moveprobe --socket <unix path> --db <sqlite path>` listens on the socket and opens the database; the files directory is `strings.TrimSuffix(dbPath, ".db")` (F4, the flat layout). The agent-owned environment adds `CARLOS_REGION`, `CARLOS_CONTROL_SOCKET` and the rest of `agentOwnedEnv`; the app reads those by name. `TestMoveprobeLaunchContract` starts the binary exactly as the backend does (`--socket`, `--db`, the agent variables) and asserts `/ready` answers on the socket.
+- The exec backend's launch contract is **flags, not variables** (`internal/activator/backend_exec.go:1541`, in `spawnChildFrom`: `exec.Command(bin, "--socket", sock, "--db", dbPath)`): `moveprobe --socket <unix path> --db <sqlite path>` listens on the socket and opens the database; the files directory is `strings.TrimSuffix(dbPath, ".db")` (F4, the flat layout). The agent-owned environment is composed by `agentOwnedVars` (`backend_exec.go:1603-1626`): `CARLOS_CONTROL_SOCKET` only when the control listener bound, `CARLOS_PURGE`, `CARLOS_REGION` only when the box has a region, and `CARLOS_HOST`. `CARLOS_ADMIN_TOKEN` and `CARLOS_INSTANCE_KEY` come from `<host>.env`, not the composer. The app reads those by name and treats a missing `CARLOS_REGION` as region `home` (preflight N7). `TestMoveprobeLaunchContract` starts the binary exactly as the backend does (`--socket`, `--db`, the agent variables) and asserts `/ready` answers on the socket.
- First boot: writes `files/instance.key` (32 random bytes, hex) beside the database; every boot: verifies the file exists and matches the row `meta(key='instance_key')` in the database; mismatch → the readiness path answers 500 with `key mismatch`.
- Requires `MOVEPROBE_BUCKET`, `MOVEPROBE_REGION`, `MOVEPROBE_ENDPOINT`, `MOVEPROBE_ACCESS_KEY`, `MOVEPROBE_SECRET_KEY` (an `objstore` binding with prefix `MOVEPROBE`) and writes `probe/<boot id>.txt` at boot, reading it back on `/ready`; requires `MOVEPROBE_SECRET` (a sealed secret) and answers 500 on `/ready` when absent.
- Registers, through the instance control socket (`CARLOS_CONTROL_SOCKET`), a declared schedule `every-minute` → `/tick` and, on `POST /arm-once?in=<seconds>`, a once-timer `once-<n>` → `/once`; `/tick` and `/once` insert rows `(kind, name, due_at, fired_at, region)`.
- A background loop inserts a row `(kind='bg', seq, region)` every 500 ms; a forked worker (`exec.Command(os.Args[0], "worker")` with `Setsid`) inserts `(kind='worker', seq, region)` every second and ignores SIGTERM for 30 s (Review Focus 1).
-- `MOVEPROBE_FAIL_READY_IN=<region>[,after-write]`: in the named region, the **before-write** mode (no suffix) suppresses every mutation on this box from the first instruction: the background loop, the forked worker, the `/tick` and `/once` handlers and the control-socket registrations all return without writing, and `/ready` answers 500, so the target's database, files and timers digests stay exactly what was installed and the rollback is clean (the backend's post-spawn readiness wait at `backend_exec.go:989` would otherwise see rows written before the refusal); the **after-write** mode writes one `bg` row (and lets the worker write) and then fails `/ready`, so the rollback is dirty with a known row to find on the source.
+- `MOVEPROBE_FAIL_READY_IN=<region>[,after-write]`: in the named region, the **before-write** mode (no suffix) suppresses every mutation on this box from the first instruction: the background loop, the forked worker, the `/tick` and `/once` handlers and the control-socket registrations all return without writing, and `/ready` answers 500, so the target's database, files and timers digests stay exactly what was installed and the rollback is clean (the readiness checks after a spawn, the wake's `waitHealthyCtx` at `backend_exec.go:1335` (GET `/`, any status below 500, `waitHealthy` at `:2344`) and the claim probe's `claimProbeSocketReady` at `internal/host/claimready_linux.go:23-41` (2xx only), would otherwise see rows written before the refusal); the **after-write** mode writes one `bg` row (and lets the worker write) and then fails `/ready`, so the rollback is dirty with a known row to find on the source.
- `/ready`: 200 when the key verifies, the object reads back and the secret is present, and the failure switch does not apply.
- `/report`: JSON with the row counts per kind and region, the once-timers fired (name, count, due_at), the key fingerprint, the object body, `has_secret`.
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index a3b6ad52e..7bee097d0 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -412,7 +412,7 @@ Paul's ruling of 2026-10-11 (N1.3 in the Plan 7 preflight): the source refuses t
| id | file | text | status |
|---|---|---|---|
-| MV-N440 | internal/host/movesreport.go | this box cannot land the move or roll it back yet | pending |
-| MV-N441 | internal/provision/provision.go | the target box cannot land the move or roll it back yet | pending |
-| MV-N442 | internal/provision/provision.go | the source box cannot recover the move if the cut fails | pending |
-| MV-N443 | move.go | The cut did not start: <reason>. | pending |
+| MV-N440 | internal/host/movesreport.go | this box cannot land the move or roll it back yet | approved |
+| MV-N441 | internal/provision/provision.go | the target box cannot land the move or roll it back yet | approved |
+| MV-N442 | internal/provision/provision.go | the source box cannot recover the move if the cut fails | approved |
+| MV-N443 | move.go | The cut did not start: <reason>. | approved |
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-rehearsal.md b/docs/superpowers/plans/2026-10-06-moves-phase1-rehearsal.md
new file mode 100644
index 000000000..87806b515
--- /dev/null
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-rehearsal.md
@@ -0,0 +1,1085 @@
+# Moves Phase 1: the rehearsal runbook
+
+This is the checklist an operator follows to rehearse Phase 1 moves (spec
+§14.4, rollout §15). It covers seven runs in the spec's order: four on dev,
+one on the flagship with the probe app, one on a disposable customer-shaped
+instance, and one on a real customer instance.
+
+**No run in this runbook can execute today.** Every run waits for P-0
+below. Each run also has its own blockers, listed at its top. Never run one
+without Paul's go in the amadan thread. Steps that change a deployment
+(publishing the floor, switching moves on, stopping a box) each have their
+own **Paul's scoped go** step just before them: a go in the amadan thread
+that names the deployment and exactly what the step changes. A go for one
+step or one run does not cover another.
+
+The probe app is `moveprobe`. How to build it, ship it and use it is in
+[`cmd/moveprobe/README.md`](../../../cmd/moveprobe/README.md). This runbook
+does not repeat it.
+
+Every command, flag, event, field and key below was checked against main,
+and moveprobe's against this branch.
+The Sources section at the end gives file and line for each.
+
+---
+
+## How to read this runbook
+
+### Placeholders
+
+| placeholder | what it is |
+|---|---|
+| `<console>` | the deployment's console origin, `https://...`, as `carlos auth login` stored it |
+| `<ops>` | the ops account's sqid, as `--account` takes it |
+| `<account>`, `<app>` | a customer run's account sqid and app (Runs 6 and 7) |
+| `<host>` | the run's host, `<label>.<instance domain>` |
+| `<deploy-bucket>` | the deployment bucket (`CARLOS_DEPLOYMENT_BUCKET` in a box's `/etc/carlos/host.env`) |
+| `<data-bucket-R>` | region R's data bucket: `data_bucket` in that region's box edges file |
+| `<profile>` | the deployment's named AWS profile. Always name one. There are no default credentials. |
+| `<move>` | the move id the console gave the request (`id` in the flat view) |
+| `<inc>` | the instance incarnation (`detail.move.incarnation`) |
+| `<pr-R>` | the region moveprobe reports on region R's box: `home` if that box has no `CARLOS_REGION` (its edges file has no `placement`), else its `CARLOS_REGION`, the placement code. `<pr-from>` and `<pr-to>` are the run's two. |
+
+### The probe's region is not always the region's name
+
+moveprobe reports the region its box gives it, not the name the CLI and
+the console use. On a home box, which has no `CARLOS_REGION`, it reports
+`home`, while `carlos move` still says `ie` (or whatever
+`CONSOLE_HOME_REGION` is). Before each deployment's first run, write down
+`<pr-R>` for every rehearsal region: read `CARLOS_REGION` in the box's
+`/etc/carlos/host.env`, or `placement` in its edges file, and confirm with
+[report]'s `region` and `region_from` on any moveprobe host there. Every
+[report] region check and every `MOVEPROBE_FAIL_READY_IN` value uses
+`<pr-R>`, never the CLI's name.
+
+### Read paths
+
+Each check says where to read it.
+
+- **[CLI]**: the sentence `carlos move` prints and its exit code. Exit codes:
+ 0 done; 3 rolled back or aborted; 4 waiting (ready for a manual cut,
+ stuck, a release past its deadline, relocating, or an outcome not
+ recorded); 5 refused; 1 the console could not be reached or read; 2 a
+ usage error.
+- **[detail]**: the operator's detail answer. No `carlos` command prints it
+ yet, so read it with your own login token. Set `ACCT` and `APP` for the
+ run: `<ops>` and `moveprobe` for Runs 1 to 5, `<account>` and `<app>`
+ for Runs 6 and 7.
+
+ ```sh
+ CONSOLE=<console>
+ ACCT=<ops> APP=moveprobe
+ TOKEN=$(jq -r --arg c "$CONSOLE" '.credentials[] | select(.console == $c) | .token' ~/.carlos/credentials)
+ detail() { curl -fsS -H "Authorization: Bearer $TOKEN" "$CONSOLE/api/cli/apps/$ACCT/$APP/instances/$1/move?detail=1"; }
+ detail <host> | jq .
+ ```
+
+ The flat fields (`state`, `phase`, `id`, `last`) sit at the top. The
+ rest is under `detail`: `move` (the record's journal), `claim`,
+ `target_report`, `source_report`, `evidence`, `log` (keys only),
+ `timings` (after the close) and `region`. **`detail.move` and
+ `detail.evidence` are gone once the move closes**, and a mutation waits
+ until it closes by default. So every run that needs them starts the
+ capture loop below before its cut and reads them from the capture file
+ afterwards.
+- **[capture]**: every detail answer of one move, saved as it happens.
+ Start it in a second terminal once the request has reached ready, before
+ the cut. It stops by itself once no move is in flight. With a third
+ argument it also saves, each time round, the host claim's
+ `obligations` from the deployment bucket (`control/hosts/<host>.json`,
+ plain JSON). The detail answer does not carry them, and they are gone
+ once the move closes, so this is the only way to keep a park's horizon
+ and retirement id. Run 3 needs it (R-8):
+
+ ```sh
+ capture() { # capture <host> <detail-file> [<claim-file>]
+ while :; do
+ if d=$(detail "$1"); then
+ printf '%s\n' "$d" >> "$2"
+ if [ -n "$3" ]; then
+ aws s3 cp --profile <profile> "s3://<deploy-bucket>/control/hosts/$1.json" - \
+ | jq -c '.obligations // []' >> "$3"
+ fi
+ [ "$(printf '%s' "$d" | jq -r .state)" = none ] && break
+ fi
+ sleep 2
+ done
+ }
+ capture <host> run-<n>-detail.jsonl # Runs 1, 2, 4, 5
+ capture <host> run-3-detail.jsonl run-3-claim.jsonl # Run 3
+ # later: the last rollback journal the move showed
+ jq -s '[.[] | .detail.move.rollback | select(. != null)] | last' run-<n>-detail.jsonl
+ ```
+
+- **[deploy]**: the deployment bucket, read with `--profile <profile>`.
+ Each move keeps its reports, stop evidence and log under
+ `moves/$ACCT/$APP/<host>/<move>/`: `source.json`, `target.json`,
+ `stopped-source.json`, `stopped-target.json` and `log/<identity>.json`.
+ The claim is `control/hosts/<host>.json`, each box's edges file
+ `control/edges/<box>.json`, each box's identity record
+ `control/boxes/<box>.json`, and the floor `control/moves-protocol.json`.
+
+ ```sh
+ P="moves/$ACCT/$APP/<host>/<move>/"
+ aws s3 ls --profile <profile> "s3://<deploy-bucket>/${P}log/"
+ aws s3 cp --profile <profile> "s3://<deploy-bucket>/${P}log/done.json" - | jq .
+ ```
+
+ The roster, `control/host-actors.json`, is a signed envelope,
+ `{"payload": <base64 JSON>, "sig": ...}`. Decode the payload to read it
+ (this does not check the signature; the console does). A box's
+ termination fence is a separate object,
+ `control/box-fences/<enrollment>.json`, using the enrollment from the
+ roster. No object means no fence:
+
+ ```sh
+ roster() { aws s3 cp --profile <profile> s3://<deploy-bucket>/control/host-actors.json - | jq -r .payload | base64 -d; }
+ roster | jq '.boxes["<box>"] | {state, enrollment, loss_ids, quarantine_revision}'
+ aws s3api head-object --profile <profile> --bucket <deploy-bucket> \
+ --key "control/box-fences/$(roster | jq -r '.boxes["<box>"].enrollment').json" # 404: no fence
+ ```
+
+- **[data]**: a region's data bucket, read with `--profile <profile>`. The
+ instance's replica is under `instances/<host>/` (the box's
+ `CARLOS_HIBERNATE_PREFIX`, `instances` in production), a move's staging
+ lane under `moves/<host>/<move>/`, and parked objects under
+ `parked/<inc>/<host>/<retirement>/` with a `manifest.json`. These buckets
+ are versioned, so list versions, not objects:
+
+ ```sh
+ aws s3api list-object-versions --profile <profile> --bucket <data-bucket-R> --prefix "instances/<host>/"
+ ```
+
+- **[report]**: moveprobe's own report, with the secret from your shell:
+
+ ```sh
+ curl -fsS -H "Authorization: Bearer $MOVEPROBE_SECRET" https://<host>/report | jq .
+ ```
+
+- **[box]**: on a box, the agent's journal (`journalctl -u carlos-edge`),
+ `/etc/carlos/host.env`, and files under the data directory (`/data`).
+
+### Log events
+
+Each transition writes one log object, keyed by its identity. The ones the
+checks use:
+
+| identity (the key) | event (the `event` field) | when |
+|---|---|---|
+| `requested` | `requested` | the request was journaled |
+| `ready` | `ready` | the target's checks passed |
+| `cutting-requested` | `cut_requested` | the cut was asked for |
+| `cutting-<phase>` | `<phase>` | `admitted`, `stopping`, `stopped`, `staged`, `archived` |
+| `cutting-flipped` | `flipped` | the record now names the target |
+| `landing-<phase>` | `<phase>` | the target's landing steps |
+| `probe_attempt-<n>` | `probe_attempt` | the landing's n-th readiness probe |
+| `committed` | `committed` | the landing committed |
+| `committed-opened` | `opened` | the target opened (`offline_s` in its fields) |
+| `committed-source_parked` | `source_parked` | the source parked what it held |
+| `rolling_back` | `rolling_back` | a rollback started (`reason` in its fields) |
+| `rolling_back-target-<phase>` | `<phase>` | `released`, `returning`, `returned` (a dirty rollback only), `cleaning`, `cleaned` |
+| `rolling_back-target-lost` | `declared_lost` | the target was declared lost |
+| `rolling_back-source-<phase>` | `<phase>` | the source's recovery, up to `recovered` and `opened` |
+| `stuck-<epoch>`, `retry-<epoch>` | `stuck`, `retry` | a stuck move and its retry |
+| `done`, `rolled_back`, `aborted` | the same | the close; `done` and `rolled_back` carry the timings |
+
+Every entry's `fields` carry `from`, `to` and, once known, `final_txid`,
+`db_sha256`, `timers_sha256`, `files_plain`, `installed_db_sha256`,
+`release`, `env_report` and `cut_budget_s`. The close's timings are
+`prepare_s`, `cut_s`, `cut_budget_s`, `land_s`, `land_budget_s`,
+`offline_s`, `rollback_s` and `release_budget_s`. There is no `stray`
+event and no `dirty` field in the log.
+
+---
+
+## P-0: what every run waits for
+
+Nothing below P-0 runs until every item here is true. Paul confirms P-0 per
+deployment before its first run.
+
+- [ ] **Claim Plan 12 merged.** Without it the landing, the target's half
+ of a rollback and the source's recovery are unarmed in production, so a
+ move with the switch on stops the source, flips, and then has nothing to
+ land it, roll it back or recover it.
+- [ ] **Moves Plan 5 Task 9b, Plan 4 Tasks 9 and 10 (the box half), and
+ Plan 5 Task 8 merged.** They wire moves onto claim Plan 12. Run 4 also
+ needs claim Plan 12's loss-bound admission.
+- [ ] **The moves cut guard merged** (branch `moves-cut-guard`, Paul's
+ ruling of 2026-10-11). The source refuses a cut unless the target
+ reports its landing armed. It adds a `T13` check to the target's report
+ and records why a cut did not start. It is not on main yet.
+- [ ] **For any run that touches the home region: the home-region parks fix
+ part 1 merged** (#236). Part 1 covers moves out of home. Moves into home
+ need part 2, and the door refuses them until then ("moves to the home
+ region are not supported yet. the home publishes no staging lane").
+- [ ] **All of the above released and rolled, consoles first, then every
+ box.** Check every box with `carlos system status --console <console>`:
+ each reads `current` on the new version.
+- [ ] **Each deployment's home region recorded:** `CONSOLE_HOME_REGION` in
+ the console's environment. A box with no `CARLOS_REGION` in its
+ `host.env` publishes no `placement` in its edges file: it is a home box.
+- [ ] **Rehearsal boxes keep instances flat.** Moves refuse a nested
+ instance. Set `CARLOS_LAYOUT_DEFAULT=flat` in each rehearsal box's
+ `host.env`. Pre-flight check, before any request: `carlos status
+ --layout --console <console>` (the layout each host is actually on,
+ box by box) shows every rehearsal host's row as `flat`, and no
+ rehearsal box is marked "(nested by default)". `carlos system layout
+ status` is not this check: it shows only the recorded intent. Once a
+ run's request exists, S-2 also checks `detail.source_report.s1.ok`,
+ the source's own refusal of a nested instance.
+- [ ] **Never set `CARLOS_MOVES_PHASE1=on` anywhere before P-0 is true.**
+
+---
+
+## Preconditions, once per deployment
+
+### P-1 The dormant release
+
+- [ ] The release carrying P-0 is rolled with `CARLOS_MOVES_PHASE1` unset
+ everywhere: consoles first, then boxes.
+- [ ] [deploy] Every box's `control/edges/<box>.json` has
+ `"moves_protocol": 1`.
+
+### P-2 Staging grants (§15 step 5, scoped tofu by Paul)
+
+- [ ] Every rehearsal region's `sibling_staging_arns` lists every other
+ rehearsal region's data bucket, and each box has the versioned-object
+ grants (`s3:ListBucketVersions`, `s3:GetObjectVersion`,
+ `s3:DeleteObjectVersion` on its own bucket; `s3:GetObjectVersion` for
+ the console).
+- [ ] Check after the first request reaches ready: [detail]
+ `detail.target_report.checks.T10.ok` is `true` (the lane probe).
+
+### P-3 The protocol floor
+
+Publish the floor only after **every box and the console run a tagged
+`vX.Y.Z` release** at or above `min_release`. Once a floor is armed, any
+build that is not `vX.Y.Z` (a SHA build, `dev`, a prerelease) is refused at
+activation, and the door treats that box as below the floor. A box never
+lowers its local floor on its own. If dev keeps rolling SHA builds, dev's
+runs go without a floor, and P-3 is done on the flagship only, before
+Run 5 (§15 step 6).
+
+- [ ] **Paul's scoped go** in the amadan thread to publish the floor on
+ this deployment, naming the deployment and `min_release`. Nothing below
+ runs without it.
+- [ ] Write it create-only from an operator shell. No `carlos` command
+ writes it (a gap to note on an amadan discussion).
+
+ ```sh
+ printf '{"min":1,"min_release":"vX.Y.Z"}' > floor.json
+ aws s3api put-object --profile <profile> --bucket <deploy-bucket> \
+ --key control/moves-protocol.json --body floor.json --if-none-match '*'
+ aws s3 cp --profile <profile> s3://<deploy-bucket>/control/moves-protocol.json -
+ ```
+
+- [ ] [deploy] Every box's edges file shows `moves_floor_min_release` equal
+ to the floor's `min_release`, and either `"prev_release": "none"` or
+ `prev_allowed_for` equal to `min_release`. A box that does not is below
+ the floor for the door (MV-N5).
+
+### P-4 Ship moveprobe
+
+- [ ] Follow `cmd/moveprobe/README.md` ("Build" and "Deploy") under the ops
+ account: ship, enable instances with private storage off, create the
+ store with `--env-prefix MOVEPROBE` and no `--environment`, set
+ `MOVEPROBE_SECRET`, and set the `every-minute` schedule.
+- [ ] Keep `MOVEPROBE_SECRET`'s value in your shell. `/report` and
+ `/arm-once` need it.
+- [ ] Leave `moves.ready_path` unset. A move then probes `/`, which answers
+ as `/ready` does.
+
+### P-5 Rehearsal hosts, created before the claim
+
+A host created while `CARLOS_HOST_CLAIM` is on keeps its data under a
+per-incarnation path, and the move door refuses it with MV-N167: "the
+instance keeps its data under a per-incarnation path, which moves cannot
+cut yet". So every rehearsal host is created **before** the claim goes on
+for its deployment (Paul, N2(a)).
+
+- [ ] Before `CARLOS_HOST_CLAIM` goes on, create one moveprobe host per
+ planned run, in the region that run starts from, plus two spares:
+
+ ```sh
+ carlos instances create --app moveprobe --account <ops> --host <label>.<instance domain> --region <region>
+ ```
+
+ On the flagship, also create Run 6's disposable customer-shaped
+ instance now.
+- [ ] Record the list of hosts, with the run each one is for, in the
+ deployment's rehearsal record (outside this repo). **Never reuse a
+ host.** A host that has run once is spent, whatever the outcome.
+- [ ] After the claim is on, check each host is adopted: [deploy]
+ `control/hosts/<host>.json` exists, `holder.location` is
+ `instances/<host>` (not `instances/<host>/<inc>`), and `op.kind` is
+ `serving`. The console's claim recovery pass adopts them.
+
+### P-6 The rest of the fleet's preconditions
+
+Paul confirms each of these per deployment before its first run.
+
+- [ ] **Claim on.** `CARLOS_HOST_CLAIM` is set (any non-empty value) in
+ the console's environment and every box's `host.env`. Check: [box] on
+ each box and the console. #250's activation blockers are closed.
+- [ ] **Roster.** Each rehearsal box is an active, enrolled entry with no
+ termination fence. Check: [deploy] the decoded roster (`roster` above)
+ has `boxes.<box>.state` `active` and `boxes.<box>.enrollment` set, and
+ `control/box-fences/<enrollment>.json` does not exist.
+- [ ] **Box identity records.** `CARLOS_REGION_HA=true` in each rehearsal
+ box's `host.env`. It takes `true`, `1` or `t`; **`on` reads as off**.
+ `CARLOS_AWS_ACCOUNT` is set in the console's environment. Check: [deploy]
+ `control/boxes/<box>.json` exists for each rehearsal box, and its edges
+ file has `"capabilities": ["region-ha-1"]`.
+- [ ] **One eligible box per rehearsal region.** Any other box in a
+ rehearsal region runs without `CARLOS_REGION_HA`, so it publishes no
+ `region-ha-1` and is not eligible. Otherwise the door answers M-C21.
+- [ ] **Secrets key.** `CARLOS_SECRETS_KEY` on every rehearsal box, from
+ the same key pair as the console's. Check: [deploy] each edges file's
+ `secrets_key_fp` is the same; once the switch is on, it equals
+ `secrets_key_fp` in `control/moves-console.json`; [detail]
+ `detail.target_report.checks.T2.ok` is `true`.
+- [ ] **Staging bucket.** Each rehearsal box's edges file has
+ `data_bucket` set.
+- [ ] **Alert sinks.** `CARLOS_ALERT_WEBHOOK` or
+ `CARLOS_ALERT_SNS_TOPIC_ARN` is set on every rehearsal box and on the
+ console. Without one, pages go nowhere.
+- [ ] **Cgroup delegation.** Check: [box] `journalctl -u carlos-edge -b |
+ grep 'exec-child accounting'` shows `armed (delegated subtree ...)`. A
+ box without it cannot produce stop evidence and never cuts.
+- [ ] **Flat rows and tagged releases**, as P-0 and P-3 say.
+
+### P-7 The switch
+
+- [ ] P-0 to P-6 are done for this deployment.
+- [ ] **The moves copy record is fully approved**: no row in
+ `docs/superpowers/plans/2026-10-06-moves-phase1-copy.md` is `pending`
+ (`grep -c '| pending |'` on it prints 0). The switch never goes on
+ while a sentence an operator could read is unapproved.
+- [ ] **Paul's scoped go** in the amadan thread to switch moves on, naming
+ the deployment, the console and the boxes.
+- [ ] `CARLOS_MOVES_PHASE1=on` in the environment of the console and of
+ the boxes the go names, and nowhere else, then restart them.
+- [ ] Check: `carlos move status --app moveprobe --account <ops> --host
+ <host>` does not say "Moves are not enabled on this console."
+
+---
+
+## Between runs
+
+Before starting the next run, wait until all of these hold for the host
+the last run used:
+
+- [ ] [detail] `state` is `none` (no move in flight) and `last.outcome` is
+ recorded.
+- [ ] [detail] `detail.claim.kind` is `serving`.
+- [ ] [deploy] the claim's `quiet_after` for the region the next move
+ goes to has passed (T12). It is set when an instance leaves a region and
+ lasts about 16 minutes. With a fresh host per run it rarely bites, but
+ check it.
+- [ ] `MOVEPROBE_FAIL_READY_IN` is unset: `carlos env list --app moveprobe
+ --account <ops>` does not show it.
+
+---
+
+## The steps and checks each run uses
+
+Runs refer to these by number. `<from>` and `<to>` are the run's regions.
+
+### Steps
+
+- **S-1 Wake and record.** `curl https://<host>/ready` answers `ready`.
+ Save [report] as `before-request.json`.
+- **S-2 Request.**
+
+ ```sh
+ carlos move --app moveprobe --account <ops> --host <host> --to <to> --manual-cut
+ ```
+
+ [CLI] M-C1 ("Moving <host> from <from> to <to>."), M-C2, M-C3, then
+ M-C12 ("Ready to cut. ..."), exit 4. A line "<to> cannot take this move
+ yet: <check>." is M-C4 and names the failing check.
+ [detail] at ready: every `detail.target_report.checks.T1` to `T12` has
+ `ok: true` (and `T13` once the cut guard lands);
+ `detail.target_report.attempts_admitted` is at least 1;
+ `detail.source_report.s1.ok` and `s2.ok` are `true`. Record `id` as
+ `<move>` and `detail.move.incarnation` as `<inc>`.
+- **S-3 Record and arm.** Save [report] as `before-cut.json`. Arm the
+ once-timer so it falls due around the cut, then cut within a few
+ seconds:
+
+ ```sh
+ curl -fsS -X POST -H "Authorization: Bearer $MOVEPROBE_SECRET" "https://<host>/arm-once?in=30"
+ ```
+
+ Record the answer's `name` and `due_at`.
+- **S-4 Capture, then cut.** In a second terminal, start [capture] for
+ this run: `capture <host> run-<n>-detail.jsonl`. Then, in the first:
+
+ ```sh
+ carlos move cut --app moveprobe --account <ops> --host <host>
+ ```
+
+ The cut waits until the move ends or needs an operator. The expected
+ sentences and exit code are in each run. Every run captures, so that
+ whatever the record held while the move was live (the evidence keys,
+ the rollback journal, a loss) can still be read after it closes.
+
+### Checks after a commit
+
+- **R-1 State arrived.** [report] on the target: `region` is `<pr-to>`;
+ `file_key_sha256` and `platform_key_sha256` equal `before-cut.json`'s;
+ `object` has no `error` and its `body` names this boot and `<pr-to>`;
+ `has_secret` is `true`. `curl https://<host>/ready` answers `ready`.
+- **R-2 No row lost.** [report] for `bg` and for `worker`: the sum of
+ `count` over all regions is at least `before-cut.json`'s, and equals the
+ highest `max_seq` (rows are numbered from 1 in one sequence per kind, so
+ any gap shows here).
+- **R-3 The schedule continues.** [report] `rows` has `kind: tick` with
+ `region: <pr-to>`, and its `count` rises about once a minute for three
+ minutes after the open. Every `fired` entry has `count: 1`. A minute
+ that fell due while the instance was offline fires at most once after
+ the open, and minutes in between are not replayed, so a short gap
+ around the cut is expected. A due time that fired twice fails the run.
+- **R-4 The once-timer fired once.** [report] `fired` has exactly one
+ entry with S-3's `name`, its `due_at` equals S-3's, and its `count` is 1.
+- **R-5 The source holds nothing live.** After [deploy]
+ `log/committed-source_parked.json` and `log/done.json` exist:
+ [data] `list-object-versions` on `<data-bucket-from>` under
+ `instances/<host>/` returns no versions and no delete markers, and
+ `parked/<inc>/<host>/` holds the retired replica objects with a
+ `manifest.json`. [deploy] the claim's `obligations` is empty.
+- **R-6 The source really stopped.** [deploy] `stopped-source.json` (the
+ key [capture] shows as `detail.evidence.source`) has `steps.tree.ok`
+ and `steps.stray.ok` both `true` and no `stray` field. [box] on the source box, the journal has no "the instance is not
+ proven stopped yet" line for `<host>` and no M-C20 line ("... is running
+ a process for <host> that this move cannot account for. ..."), and
+ `/data/.moves/.page-outbox/<host>/<move>/stray.json` does not exist.
+ The moveprobe worker outlives its app on purpose, so a worker that
+ survived the stop fails the cut's tree step and shows here, then
+ reverts at the cut's deadline. It does not page.
+
+### Checks after a rollback (Runs 2, 3 and 4)
+
+R-1 to R-4 and R-6 apply as written, with the source (where the
+instance is back) in place of the target: `<pr-from>` in place of
+`<pr-to>` in R-1 and R-3. R-5 is replaced:
+
+- **R-5b The target holds nothing live.** After [deploy]
+ `log/rolling_back-target-cleaned.json` and `log/rolled_back.json` exist:
+ [data] `list-object-versions` on `<data-bucket-to>` under
+ `instances/<host>/` and under `moves/<host>/<move>/` returns no versions
+ and no delete markers.
+- **R-7 The target's writes came back** (dirty rollbacks only). [report]
+ on the source: `rows` has a `bg` row with `region: <pr-to>` (the
+ target's probe region, `home` if the target is the home box), written
+ on the target while it ran there.
+- **R-8 The return lane is parked** (dirty rollbacks only). A dirty
+ rollback returns the target's writes through its own lane,
+ `moves/<host>/<rb>/` in the **source's** data bucket, where `<rb>` is
+ the rollback's id (from [capture]: the last rollback journal's `id`).
+ The source queues the lane's park in the host claim when it has probed.
+ The park runs only after its horizon (the return seal's bound) plus the
+ signature age (15 minutes) and the request limit (2 minutes), and the
+ rollback does not close until every obligation of the move is
+ acknowledged. So expect the cut's wait to last at least that long.
+ The obligation exists only while the move is live, so read it from the
+ claim file [capture] saved, never from the bucket afterwards:
+
+ ```sh
+ jq -s --arg loc "moves/<host>/<rb>" '[.[][] | select(.park.location == $loc)] | first' run-3-claim.jsonl
+ ```
+
+ It has `kind` `park-replica`, `region` set to the source box's
+ `CARLOS_REGION` (absent when the source is the home box), an `id`, a
+ `retirement`, and `park` with `location`, `incarnation`, `evidence_at`
+ and `request_limit_ns`. Record `retirement` as `<ret>`,
+ `park.incarnation` as `<pinc>` and `park.evidence_at`. An empty answer
+ means the capture missed the obligation: the check fails, and the run
+ is repeated on a spare host with the claim capture running.
+ After `park.evidence_at` plus 15 minutes plus `request_limit_ns`, and
+ once `log/rolled_back.json` exists:
+ [data] `list-object-versions` on `<data-bucket-from>` under
+ `moves/<host>/<rb>/` returns no versions and no delete markers, and
+ `parked/<pinc>/<host>/<ret>/manifest.json` exists in
+ `<data-bucket-from>`.
+
+A clean rollback needs two things: the stopped target's digests equal
+what the landing installed, and `src_covered` (the source's own replica
+reached the stopped position). The target box logs both:
+[box] `journalctl -u carlos-edge | grep 'the target of rollback'` shows
+"... stopped at TXID <n> (clean <bool>, src_covered <bool>)".
+
+---
+
+## Dev (Runs 1 to 4)
+
+**Dev topology (Paul, N4).** Dev's second region is applied: `uk`,
+`eu-west-2`, box `carlos-dev-uk`, data bucket
+`carlos-dev-uk-640012953448` (platform-infrastructure#42). Before Run 1:
+
+- [ ] Bootstrap `carlos-dev-uk`.
+- [ ] Copy the SSM parameters a dev box reads at boot to `eu-west-2`.
+- [ ] `CARLOS_REGION_HA=true` on `carlos-dev-ie` and `carlos-dev-uk`, and
+ `CARLOS_AWS_ACCOUNT` on the dev console.
+- [ ] Keep `carlos-dev-ie-b` ineligible: no `CARLOS_REGION_HA` in its
+ `host.env`, so `ie` has one eligible box (M-C21).
+- [ ] Decide Runs 2 and 3's arrangement (see "Runs 2 and 3: the
+ regions"), then create the dev rehearsal hosts (P-5) after
+ `carlos-dev-uk` serves and before dev's claim goes on: Run 1's and Run
+ 4's in `ie`, Runs 2 and 3's in their arrangement's `<from>`, and two
+ spares.
+
+**Dev's home region.** Dev Ireland is probably the home region (its box
+has no `CARLOS_REGION`). Confirm it with the dev console's
+`CONSOLE_HOME_REGION` and record it here: `____`. If `ie` is home:
+
+- Run 1 (`ie` to `uk`) moves out of home and needs **home fix part 1**.
+- Run 4 (`ie` to `uk`) moves out of home and then rolls back into it. It
+ needs **part 1**, and part 1 must cover a rollback whose source is home.
+- Runs 2 and 3 as written (`uk` to `ie`) move into home and need **home
+ fix part 2**. The door refuses them until then.
+- **Until part 2 lands, only out-of-home moves run.** Run 1 goes ahead on
+ part 1. Runs 2 and 3 then use the **out-of-home arrangement**: the host
+ is created in `ie`, the move goes `ie` to `uk`, and the failure switch
+ names `<pr-uk>`. Each rolls back into a home source, as Run 4 does, so
+ this needs part 1 to cover a rollback whose source is home. Check that
+ in part 1's own tests first. Part 1 is not built yet, so this is open.
+ If part 1 does not cover it, Runs 2, 3 and 4 all wait for the fix that
+ does.
+- Run 4 runs only after Runs 2 and 3 have passed, in whichever
+ arrangement they ran. No run is skipped.
+- On the home box, moveprobe reports `home`, so `<pr-ie>` is `home`. Any
+ [report] check or `MOVEPROBE_FAIL_READY_IN` aimed at that box uses
+ `home`, never `ie`.
+
+Record the dev probe regions here: `<pr-ie>` = `____`, `<pr-uk>` = `____`.
+
+If `ie` is not home, the runs go as written and need no home fix.
+
+### Run 1: dev, `ie` to `uk`, a commit
+
+**Blockers:** P-0; home fix part 1 if `ie` is home; dev topology above; P-1
+to P-7 on dev; Paul's go.
+
+- [ ] Host used: `____` (created in `ie` before the claim).
+- [ ] S-1, S-2 with `--to uk`.
+- [ ] S-3, S-4. [CLI] M-C5 ("uk is ready. Starting the cut. <host> will be
+ offline for about <estimate>."), M-C6 ("Stopped in ie. Starting in
+ uk."), M-C7 ("Moved. <host> is served from uk. It was offline for
+ <seconds> seconds."), exit 0.
+- [ ] R-1 to R-6.
+- [ ] [deploy] `log/` has `cutting-flipped`, `committed`,
+ `committed-opened`, `committed-source_parked` and `done`. [detail]
+ `last.outcome` is `done`; `detail.timings` holds `offline_s` and
+ `cut_budget_s`.
+
+### Runs 2 and 3: the regions
+
+Runs 2 and 3 use one of two arrangements, chosen before the dev hosts are
+created (see "Dev's home region"):
+
+- **As written**: the host is created in `uk`, and the move goes `uk` to
+ `ie`. Needs home fix part 2 if `ie` is home.
+- **Out of home** (`ie` is home and part 2 has not landed): the host is
+ created in `ie`, and the move goes `ie` to `uk`. Needs part 1 to cover a
+ rollback whose source is home.
+
+Below, `<from>` and `<to>` are the arrangement's regions as the CLI names
+them, and `<pr-from>`, `<pr-to>` their probe regions. Record the
+arrangement used in each run.
+
+### Run 2: dev, forced clean rollback
+
+**Blockers:** P-0; the home fix its arrangement needs (above); Run 1
+passed; Paul's go.
+
+The target fails its readiness check in before-write mode, so it writes
+nothing and the rollback should be clean.
+
+- [ ] Arrangement: `____`. Host used: `____` (created in `<from>` before
+ the claim).
+- [ ] Before the request, set the failure switch to the target's probe
+ region:
+
+ ```sh
+ carlos env set --app moveprobe --account <ops> MOVEPROBE_FAIL_READY_IN=<pr-to>
+ ```
+
+ The variable is app-wide. Any moveprobe instance that starts in that
+ region while it is set answers 503 and writes nothing. That is expected.
+- [ ] S-1, S-2 with `--to <to>`, S-3, S-4 (capture first). [CLI] M-C8
+ ("<to> could not start <host>: <reason>. Moving it back to <from>."),
+ then M-C9 ("<host> is back in <from> and serving. Nothing was lost."),
+ exit 3.
+- [ ] From [capture], once the cut has returned and the capture has
+ stopped: the last rollback journal has `target_phase` `released` or
+ later, and `dirty` absent or `false`. Record its `id` as `<rb>`, and the
+ last `detail.evidence.target`.
+- [ ] Digests: [deploy] `stopped-target.json`'s `db_sha256`, `files_plain`
+ and `timers_sha256` equal `log/rolling_back-target-released.json`'s
+ fields `installed_db_sha256`, `files_plain` and `timers_sha256`.
+- [ ] [box] on the `<to>` box: "clean true, src_covered true".
+- [ ] R-1 to R-4 and R-6 on `<from>` (probe region `<pr-from>`), R-5b on
+ `<to>`.
+- [ ] [deploy] `log/` has `rolling_back`, `rolling_back-target-released`,
+ `rolling_back-target-cleaned`, `rolling_back-source-opened` and
+ `rolled_back`, and no `rolling_back-target-returning`.
+- [ ] After the checks are recorded:
+
+ ```sh
+ carlos env unset --app moveprobe --account <ops> MOVEPROBE_FAIL_READY_IN
+ ```
+
+A dirty result here, with moveprobe in before-write mode, is a platform
+finding, not a probe failure. Stop the series and report it.
+
+### Run 3: dev, forced dirty rollback
+
+**Blockers:** P-0; the home fix its arrangement needs; Run 2 passed;
+Paul's go.
+
+The target writes one row, starts its worker, and then fails readiness, so
+the rollback is dirty and the target's writes must come back.
+
+- [ ] Arrangement: `____`. Host used: `____` (created in `<from>` before
+ the claim).
+- [ ] Before the request:
+
+ ```sh
+ carlos env set --app moveprobe --account <ops> MOVEPROBE_FAIL_READY_IN=<pr-to>,after-write
+ ```
+
+- [ ] S-1, S-2 with `--to <to>`, S-3, S-4, with the capture started with
+ its claim file: `capture <host> run-3-detail.jsonl run-3-claim.jsonl`.
+ [CLI] M-C8, then M-C9, exit 3. The cut's wait includes the return
+ lane's park horizon (R-8), about 17 minutes or more.
+- [ ] From [capture]: the last rollback journal has `target_phase`
+ `released` or later and `dirty` `true`. Record its `id` as `<rb>`
+ before anything else; R-8 needs it.
+- [ ] Digests: [deploy] `stopped-target.json`'s `db_sha256` differs from
+ `log/rolling_back-target-released.json`'s `installed_db_sha256` (the
+ target wrote).
+- [ ] [box] on the `<to>` box: "clean false".
+- [ ] [deploy] `log/` has `rolling_back-target-returning` and
+ `rolling_back-target-returned`, then `rolling_back-target-cleaned` and
+ `rolled_back`.
+- [ ] R-1 to R-4 and R-6 on `<from>` (probe region `<pr-from>`), R-5b on
+ `<to>`, R-7 (a `bg` row with `region: <pr-to>`), and R-8 on `<from>`
+ (the return lane `moves/<host>/<rb>/` in `<data-bucket-from>`).
+- [ ] Unset `MOVEPROBE_FAIL_READY_IN` once the checks are recorded.
+
+### Run 4: dev, declare-lost
+
+**Blockers:** P-0, including claim Plan 12's loss-bound admission and Plan
+4 Task 10's box half; home fix part 1 if `ie` is home, covering a
+rollback whose source is home; **Runs 1, 2 and 3 passed** (Runs 2 and 3
+in whichever arrangement they ran; Run 4 is never run ahead of them);
+Paul's go.
+
+The target box is stopped mid-move and declared lost. With only two dev
+regions, the target is `carlos-dev-uk`, and `uk` has no eligible box until
+the readmit at the end. **Stop the box. Do not terminate it.** A terminated
+box cannot be readmitted.
+
+- [ ] Host used: `____` (created in `ie` before the claim).
+- [ ] **Paul's scoped go** in the amadan thread, before anything below,
+ to stop `carlos-dev-uk` during this run and take it out of service
+ until the readmit.
+- [ ] So the landing cannot commit before the box stops, set the failure
+ switch on the target first:
+
+ ```sh
+ carlos env set --app moveprobe --account <ops> MOVEPROBE_FAIL_READY_IN=<pr-uk>
+ ```
+
+ Its probe then fails however long the stop takes. If the target finishes
+ its release before the box stops, the run is an ordinary clean rollback:
+ record it and repeat Run 4 on a spare host.
+- [ ] Before the run, record the box's roster entry: [deploy]
+ `roster | jq '.boxes["carlos-dev-uk"]'` shows `state` `active`; note
+ its `enrollment` and `quarantine_revision`.
+- [ ] S-1, S-2 with `--to uk`, S-3, S-4 (capture first). The capture runs
+ until the move closes, through the declaration below.
+- [ ] When [capture] shows `state` `landing` (after `cutting-flipped`),
+ stop `carlos-dev-uk` through the AWS console (dev only).
+- [ ] The cut's wait ends with M-C10 ("Waiting for carlos-dev-uk in uk to
+ confirm it has stopped <host>. <host> is offline."), exit 4. Until the
+ landing deadline passes, it keeps waiting.
+- [ ] Declare it lost, with your own AWS credentials for dev in the shell
+ (the command reads EC2 with them and refuses to run as root):
+
+ ```sh
+ carlos move resolve --app moveprobe --account <ops> --host <host> --declare-lost carlos-dev-uk
+ ```
+
+ [CLI] M-C18 ("This declares carlos-dev-uk lost. Anything <host> wrote
+ on carlos-dev-uk after <time> is lost. Type the host name to
+ confirm."). Type `<host>`. The command then waits for the outcome:
+ expect M-C9 and exit 3.
+- [ ] Expected outcome: **rolled back, with the loss on the record**, not
+ necessarily clean. From [capture]: the last rollback journal has
+ `loss_ref` and `lost_at` set; record them and its `id`. [deploy]
+ `log/rolling_back-target-lost.json` (event `declared_lost`) and
+ `log/rolled_back.json` exist. [deploy] the decoded roster:
+ `boxes.carlos-dev-uk.state` is `quarantined`, `quarantine_revision` is
+ one more than before, and `loss_ids` holds the loss.
+- [ ] R-1 to R-4 and R-6 on `ie`, with probe region `<pr-ie>` (`home` if
+ `ie` is home).
+- [ ] Unset `MOVEPROBE_FAIL_READY_IN`.
+- [ ] Start `carlos-dev-uk` again. It stays quarantined: the deployment's
+ admission gate refuses it for every host until the readmit. Watch [box]
+ its journal from boot, and record anything it does for `<host>` before
+ the readmit. Doing anything for it there is a finding.
+- [ ] [deploy] the box retires its local state first and records it:
+ `control/host-retirement/carlos-dev-uk/<enrollment>/<quarantine_revision>.json`
+ exists with `"complete": true` and this boot's `boot_id`. Readmit
+ refuses without it.
+- [ ] Readmit it. `carlos ops host` exists only with `CARLOS_HOST_CLAIM`
+ set in your shell:
+
+ ```sh
+ CARLOS_HOST_CLAIM=1 carlos ops host resolve <host> --readmit carlos-dev-uk --console <console>
+ ```
+
+ Answer `readmit` at its prompt. The box must be running and reporting:
+ readmit refuses a box with a termination fence, and one whose private
+ report is older than 15 minutes. Before it, check [deploy]
+ `control/box-fences/<enrollment>.json` does not exist (a stopped box
+ has none; a terminated one does).
+- [ ] [deploy] the decoded roster: `boxes.carlos-dev-uk.state` is `active`
+ again, with the same `enrollment`.
+
+---
+
+## Flagship (Run 5)
+
+### Run 5: the flagship, Runs 1 to 3 again
+
+**Blockers:** P-0; P-1 to P-6 on the flagship; **P-3, the floor, published**
+(§15 step 6: before any protected production rehearsal); the moves copy
+record fully approved (no `pending` row); **dev Runs 1 to 4 passed**
+(each run gates the next, spec §15 step 7; claim Plan 8b is on main, so
+the old clause that let Run 5 go ahead with Run 4 skipped no longer
+applies); Paul's explicit go in the thread, for each of the three.
+
+- **The switch** (Paul, N17): `CARLOS_MOVES_PHASE1=on` in the environment
+ of the console and of the two boxes in play, and nowhere else. The
+ compiled default stays off. It flips later, in its own commit.
+- **Regions** (N3): the flagship's home is Ireland, so Run 5 uses two
+ non-home regions, `uk` and `us`, until the home fix parts land. Run 1's
+ shape goes `uk` to `us`. Runs 2 and 3 go `us` to `uk`, with
+ `MOVEPROBE_FAIL_READY_IN=<pr-uk>` and `<pr-uk>,after-write`. Record the
+ flagship's probe regions: `<pr-uk>` = `____`, `<pr-us>` = `____`.
+
+- [ ] P-3 on the flagship, after its own scoped go.
+- [ ] P-7 on the flagship: the copy record fully approved, then Paul's
+ scoped go naming the console and the `uk` and `us` boxes, then the
+ switch on those three only.
+- [ ] Hosts used: Run 1 shape `____` (created in `uk`), Run 2 shape `____`
+ and Run 3 shape `____` (created in `us`), all before the flagship's
+ claim went on.
+- [ ] Each of the three as on dev, with every check.
+- [ ] After the third, and only once **all three moves are closed**: for
+ each of the three hosts, [detail] `state` is `none` (no move in flight)
+ and `detail.claim.kind` is `serving`. If any move is unfinished (stuck,
+ a release past its deadline, relocating, or any exit 4), leave the
+ switch on and resolve it first with `carlos move resolve` (`--retry`
+ once the cause is fixed, or `--declare-lost <box>` for a lost target),
+ each with Paul's scoped go in the thread. The switch must stay on until the move is closed: with it off, a box's
+ moves pass skips every Phase 1 record, so an unfinished move would stop
+ being reconciled and stay where it is.
+- [ ] Then remove `CARLOS_MOVES_PHASE1` from the two boxes and the
+ console unless Paul says to keep it, and restart them. Check:
+ `carlos move status --app moveprobe --account <ops> --host <host>` says
+ "Moves are not enabled on this console.", and neither box's `host.env`
+ still sets it.
+
+---
+
+## Customer instances (Runs 6 and 7)
+
+The runbook names no tenant. The real instances, their owners and the
+agreed windows live in the flagship's deployment repo and in the amadan
+thread where Paul gives each go.
+
+These runs move a customer's app, not moveprobe. **Do not call `/ready`,
+`/arm-once` or `/report` on a customer host, and do not set
+`MOVEPROBE_FAIL_READY_IN`.** Those are moveprobe's own, and S-1, S-3, R-1
+to R-4 and R-7 do not apply. Set `ACCT=<account>` and `APP=<app>` for
+[detail], [capture] and the log prefix, and use `--app <app> --account
+<account>` on every command. `<from>` and `<to>` are the regions Paul
+approves for the run.
+
+### Customer steps
+
+- **C-0 Switch on for this run only.** Run 5's switch was removed after
+ it, and this run's participants differ (the source is the home box).
+ - The moves copy record is fully approved (P-7).
+ - **Paul's scoped go** in the amadan thread naming the flagship, this
+ run, the console, and this run's two participant boxes: the source
+ box in `<from>` (the instance's box now) and the target box in
+ `<to>`.
+ - `CARLOS_MOVES_PHASE1=on` in the environment of the console and of
+ those two boxes, and nowhere else; restart them.
+ - Check: `carlos move status --app <app> --account <account> --host
+ <host>` does not say "Moves are not enabled on this console." [box]
+ each of the two boxes' `host.env` sets it, and no other box's does.
+- **C-1 Health check, before.** The owner names one health check for the
+ instance (a URL and the answer that means it is serving) and approves
+ it for the run. Record it in the deployment repo. Run it and record the
+ answer.
+- **C-2 Request.**
+
+ ```sh
+ carlos move --app <app> --account <account> --host <host> --to <to> --manual-cut
+ ```
+
+ [CLI] M-C1, M-C2, M-C3, then M-C12, exit 4. [detail] at ready, the same
+ checks as S-2: every target check `ok`, `attempts_admitted` at least 1,
+ `source_report.s1.ok` and `s2.ok` `true`. Record `<move>` and `<inc>`.
+- **C-3 Capture, then cut.** Start [capture] in a second terminal, then,
+ in the window agreed with the owner:
+
+ ```sh
+ carlos move cut --app <app> --account <account> --host <host>
+ ```
+
+ [CLI] M-C5, M-C6, M-C7, exit 0.
+- **C-4 Health check, after.** The owner's check from C-1 gives the
+ serving answer once M-C7 has printed. The owner confirms it.
+- **C-5 Platform checks.** R-5 as written. R-6 as written, except its
+ last two sentences, which are about moveprobe's worker. [deploy] `log/`
+ has `cutting-flipped`, `committed`, `committed-opened`,
+ `committed-source_parked` and `done`.
+
+If the move rolls back (M-C8 then M-C9, exit 3), the instance is back in
+`<from>`: run C-4 there, then R-5b and R-6, and stop the series. C-6
+still applies, once the rollback has closed.
+
+- **C-6 Switch off, once the move is closed.** First confirm the move is
+ closed: [detail] `state` is `none` (no move in flight) and
+ `detail.claim.kind` is `serving`. If it is not (stuck, a release past
+ its deadline, relocating, or any exit 4), leave the switch on and
+ resolve the move first with `carlos move resolve --app <app> --account
+ <account> --host <host>` (`--retry` once the cause is fixed, or
+ `--declare-lost <box>` for a lost target), with Paul's scoped go in the
+ thread, then check again. The switch must stay on until the move is closed: with it off, a box's
+ moves pass skips every Phase 1 record, so an unfinished move would stop
+ being reconciled and stay where it is.
+ Once it is closed, and after the run's checks are recorded: remove
+ `CARLOS_MOVES_PHASE1` from the console and the two boxes, unless Paul
+ says in the thread to keep it, and restart them. Check: `carlos move
+ status --app <app> --account <account> --host <host>` says "Moves are
+ not enabled on this console.", and [box] neither box's `host.env` sets
+ it.
+
+### Run 6: a disposable customer-shaped instance
+
+**Blockers:** P-0; **the home-region parks fix, both parts, merged and
+rolled** (customer instances live in the home region); a dev rehearsal
+from home passed once the fix is in; Run 5 passed; the owner's agreement;
+Paul's go.
+
+A disposable instance of a customer app with the same shape as moveprobe
+(a key file, a store, a sealed secret), created for the run with the
+owner's agreement, before the flagship's claim went on (P-5).
+
+- [ ] Instance used: recorded in the deployment repo.
+- [ ] C-0 to C-6.
+
+### Run 7: the first production customer instance
+
+**Blockers:** everything Run 6 needs; Run 6 passed; Paul's approval of the
+instance; a window agreed with its owner.
+
+The first production customer instance Paul approves, in a window agreed
+with its owner.
+
+- [ ] The instance was created before the flagship's claim went on, so it
+ is adopted at `instances/<host>` (P-5). One created later cannot move.
+- [ ] C-0 to C-6, with C-3 inside the agreed window. Run 6's switch was
+ removed at its C-6, so C-0 needs its own go. The cut window is the
+ default 120 s.
+- [ ] Report `offline_s` back to the owner: [detail] `last.offline_s`, or
+ `offline_s` in `log/done.json`'s fields.
+
+---
+
+## After every run
+
+- [ ] Attach to the amadan thread: the `done` or `rolled_back` log entry
+ (its timings), the bucket listings, and the host used. For Runs 1 to 5,
+ also the [report] files from before and after. For Runs 6 and 7, which
+ never call `/report`, attach instead the owner-approved health check's
+ answers from C-1 and C-4, and the owner's confirmation.
+- [ ] A failed check stops the series. Fix its cause and repeat the run on
+ a fresh host before going on.
+
+---
+
+## Sources
+
+Paths are from the repo root. Line numbers are main's (`bc25674a`, the
+base of this branch); `cmd/moveprobe` is this branch's and not on main
+yet. Spec is `docs/superpowers/specs/2026-10-02-move-any-app-design.md`;
+copy is `docs/superpowers/plans/2026-10-06-moves-phase1-copy.md`.
+
+**`carlos move` (the CLI)**
+
+| what the runbook says | where |
+|---|---|
+| verbs `status`, `cut`, `abort`, `resolve`; flags `--host`, `--app`, `--account`, `--console` | `move.go:136-149` |
+| `--to`, `--manual-cut`, `--no-wait` on a request; `--wait` on status; `--retry`, `--declare-lost`, `--no-wait` on resolve | `move.go:151-163` |
+| exit codes 0, 1, 2, 3, 4, 5 | `move.go:40-45`, `:53-61` |
+| "carlos move needs --app" | `move.go:66`, `:230` |
+| "Moves are not enabled on this console." | `move.go:65`, `:240-243` |
+| refuses to run as root; declare-lost reads EC2 with your own credentials | `move.go:211-217`, `:80`, `:900-903` |
+| the request sends `region` and `cut` | `move.go:248-253` |
+| M-C2, M-C3, M-C4 | `move.go:703`, `:707`, `:717` |
+| M-C12 at ready with a manual cut, exit 4 | `move.go:722-726` |
+| M-C10, exit 4 | `move.go:740-754` |
+| M-C7 on done, M-C9 on rolled_back, exit 3 | `move.go:788-803` |
+| M-C18 and the typed host name | `move.go:860-882` |
+| "No move is in flight" | `move.go:595` |
+| M-C1, M-C5, M-C6, M-C7, M-C8, M-C9 texts | `internal/moves/operatorcopy.go:11-38`; copy record rows M-C1 to M-C22, `copy:24-45` |
+| the detail path, `/api/cli/apps/<acct>/<app>/instances/<host>/move?detail=1` | `move.go:230-232`, `:351-356`; `apimode.go:460-461` |
+| login tokens in `~/.carlos/credentials`, keyed by `console` | `auth.go:113-120`, `:139-148` |
+
+**The detail answer**
+
+| what | where |
+|---|---|
+| detail=1 only for operators, only with the switch on | `internal/console/apiinstancemove.go:312-327` |
+| `detail.move` is nil once no move is in flight, so the capture loop keeps it | `internal/console/apiinstancemove.go:1291-1292`, `:1416-1423` |
+| a mutation waits for the outcome unless `--no-wait` | `move.go:186-188` |
+| flat fields `state`, `target`, `id`, `last`, `protocol`, `phase` | `internal/console/apiinstancemove.go:160-170` |
+| `detail.move`, `claim`, `target_report`, `source_report`, `evidence`, `log`, `timings`, `from`, `region` | `internal/console/apiinstancemove.go:1290-1318` |
+| `evidence.source` and `evidence.target` come from the record in flight | `internal/console/apiinstancemove.go:1367-1370`, `:1416-1423` |
+| `last` fields `id`, `outcome`, `at`, `offline_s` | `internal/provision/provision.go:1059-1064` |
+| `move.incarnation` | `internal/provision/provision.go:961` |
+| `rollback.dirty`, `target_phase`, `evidence`, `loss_ref`, `lost_at` | `internal/provision/provision.go:1067-1088` |
+| `landing.installed_db_sha256` | `internal/provision/provision.go:1052` |
+| `target_report.checks` T1 to T12, `attempts_admitted`, `cut_budget_s` | `internal/moves/readiness.go:18-31`, `:58-67` |
+| `source_report.s1`, `s2`; S1 refuses a nested instance | `internal/moves/readiness.go:102-109`; `internal/host/movesphase1.go:41`, `:537` |
+
+**The log and the buckets**
+
+| what | where |
+|---|---|
+| identities and events: `requested`, `cutting-requested`/`cut_requested`, `cutting-flipped`/`flipped`, `committed-opened`/`opened`, `committed-source_parked`/`source_parked`, `rolling_back`, `rolling_back-target-<phase>`, `rolling_back-target-lost`/`declared_lost`, `rolling_back-source-<phase>`, `stuck-<epoch>`, the closes; the state as the event for `ready` and `committed` | `internal/moves/transition.go:122-253` |
+| `retry-<epoch>` | `internal/host/movesevents.go:109` |
+| the fields every entry carries | `internal/moves/transition.go:42-71` |
+| the timing keys | `internal/moves/log.go:126-133` |
+| target phases `releasing`, `released`, `returning`, `returned`, `cleaning`, `cleaned`, `lost`; source phases up to `recovered`, `opened` | `internal/provision/provision.go:1148-1172` |
+| released goes to returning only when dirty, else to cleaning | `internal/host/movesrollback.go:566-575` |
+| report prefix `moves/<acct>/<app>/<host>/<op>/`, `source.json`, `target.json`, `stopped-<role>.json`, `log/<identity>.json` | `internal/moves/staging.go:39-50` |
+| the staging lane `moves/<host>/<op>/` in the target's data bucket | `internal/moves/staging.go:6` |
+| the record's account is the sqid | `internal/console/apiinstances.go:1240` |
+| the replica under `<CARLOS_HIBERNATE_PREFIX>/<host>` | `internal/activator/activator.go:119`, `:144-146`; `internal/host/movespass.go:471` |
+| parked objects at `parked/<inc>/<host>/<retirement>/`, a `manifest.json`, the source versions deleted | `internal/park/remote.go:230-232`, `:473`, `:497` |
+| stopped report `steps.tree`, `steps.stray`, `stray` (omitted when empty), digests | `internal/moves/evidence.go:11-20`, `:38-53` |
+| a clean rollback: `src_covered` and unchanged digests; the "clean ..., src_covered ..." line; `dirty` and `evidence` written at released | `internal/host/movesrollback.go:394-398`, `:521-526` |
+| the rollback's `id` | `internal/provision/provision.go:1068` |
+| a dirty rollback's return lane is `moves/<host>/<rb>`, parked in the source's region, queued when the source has probed, horizon the return seal's bound | `internal/moves/parkscope.go:66-72`; `internal/host/movesrecover.go:658-675`; `internal/hostclaim/move.go:1093-1108` |
+| a claim obligation's fields: `id`, `op`, `kind`, `region`, `retirement`, and `park` with `location`, `incarnation`, `evidence_at`, `request_limit_ns`; the claim's `obligations` list | `internal/hostclaim/claim.go:116-121`, `:137-147`, `:166` |
+| the return lane's obligation `region` is the source box's `CARLOS_REGION` (omitted when empty) | `internal/host/movesrecover.go:675`; `internal/host/movespass.go:206`; `internal/hostclaim/claim.go:144` |
+| the claim at `control/hosts/<host>.json` is plain JSON, read as is | `internal/hostclaim/claim.go:178-182`; `internal/hostclaim/store.go:38-60` |
+| the detail answer's claim view carries no obligations | `internal/console/apiinstancemove.go:1356-1362` |
+| a rollback closes only when nothing is owed under its move; a finished park removes its obligation | `internal/host/movesrecover.go:805-809`; `internal/host/movesparks.go:134-142`; `internal/park/remote.go:568`; `internal/hostclaim/delete.go:83-98` |
+| the parked archive is `parked/<park.incarnation>/<host>/<retirement>/` with `manifest.json` | `internal/park/remote.go:230-232`, `:473` |
+| a park runs only after `evidence_at` plus the signature age (15 min) and the request limit (2 min) | `internal/park/remote.go:21`, `:516-519`; `internal/moves/parkscope.go:23` |
+
+**The door and the fleet**
+
+| what | where |
+|---|---|
+| no moves into home, and its sentence | `internal/console/apiinstancemove.go:537-540` |
+| a region with no staging bucket is refused | `internal/console/apiinstancemove.go:580-583`; staging from `data_bucket`, `internal/console/instancesplacement.go:366` |
+| M-C21, more than one box | `internal/console/apiinstancemove.go:595-600` |
+| MV-N5, below the floor | `internal/console/apiinstancemove.go:613-616`; `copy:55` |
+| a box below the protocol, or not armed for the floor, or with an unjudged `carlos.prev`, is below the floor | `internal/console/boxselect.go:100-108` |
+| MV-N167 for a holder at `instances/<host>/<inc>` | `internal/console/apiinstancemove.go:143`, `:647-651`; `copy:217` |
+| a created holder is at `instances/<host>/<inc>`, an adopted one at `instances/<host>` | `internal/hostclaim/create.go:75`; `internal/hostclaim/adopt.go:20` |
+| the source must be an active, enrolled roster entry with no termination fence | `internal/console/apiinstancemove.go:657-684`, `:1267-1273` |
+| eligibility: fresh, enrolled, `region-ha-1`, the console's secrets fingerprint, not draining | `internal/boxchoice/boxchoice.go:21`, `:109-148` |
+| `region-ha-1` and the identity record only with `CARLOS_REGION_HA` | `internal/host/edgesfile.go:156-164`; `internal/host/boxrecord.go:25-28`, `:158-166` |
+| `CARLOS_REGION_HA` reads only ParseBool true values, so `on` is off | `internal/host/boxrecord.go:21-28` |
+| identity records at `control/boxes/<box>.json` | `internal/boxid/record.go:21`, `:28` |
+| `CARLOS_AWS_ACCOUNT` and `CONSOLE_HOME_REGION` on the console | `cmd/console/main.go:318`, `:321` |
+| the claim at `control/hosts/<host>.json`; `holder.location`, `obligations`, `quiet_after` | `internal/hostclaim/claim.go:56`, `:166`, `:171`, `:182` |
+| T12 reads `quiet_after`; H is 15 min plus 1 min skew | `internal/host/movesreport.go:756-762`; `internal/moves/lease.go:13`, `:22`; spec:2573 |
+| the console's claim recovery pass adopts with the claim on | `internal/console/hostclaimbackfill.go:40-48` |
+| the roster at `control/host-actors.json`; `boxes`, `state`, `enrollment`, `quarantine_revision`, `loss_ids`; a declaration quarantines and adds one to the revision | `internal/hostactors/roster.go:21`, `:30`, `:38-40`, `:72`, `:265-279` |
+| the roster is written as a root-signed envelope `{payload, sig}`, the payload base64 JSON | `internal/hostactors/roster.go:249`; `internal/release/rootenvelope.go:29-44`; `internal/release/release.go:377-379` |
+| a termination fence is `control/box-fences/<enrollment>.json`; no object, no fence | `internal/hostactors/fence_authority.go:14-34` |
+| a quarantined box is refused admission | `internal/hostquarantine/quarantine.go:17-25` |
+| `CARLOS_HOST_CLAIM` is any non-empty value | `cmd/console/main.go:1364-1366`; `hostclaim_cli.go:22` |
+| `carlos ops host` exists only with `CARLOS_HOST_CLAIM` in the shell; `--readmit`, `--console`, the `readmit` prompt | `hostclaim_cli.go:35-39`, `:61-75`, `:143` |
+| readmit refuses a fenced box and a private report older than 15 min | `internal/hostresolve/readmit.go:18-40` |
+| readmit needs the box's complete retirement receipt for this boot at `control/host-retirement/<box>/<enrollment>/<revision>.json`, then sets the box `active` | `internal/hostresolve/readmit.go:52-66`, `:134`; `internal/hostquarantine/quarantine.go:72-82`, `:91-96` |
+
+**The floor, the switch and the boxes**
+
+| what | where |
+|---|---|
+| the floor key and shape, `{"min","min_release"}` | `internal/moves/floor.go:20`, `:61-64` |
+| `vX.Y.Z` only; anything else fails closed | `internal/moves/floor.go:85-99` |
+| no code writes the floor; only reads | `internal/moves/floor.go:66-82` |
+| a box's floor only rises; lowering it is a manual step per box | `internal/host/movesfloor.go:31-36` |
+| `moves_protocol`, `moves_floor_min_release`, `prev_release`, `prev_allowed_for`, `data_bucket`, `secrets_key_fp` in the edges file at `control/edges/<box>.json` | `internal/host/edgesfile.go:218`, `:270`, `:295-304`; `internal/console/instancesplacement.go:22`, `:83-86` |
+| `control/moves-console.json` and its `secrets_key_fp` | `internal/moves/floor.go:25-33` |
+| the switch, `on`/`1`/`true`, default off | `internal/moves/switch.go:6-23` |
+| the copy record's `pending` rows keep the compiled default off; P-7 applies the same rule to the env switch (Paul's ruling) | `approved_moves_copy_test.go:811`; `docs/superpowers/plans/2026-10-06-moves-phase1-copy.md` |
+| with the switch off the CLI says "Moves are not enabled on this console." (the switch-off check in Run 5 and C-6) | `move.go:65`, `:240-243` |
+| with the switch off a box's moves pass skips every Phase 1 record, so the switch goes off only after the move closes | `internal/host/movespass.go:797-805` |
+| `carlos move resolve --retry` and `--declare-lost <box>` | `move.go:159-163`, `:258-262` |
+| the copy gate keeps the default off while a row is pending | `approved_moves_copy_test.go:811` |
+| the default cut window, 120 s | `internal/moves/budget.go:17` |
+| `CARLOS_LAYOUT_DEFAULT=flat` | `internal/host/layoutleg.go:68-70`; `system.go:714` |
+| `carlos system status` | `system.go:75-76`; `README.md:272` |
+| `carlos status --layout` shows each host's actual layout, box by box, and marks a box "(nested by default)"; `carlos system layout status` shows only the recorded intent | `cli.go:3528`; `status_instances.go:142-163`, `:224`; `system.go:709-716` |
+| the agent is `carlos-edge.service`, reading `/etc/carlos/host.env`, with a delegated cgroup | `internal/host/units/carlos-edge.service:18-19`, `:101` |
+| "exec-child accounting armed"; no manager, no stop evidence | `internal/cgroup/cgroup.go:181-184`; `internal/host/movespass.go:376-392` |
+| "not proven stopped yet ... retrying until the cut's deadline" | `internal/host/movescut.go:422-427` |
+| M-C20 and the page outbox `<DataDir>/.moves/.page-outbox/<host>/<move>/<rule>.json`, rule `stray` | `internal/host/movespaging.go:47-50`, `:87`, `:546-554`, `:586-588` |
+| the tree step's "could not be stopped" | `internal/stopevidence/reconstruct_linux.go:185` |
+| alert sinks `CARLOS_ALERT_WEBHOOK`, `CARLOS_ALERT_SNS_TOPIC_ARN`; the console arms them only with the claim and the switch on | `internal/alerting/alerting.go:191-209`; `cmd/console/main.go:1352-1362` |
+| `CARLOS_SECRETS_KEY` on the box | `internal/host/movespass.go:311-316` |
+| `/data` is the data directory | `internal/ops/ops.go:27` |
+
+**Timers and moveprobe**
+
+| what | where |
+|---|---|
+| a held timer row is skipped while the instance moves | `internal/schedule/runner.go:190-197` |
+| a periodic schedule's next fire is the next boundary after now (no replay) | `internal/schedule/spec.go:119-125` |
+| moveprobe's region is `home` with no `CARLOS_REGION`, else `CARLOS_REGION`; `/report` says which in `region_from` | `cmd/moveprobe/main.go:94-100` |
+| the failure switch matches the probe's own region, so it must name `<pr-R>` | `cmd/moveprobe/main.go:103-106`, `:137-138` |
+| a box's edges `placement` is its `CARLOS_REGION`, absent when unset (the home box) | `internal/host/edgesfile.go:205-211`, `:758` |
+| `bg` and `worker` rows each numbered from 1 in one sequence | `cmd/moveprobe/main.go:543-550`, `:158`, `:674` |
+| `/report`'s fields, `/arm-once`'s answer, the failure switch's modes | `cmd/moveprobe/README.md`, "Use it", "Make readiness fail on purpose" and "Strings" |
+
+**Other CLI commands**
+
+| what | where |
+|---|---|
+| `carlos env set`, `unset`, `list` with `--app` and `--account` | `appconfig_cli.go:342-350`, `:408-417`, `:462-467`; `cli.go:2045-2051` |
+| `carlos instances create --host --region --account` | `instances_cli.go:297-310` |
+
+**Not on main, cited by name**
+
+| what | where |
+|---|---|
+| the cut guard and T13 | branch `moves-cut-guard`, commits `083a6eaa` and `6871d2e3` |
+| the home parks fix parts 1 and 2 | branch `moves-home-parks` (no commits yet); #236 |
+| claim Plan 12; moves Plan 5 Tasks 9b and 8, Plan 4 Tasks 9 and 10 | plans only; the landing, rollback and recovery are unarmed until then, `internal/host/movespass.go:335-340` |
+| dev's second region | platform-infrastructure#42 |
+
+**Spec**
+
+| what | where |
+|---|---|
+| the runs and the pass rule | spec §14.4, spec:6263-6276 |
+| the rollout order, the floor before any protected production rehearsal | spec §15, spec:6278-6294 |
+| each rehearsal gates the next: Run 4 needs Runs 2 and 3, Run 5 needs Run 4 | spec:6294; controller rulings, fix round 1 and final review |
+| the flagship's home is Ireland | spec:1032 |
diff --git a/internal/console/apiinstancemove_test.go b/internal/console/apiinstancemove_test.go
index af45ff58c..ba1d8b63f 100644
--- a/internal/console/apiinstancemove_test.go
+++ b/internal/console/apiinstancemove_test.go
@@ -471,6 +471,10 @@ func TestMoveDoorRefusesTheFleet(t *testing.T) {
unarmed.FloorUnarmed = true
unverified := ukBox("uk-box")
unverified.Unverified = true
+ // A box from before moves: its edges file has no moves_protocol, which
+ // decodes as 0, the same as an explicit 0 (§14.3).
+ preProtocol := ukBox("uk-box")
+ preProtocol.NoProtocol = true
for _, c := range []struct {
name, floor string
boxes []fleetBox
@@ -482,6 +486,8 @@ func TestMoveDoorRefusesTheFleet(t *testing.T) {
http.StatusBadRequest, "uk has more than one box. Moves between regions with several boxes are not supported yet."},
{"a box that has not armed the floor", floor, []fleetBox{ieBox("ie-box"), unarmed}, "",
http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"},
+ {"a pre-protocol box, no floor published", "", []fleetBox{ieBox("ie-box"), preProtocol}, "",
+ http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"},
{"no verified box in the target", "", []fleetBox{ieBox("ie-box"), unverified}, "",
http.StatusBadRequest, "no box in uk publishes an edges file yet"},
{"no eligible box in the source", "", []fleetBox{ukBox("uk-box"), {Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}}, "",
diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go
index e7ba07d95..7b1e1934b 100644
--- a/internal/provision/provision_test.go
+++ b/internal/provision/provision_test.go
@@ -979,9 +979,14 @@ func TestSetRecordMoveTransitions(t *testing.T) {
// Abort names the states it may leave, so a state a newer box writes is
// refused here rather than aborted (G16): "landing" is the target restoring
-// from the staging an abort would delete.
+// from the staging an abort would delete. The Phase 1 states past the cut
+// (committed, rolling_back, reverting, stuck) are refused the same way. This
+// is the real function, not a copy: v0.22.2's MoveAbort
+// (internal/provision/provision.go:1022-1039 at that tag) allows exactly the
+// same cases, preparing and ready, so a v0.22 console refuses all of them too
+// (§14.3).
func TestMoveAbortRefusesStatesItDoesNotKnow(t *testing.T) {
- for _, state := range []string{"landing", "freezing", "done", ""} {
+ for _, state := range []string{MoveLanding, MoveCommitted, MoveRollingBack, MoveReverting, MoveStuck, "freezing", "done", ""} {
rec := &Record{Move: &MoveState{To: "uk", State: state}}
if err := MoveAbort()(rec); !errors.Is(err, ErrMoveConflict) {
t.Errorf("abort from %q = %v, want ErrMoveConflict", state, err)