carlos / platform Public

moves-offline-fixes: harness Run 9 defects. Config sync writes a nested row's config.env as soon as its instance directory appears, keeping it pending until delivered (6); each offline door step sees a current report: source boxes and the sealer refresh reports every 15 s while a move is active without renewing the tombstone, and a stale report answers MV-O178 (7); a source box's scheduler leaves a held or moved host alone, excluding it before the batch limit (8). Astra reviewed (two rounds); census-ruling-moves-offline-fixes (platform-1c); MV-O178 approved by Paul.

Squashed from moves-offline-fixes into main.

claude 4 days ago e0986eabc5bc2a7ae5e17e58b19c9a6b92beecfe parent 2185035

19 files changed, +1474 −52

approved_moves_copy_test.go +1 −0
docs/superpowers/notes/census-proposal-moves-offline-fixes.md +93 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +6 −0
docs/superpowers/plans/CODEX-BRIEF-MOVES-OFFLINE-FIXES.md +16 −0
internal/appconfig/syncexec.go +63 −1
internal/console/apiinstancemove_offline.go +31 −6
internal/console/apiinstancemove_offline_test.go +9 −4
internal/host/agent.go +161 −28
internal/host/configdirpending_test.go +264 −0
internal/host/movesoffline.go +118 −1
internal/host/movesoffline_e2e_test.go +248 −5
internal/host/movesoffline_seal.go +51 −0
internal/host/movesoffline_seal_test.go +72 −0
internal/host/movesoffline_test.go +109 −0
internal/host/schedule_wiring.go +24 −0
internal/provision/plan14_root_reads_test.go +16 −0
internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv +31 −0
internal/schedule/runner.go +69 −7
internal/schedule/runner_test.go +92 −0

CI passed — run details

CI log

=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	316.211s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	2.886s
ok  	github.com/carlosframework/platform/cmd/moveprobe	93.243s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.707s
ok  	github.com/carlosframework/platform/internal/activator	250.798s
ok  	github.com/carlosframework/platform/internal/agentsock	0.015s
ok  	github.com/carlosframework/platform/internal/alerting	0.005s
ok  	github.com/carlosframework/platform/internal/analytics	0.019s
ok  	github.com/carlosframework/platform/internal/appconfig	2.438s
ok  	github.com/carlosframework/platform/internal/apperr	0.072s
ok  	github.com/carlosframework/platform/internal/applog	0.126s
ok  	github.com/carlosframework/platform/internal/awsinv	0.007s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.679s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.008s
ok  	github.com/carlosframework/platform/internal/caps	0.012s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.710s
ok  	github.com/carlosframework/platform/internal/changelog	0.335s
ok  	github.com/carlosframework/platform/internal/claimactivation	0.762s
ok  	github.com/carlosframework/platform/internal/clamd	0.054s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	415.313s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.203s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.010s
ok  	github.com/carlosframework/platform/internal/dbverify	0.004s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.005s
ok  	github.com/carlosframework/platform/internal/domains	2.394s
ok  	github.com/carlosframework/platform/internal/doorkey	0.006s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	45.014s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.131s
ok  	github.com/carlosframework/platform/internal/gate	1.655s
ok  	github.com/carlosframework/platform/internal/genselect	0.014s
ok  	github.com/carlosframework/platform/internal/geo	0.018s
ok  	github.com/carlosframework/platform/internal/geoip	0.030s
ok  	github.com/carlosframework/platform/internal/historyreadback	0.159s
ok  	github.com/carlosframework/platform/internal/host	188.904s
ok  	github.com/carlosframework/platform/internal/hostactors	6.052s
ok  	github.com/carlosframework/platform/internal/hostactors/awsmembership	0.005s
ok  	github.com/carlosframework/platform/internal/hostbox	2.679s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.644s
ok  	github.com/carlosframework/platform/internal/hostenv	0.006s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.012s
ok  	github.com/carlosframework/platform/internal/hostexec	5.011s
ok  	github.com/carlosframework/platform/internal/hostfence	0.035s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.464s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.016s
ok  	github.com/carlosframework/platform/internal/hostguard	1.003s
ok  	github.com/carlosframework/platform/internal/hostops	1.642s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.005s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.003s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.505s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.146s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.003s
ok  	github.com/carlosframework/platform/internal/infraview	0.003s
ok  	github.com/carlosframework/platform/internal/instancectl	5.225s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.733s
ok  	github.com/carlosframework/platform/internal/instancefs	0.011s
ok  	github.com/carlosframework/platform/internal/instancekey	0.004s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.010s
ok  	github.com/carlosframework/platform/internal/layout	1.577s
ok  	github.com/carlosframework/platform/internal/lbhealth	0.010s
ok  	github.com/carlosframework/platform/internal/lease	9.922s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.396s
ok  	github.com/carlosframework/platform/internal/ledger	0.014s
ok  	github.com/carlosframework/platform/internal/mail	0.185s
ok  	github.com/carlosframework/platform/internal/maintenance	0.080s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.525s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.100s
ok  	github.com/carlosframework/platform/internal/metrics	0.005s
ok  	github.com/carlosframework/platform/internal/moves	21.238s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.835s
ok  	github.com/carlosframework/platform/internal/objstore	0.205s
ok  	github.com/carlosframework/platform/internal/ops	0.188s
ok  	github.com/carlosframework/platform/internal/park	0.244s
ok  	github.com/carlosframework/platform/internal/privatefile	0.005s
ok  	github.com/carlosframework/platform/internal/provision	37.453s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.008s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	5.532s
ok  	github.com/carlosframework/platform/internal/relay	7.229s
ok  	github.com/carlosframework/platform/internal/release	24.589s
ok  	github.com/carlosframework/platform/internal/rollout	3.776s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.004s
ok  	github.com/carlosframework/platform/internal/rootaws	0.008s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.064s
ok  	github.com/carlosframework/platform/internal/rootstate	0.027s
ok  	github.com/carlosframework/platform/internal/runs	0.578s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.005s
ok  	github.com/carlosframework/platform/internal/schedule	2.584s
ok  	github.com/carlosframework/platform/internal/seeds	0.011s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	9.695s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.100s
ok  	github.com/carlosframework/platform/internal/status	0.068s
ok  	github.com/carlosframework/platform/internal/steering	0.006s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.321s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.003s
ok  	github.com/carlosframework/platform/internal/updatewords	0.004s
ok  	github.com/carlosframework/platform/internal/usage	0.007s
ok  	github.com/carlosframework/platform/internal/version	0.002s
ok  	github.com/carlosframework/platform/internal/vet	1.586s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 49807522 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.03s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.02s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.033s
=== RUN   TestT4StorageProbeMinio
2026/10/20 21:01:17 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18E0589BBD8A5D47, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.026s
minio contract: PASS

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index 47f478a5b..99219804e 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -105,6 +105,7 @@ var movesCopyIDs = []string{

"MV-O142", "MV-O143", "MV-O144", "MV-O145", "MV-O146", "MV-O147", "MV-O148", "MV-O149", "MV-O150", "MV-O151", "MV-O152", "MV-O153", "MV-O154", "MV-O155", "MV-O156", "MV-O157",

"MV-O158", "MV-O159", "MV-O160", "MV-O161", "MV-O162", "MV-O163", "MV-O164", "MV-O165", "MV-O166", "MV-O167", "MV-O168", "MV-O169", "MV-O170", "MV-O171",

"MV-O172", "MV-O173", "MV-O174", "MV-O175", "MV-O176", "MV-O177",

+ "MV-O178",

}

// copyRow is one row of the record. The File cell is the file the sentence is

diff --git a/docs/superpowers/notes/census-proposal-moves-offline-fixes.md b/docs/superpowers/notes/census-proposal-moves-offline-fixes.md

new file mode 100644

index 000000000..ff13f68ec

--- /dev/null

+++ b/docs/superpowers/notes/census-proposal-moves-offline-fixes.md

@@ -0,0 +1,93 @@

+# Census: the offline move's harness fixes (Run 9, defects 6 to 8)

+Status: RULED. Every row carries `rev3=PROPOSED-moves-offline-fixes`. The

+ruling is asked of the census owner, platform-1c.

+The branch is `moves-offline-fixes`, from main after 54ae7fb9 (v0.28.5). It

+fixes three defects the deployment harness found in the offline operator move

+(harness RUNLOG, Run 9). It adds one census overlay,

+`internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv`,

+loaded last in `TestPlan14RootReadCensus`, after armed-box-updates. The

+record-write census (`TestPlan9B8RecordWriteCensus`) does not change: no

+record write is added or moved.

+## The footing

+Every unit in the overlay runs in the box agent (`/opt/carlos/carlos agent`,

+carlos-edge.service, `User=carlos`). None is reached from a root process, and

+none makes a root decision. So each is proposed accepted legacy at the

+disposition main gives it or its siblings: J1 (root-route) for registry and

+route reads, J4 (root-input) for file reads, J1,J4 where a unit does both.

+That is the footing census-ruling-moves-offline accepted for the offline

+move's own units.

+## The units, by defect

+### Defect 6: config.env for a nested row whose instance directory is not made yet

+|---|---|---|---|

+| `internal/appconfig/syncexec.go` `SyncExecWithCallbacks` | changed legacy: a write that found no instance directory is named and skipped (`errInstanceDirAbsent`, `WhenInstanceDirAbsent`), and the pass goes on; no new read site | the agent, through `syncExecSerialized` (config loops) | re-frozen, J1,J4 |

+| `internal/appconfig/syncexec.go` `renderExecConfigBefore` | changed legacy: one `Lstat` of the nested row's instance directory after its write failed, to tell "not made yet" from any other failure; follows no link | the agent: config sync and the moves worker's held render | re-frozen, J1,J4 |

+| `internal/host/agent.go` `configGenTick` | new: `configGenLoop`'s tick over the routes it listed; also runs the agent's own config pass once a directory the last pass waited for exists | the agent | accepted legacy, J1 (as `configGenLoop`) |

+| `internal/host/agent.go` `pendingDirs.appeared` | new: an `Lstat` (never follows a link) of the instance directories the last config pass could not write into | the agent | accepted legacy, J4 (as the moves units' `isDir`, `lexists`) |

+Envelopes re-pinned: `internal/appconfig`, `internal/host`.

+### Defect 7: fresh reports at every door step

+No unit row. The re-assert (`movesWorker.offlineReassert`, run by

+`offlineReassertLoop` in the agent every 15 s or so) reads the record from the

+deployment store and calls the source and sealer steps

+census-ruling-moves-offline already classified (`offlineSourceStop`,

+`offlineKeepStop`); it has no root read of its own. The console's stale-report

+refusal (MV-O178) reads nothing new. Envelopes re-pinned: `internal/console`,

+`internal/host`.

+### Defect 8: the source's scheduler wrote the moved host's env file after open

+The schedule runner is built only by the agent (`armSchedule`, from

+`runAgent`); no root process makes one.

+|---|---|---|---|

+| `internal/host/schedule_wiring.go` `scheduleSkip` | new: reads a row's status (any hold) and host (the armed offline view says the record names another region) to tell the runner to leave the host alone; it only withholds a tick | the agent's schedule runner | accepted legacy, J1 (as the runner's own hold checks) |

+| `internal/schedule/runner.go` `Runner.Tick` | changed legacy: a host `Skip` names gets no tick and no bookkeeping at the pre-filter | the agent's schedule runner | re-frozen at census-ruling-seeded-instances' disposition |

+| `internal/schedule/runner.go` `Runner.fire` | re-pinned: with `Skip` set, the row is read again before the token, and a skipped host gets no token and no record | the agent's schedule runner (overlay e names the service controller and root callbacks for the claim design; neither sets `Skip`) | hint, E2, unchanged |

+Envelopes re-pinned: `internal/host`, `internal/schedule`.

+### Fix round 1 (Astra's review)

+|---|---|---|---|---|

+| 1 | `internal/schedule/runner.go` `Runner.dueRows` | new: held hosts left out by the query when `Skip` is set, and hosts `Skip` names paged past before the limit (one row read per host in the batch) | the agent's schedule runner | accepted legacy, J1 (as `Runner.Tick`) |

+## The seal

+The accepted-legacy rows add frozen declarations, so the legacy seal moves,

+from `393e7f45…bec29` (main, after moves-offline and armed-box-updates) to

+`edc04f7349acf2b9ea98ffc8a1d3b71516a6ddb95505abfc2e38e49fd802ccc2` (after fix round 1). The test

+checks main's seal before it loads this overlay, so a change on main's side

+still shows on its own.

+## What the ruling is asked to decide

+1. That every unit above runs only in the agent and makes no root decision,

+ so accepted legacy at main's disposition is the right footing.

+2. That `Runner.fire` keeps its class and task (hint, E2): the change only

+ withholds a token and a record for a host `Skip` names.

+3. That the re-assert of defect 7 needs no row: it calls only steps already

+ classified under census-ruling-moves-offline.

+## Ruling (platform-1c, 2026-10-20): census-ruling-moves-offline-fixes, ACCEPTED as proposed

+Accepted at d75b7e0b (F1 to F4) and re-ruled for fix round 1 at 7cd5db4d (F5 to F7). After Astra's scoped re-review, the backoff counter saturates (agent.go `pendingDirs.record`): an internal/host envelope re-pin only. The rev3 tag was rewritten to the ruling's name, so the seal moves to 826c54be…a62f.

+- After merging main 21850356 (install-console-upgrade, loaded before this overlay): internal/host envelope re-pinned to the combined tree; seal unchanged.

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index 504dcc7a4..10640ff9c 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -682,3 +682,9 @@ The landing's item reasons (`internal/host/movesoffline_land.go`, Task 5 fix rou

+A door's answer to a box report it cannot believe for its age (harness Run 9, finding 7). `flip` answered MV-O31 for a source report older than two minutes, and an operator who reads "still runs a process" goes looking for a process that is not there. `seal`, `flip` and `open` now answer MV-O178 for any report that is there but stale, the sealer's included; a report that is missing keeps M-C10. A box taking part in a move rewrites its report about every 15 seconds, so the step is retried rather than looked into. `<box>` is the box whose report it is. Pending until Paul reviews it.

+|---|---|---|---|

diff --git a/docs/superpowers/plans/CODEX-BRIEF-MOVES-OFFLINE-FIXES.md b/docs/superpowers/plans/CODEX-BRIEF-MOVES-OFFLINE-FIXES.md

new file mode 100644

index 000000000..3d93c14f2

--- /dev/null

+++ b/docs/superpowers/plans/CODEX-BRIEF-MOVES-OFFLINE-FIXES.md

@@ -0,0 +1,16 @@

+# Review brief: moves-offline-fixes (harness Run 9 defects 6, 7, 8)

+You are an independent reviewer (Astra). Review branch `moves-offline-fixes` against `origin/main` (`git diff origin/main...HEAD`). It fixes three defects the deployment harness found in the offline operator move (docs/superpowers/plans/2026-10-20-offline-operator-move.md is the design):

+- 6: config sync skipped writing `<host>.config.env` for a nested row whose instance dir did not exist yet and only retried hourly. It now skips the row and the 3 s config loop writes it on the first tick after the dir appears.

+- 7: the console doors trust box reports for 2 minutes but boxes rewrote them only every 5-6 minutes, so a stale report at flip was refused as "still runs a process". Now, while a move is between start and landed, source boxes and the sealer re-assert their reports every 15 s, and a stale report answers MV-O178.

+- 8: after open, the source's scheduler re-created the moved instance's env file with a fresh admin token. The scheduler now skips held rows and hosts whose move record names another region.

+Check especially:

+- Dormant behaviour: with no offline move anywhere, nothing changes except defect 6's general config-sync fix. Is that fix safe for every non-move instance? Could skipping a row hide a real failure, or loop?

+- Re-assert in 7 must never re-run seal or landing, never re-take or extend the tombstone, and never make a stale or wrong claim fresh: a re-asserted report must reflect current box state, not copy old state forward with a new timestamp.

+- Could MV-O178 mask a real "process still running" case?

+- 8 must not stop timers for instances that are NOT moving, and must not drop a due timer that should fire on the target.

+- Every operator string is a recorded copy row, and no err.Error() reaches an operator.

+Write findings by severity (BLOCKER/HIGH/MEDIUM/LOW), each with file:line, a failure scenario and a fix. Verify against the code. End with a verdict: MERGE, MERGE AFTER FIXES, or DO NOT MERGE.

diff --git a/internal/appconfig/syncexec.go b/internal/appconfig/syncexec.go

index e693235a0..493627a7a 100644

--- a/internal/appconfig/syncexec.go

+++ b/internal/appconfig/syncexec.go

@@ -439,11 +439,15 @@ func SyncExecWithCallbacks(ctx context.Context, reg *registry.Registry, s releas

// the errors.As branch below.

said := map[appKey]bool{}

+ // done is WithHostOutcome's answer per in-scope host: false until the

+ // host's file is as this pass resolved it.

+ done := map[string]bool{}

for _, rt := range routes {

in, addrShape, sidecarShape := inScope(rt)

if !in {

continue

}

+ done[rt.Host] = false

// A host the layout pass holds has its files between two

// directories, and either path this pass could pick would be the

// wrong one by the time the pass finishes. Nothing is written or

@@ -635,9 +639,31 @@ func SyncExecWithCallbacks(ctx context.Context, reg *registry.Registry, s releas

hostCallbacks.Delivered = nil

}

- if err := syncExecHostClaimed(ctx, selected[rt.Host], rt, m, Key(k.acct, k.app), priv, dataDir, hostBounce, header, instETag, ack, hostCallbacks); err != nil && hostWrites.Enabled {

+ err := syncExecHostClaimed(ctx, selected[rt.Host], rt, m, Key(k.acct, k.app), priv, dataDir, hostBounce, header, instETag, ack, hostCallbacks)

+ // A nested row's bundle lives in its instance directory, which the

+ // activator makes at the first wake. Before then the write cannot

+ // land, so the row is named and the pass goes on to the other

+ // routes, claims on or off; the agent runs the pass again once the

+ // directory is there (WhenInstanceDirAbsent). A render that failed

+ // fails as before, directory or not.

+ if errors.Is(err, errInstanceDirAbsent) {

+ fmt.Printf(" %s: instance directory not made yet; config is written once it is\n", rt.Host)

+ if o.dirAbsent != nil {

+ o.dirAbsent(rt.Host, filepath.Dir(ConfigEnvPath(dataDir, rt)))

+ }

+ continue

+ }

+ if err != nil && hostWrites.Enabled {

return err

}

+ if err == nil {

+ done[rt.Host] = true

+ }

+ if o.outcome != nil {

+ for host, ok := range done {

+ o.outcome(host, ok)

+ }

}

// Stale-secrets notice (issue #18 design ruling, ratified 2026-08-03

// review): an addr-shaped route whose app is DEFINITIVELY not opted in

@@ -726,10 +752,25 @@ type SyncOption func(*syncOptions)

type syncOptions struct {

fenced func(host string) bool

+ dirAbsent func(host, dir string)

+ outcome func(host string, done bool)

hostWrites hostwrite.Config

hostWriteCount int

}

+// WhenInstanceDirAbsent hands SyncExec a callback for every nested row whose

+// bundle it could not write because the row's instance directory does not

+// exist yet: the activator makes that directory at the instance's first

+// wake (prepare), so a nested row created with settings has nowhere to put

+// its <host>.config.env until then. The pass skips such a row cleanly, says

+// so, and calls dirAbsent with the host and the directory it waits for, so

+// the agent can run the pass again as soon as the directory appears rather

+// than leaving the instance without its settings until the hourly floor

+// (harness Run 9, finding 6).

+func WhenInstanceDirAbsent(dirAbsent func(host, dir string)) SyncOption {

+ return func(o *syncOptions) { o.dirAbsent = dirAbsent }

func WithHostWrite(cfg hostwrite.Config) SyncOption {

return func(o *syncOptions) {

o.hostWrites = cfg

@@ -748,6 +789,20 @@ func SkipFenced(fenced func(host string) bool) SyncOption {

return func(o *syncOptions) { o.fenced = fenced }

}

+// WithHostOutcome hands SyncExec a callback it calls, once a pass has gone

+// through every route without stopping, for each in-scope route: done is

+// true when the route's file is as the pass resolved it (written, already up

+// to date, removed, or rightly left absent), and false when the pass left it

+// as it was for any reason (an unresolved bundle or overlay, a guard, a

+// layout migration in flight, a write that failed, an instance directory not

+// made yet). A route out of scope is not named. The agent keeps a host whose

+// instance directory it waits for until the host is done (fix round 1,

+// Astra's finding 3: a pass that skipped a host on a transient read failure

+// returned nil, and the agent took that as delivered).

+func WithHostOutcome(outcome func(host string, done bool)) SyncOption {

+ return func(o *syncOptions) { o.outcome = outcome }

// execRouteBase is the lifecycle-managed-instance shape every exec route

// kind shares: the unit path's syncable() exclusions, narrowed to

// non-unit backing (unit routes belong to the root adopt path). Neither

@@ -1311,6 +1366,10 @@ func syncExecHostClaimed(ctx context.Context, work hostwrite.Work, rt registry.R

return nil

}

+// errInstanceDirAbsent is a nested row's write that found no instance

+// directory: the activator has not woken the instance here yet.

+var errInstanceDirAbsent = errors.New("the instance directory is not made yet")

// execRender is what renderExecConfig did: the file's path, the bytes it

// holds now, whether this call wrote them, and the identity the writes were

// guarded under (zero when nothing was written).

@@ -1433,6 +1492,9 @@ func renderExecConfigBefore(ctx context.Context, work hostwrite.Work, rt registr

// temp-then-rename it always used.

if nestedRoute(dataDir, rt) {

if err := guarded(ctx, work, id, func() error { return instancefs.WriteAtomic(filepath.Dir(path), filepath.Base(path), rendered, 0o600) }); err != nil {

+ if _, lerr := os.Lstat(filepath.Dir(path)); errors.Is(lerr, os.ErrNotExist) {

+ return execRender{}, fmt.Errorf("%w: %s", errInstanceDirAbsent, filepath.Dir(path))

+ }

fmt.Printf(" %s: write %s: %v\n", rt.Host, path, err)

return execRender{}, err

}

diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go

index 4d0813f9a..74d178a72 100644

--- a/internal/console/apiinstancemove_offline.go

+++ b/internal/console/apiinstancemove_offline.go

@@ -53,7 +53,10 @@ import (

const offlineMoveCapability = "offline-move-1"

// offlineReportFresh is how recently a box's report must be written for a

-// door to believe it. Boxes rewrite their reports on every moves pass.

+// door to believe it. Boxes rewrite their reports on every moves pass, and a

+// source or sealer box every 15 s or so while its report is one a door reads

+// (internal/host offlineReassertEvery), so a report older than this is from a

+// box that has stopped reporting, and the door says so (MV-O178).

const offlineReportFresh = 2 * time.Minute

// Every sentence a door speaks has its row in the copy record (MV-O20 to

@@ -113,6 +116,16 @@ func (s *Server) offlineWaitRefusal(box, from, host string) *offlineRefusal {

return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("Waiting for %s in %s to confirm it has stopped %s. %s is offline.", box, from, host, host)}

}

+// offlineStaleRefusal is MV-O178: box's report is there but older than

+// offlineReportFresh. That says nothing about the box's process or its stop,

+// only that the door cannot believe the report yet; a box taking part in a

+// move rewrites it every few seconds, so the step is retried, not looked into

+// (harness Run 9, finding 7: flip answered MV-O31 for a stale report, and an

+// operator went looking for a process that was not there).

+func offlineStaleRefusal(box string) *offlineRefusal {

+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("the report from %s is not current. it reports again within a minute. try again then", box)}

// offlineRefusal is what a door refuses with: the status and the sentence.

type offlineRefusal struct {

status int

@@ -798,7 +811,10 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis

var data []string

for _, b := range src {

rep, ok := ev.get(moves.OfflineRoleSource, b.Box)

- if !ok || !offlineFresh(rep.OfflineReport, now) || !offlineStopped(rep.OfflineReport) || offlineServes(b, host) {

+ if ok && !offlineFresh(rep.OfflineReport, now) {

+ return "", nil, offlineStaleRefusal(b.Box)

+ }

+ if !ok || !offlineStopped(rep.OfflineReport) || offlineServes(b, host) {

return "", nil, s.offlineWaitRefusal(b.Box, from, host)

}

boxes = append(boxes, b.Box)

@@ -841,8 +857,11 @@ func (s *Server) offlineFlipCheck(ev offlineEvidence, mv *provision.MoveState, h

}

for _, box := range mv.Boxes {

rep, ok := ev.get(moves.OfflineRoleSource, box)

- if !ok || !offlineFresh(rep.OfflineReport, now) {

- return processRefusal(box, host)

+ if !ok {

+ return s.offlineWaitRefusal(box, from, host)

+ }

+ if !offlineFresh(rep.OfflineReport, now) {

+ return offlineStaleRefusal(box)

}

if !offlineStopped(rep.OfflineReport) {

return s.offlineWaitRefusal(box, from, host)

@@ -874,12 +893,18 @@ func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, h

return offlineStateRefusal(mv.State, host)

}

seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer)

- if !ok || !offlineFresh(seal.OfflineReport, now) || seal.TombstoneOwner != moves.TombstoneOwner(mv.ID) || !seal.TombstoneUntil.After(now) {

+ if ok && !offlineFresh(seal.OfflineReport, now) {

+ return offlineStaleRefusal(mv.Sealer)

+ }

+ if !ok || seal.TombstoneOwner != moves.TombstoneOwner(mv.ID) || !seal.TombstoneUntil.After(now) {

return offlineStateRefusal(mv.State, host)

}

for _, box := range mv.Boxes {

rep, ok := ev.get(moves.OfflineRoleSource, box)

- if !ok || !offlineFresh(rep.OfflineReport, now) || !offlineStopped(rep.OfflineReport) {

+ if ok && !offlineFresh(rep.OfflineReport, now) {

+ return offlineStaleRefusal(box)

+ }

+ if !ok || !offlineStopped(rep.OfflineReport) {

return s.offlineWaitRefusal(box, from, host)

}

diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go

index acf194a07..e588d9da8 100644

--- a/internal/console/apiinstancemove_offline_test.go

+++ b/internal/console/apiinstancemove_offline_test.go

@@ -428,6 +428,9 @@ func msgO25(state string) string {

func msgMC10(box string) string {

return "Waiting for " + box + " in ie to confirm it has stopped " + offlineHost + ". " + offlineHost + " is offline."

}

+func msgO178(box string) string {

+ return "the report from " + box + " is not current. it reports again within a minute. try again then"

func msgO31(box string) string {

return box + " still runs a process for " + offlineHost + ". wait for it to stop, then flip"

}

@@ -732,7 +735,7 @@ func TestOfflineSealDoor(t *testing.T) {

r.At = time.Now().Add(-5 * time.Minute)

}

}, "ie-a", "ie-b")

- w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))

+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO178("ie-b"))

// A box still listing the host has not stopped serving it.

w.stopped(nil, "ie-a", "ie-b")

still := ieOff("ie-b")

@@ -830,8 +833,10 @@ func TestOfflineFlipDoor(t *testing.T) {

w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))

w.stopped(func(_ string, r *moves.OfflineReport) { r.LayoutJournal = true }, "ie-box")

w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))

+ // A stale report is not a process (harness Run 9, finding 7): MV-O178,

+ // never MV-O31.

w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-3 * time.Minute) }, "ie-box")

- w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))

+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO178("ie-box"))

w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")

w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgMC10("ie-box"))

// All clear.

@@ -876,7 +881,7 @@ func TestOfflineOpenDoorRechecksTheStop(t *testing.T) {

r.TombstoneOwner = moves.TombstoneOwner(w.id)

r.TombstoneUntil = time.Now().Add(time.Hour)

})

- w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))

+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO178("ie-box"))

reset()

// A tombstone that is another move's, or none, or run out.

for name, edit := range map[string]func(*moves.OfflineReport){

@@ -901,7 +906,7 @@ func TestOfflineOpenDoorRechecksTheStop(t *testing.T) {

w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")

w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))

w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-10 * time.Minute) }, "ie-box")

- w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))

+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO178("ie-box"))

reset()

// The seal changed under the move.

sealKey := moves.OfflineSealKey(offlineHost, w.id)

diff --git a/internal/host/agent.go b/internal/host/agent.go

index ce93fb963..483700f2d 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -1375,17 +1375,119 @@ var configSyncMu sync.Mutex

func syncExecSerialized(ctx context.Context, reg *registry.Registry, store release.Store, priv *ecdh.PrivateKey, dd string, svc *activator.Service, writes ...hostwrite.Config) error {

configSyncMu.Lock()

defer configSyncMu.Unlock()

- opts := []appconfig.SyncOption{appconfig.SkipFenced(func(host string) bool { return layout.JournalPresent(dd, host) })}

+ pending := map[string]string{}

+ outcomes := map[string]bool{}

+ opts := []appconfig.SyncOption{

+ appconfig.SkipFenced(func(host string) bool { return layout.JournalPresent(dd, host) }),

+ appconfig.WhenInstanceDirAbsent(func(host, dir string) { pending[host] = dir }),

+ appconfig.WithHostOutcome(func(host string, done bool) { outcomes[host] = done }),

+ }

for _, cfg := range writes {

opts = append(opts, appconfig.WithHostWrite(cfg))

}

// CARLOS_SWAP=overlap: the scheduler restarts, paced, and acknowledges a

// revision once it is active; refusing: no restart. configSyncMu above

// is what makes ConfigDelivered serial per host.

+ var err error

if cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{awake: managedAwake(svc, reg), restart: bounce(svc)}); ok {

- return appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, dd, os.Getenv("PLATFORM_APPS_DOMAIN"), cb, opts...)

+ err = appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, dd, os.Getenv("PLATFORM_APPS_DOMAIN"), cb, opts...)

+ } else {

+ err = appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"), configAckFunc(ctx, store, "agent"), opts...)

+ }

+ configDirsPending.record(pending, outcomes, err == nil)

+ return err

+// configDirsPending is what the last config pass could not write because a

+// nested row's instance directory did not exist yet (harness Run 9, finding

+// 6): host -> the directory. The activator makes that directory at the

+// first wake, and before this the only pass that came back for the file was

+// configSyncLoop's hourly one, so a nested instance created with settings

+// woke and ran without them, its store binding included, for up to an hour.

+// configGenLoop now looks at these directories on every tick and runs the

+// pass again as soon as one is there.

+var configDirsPending pendingDirs

+type pendingDirs struct {

+ mu sync.Mutex

+ dirs map[string]*pendingDir

+// pendingDir is one host the config pass waits for: its instance directory,

+// and, once the directory is there, how many passes since have left the host

+// undone and how many ticks to let go by before the next retry.

+type pendingDir struct {

+ dir string

+ failures int

+ waitTicks int

+// pendingRetryCap bounds the retry backoff: at most 16 generation-loop ticks

+// (about a minute) between passes for a host whose directory is there but

+// whose config a pass keeps leaving undone (a bundle that will not read).

+const pendingRetryCap = 16

+// record keeps what one pass found: absent, the hosts whose instance

+// directory was not there; outcomes, every in-scope host the pass named,

+// done or not (appconfig.WithHostOutcome). A host stays waited for until a

+// pass reports it done (fix round 1, Astra's finding 3: a pass that left it

+// undone on a read failure still returned nil), or until a complete pass no

+// longer has it in scope (its row is gone or no longer takes a bundle). A

+// pass that failed part way names fewer hosts than it might have, so it only

+// adds. A host a pass leaves undone with its directory there backs off,

+// doubling from the next tick up to pendingRetryCap ticks.

+func (p *pendingDirs) record(absent map[string]string, outcomes map[string]bool, complete bool) {

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ if p.dirs == nil {

+ p.dirs = map[string]*pendingDir{}

+ }

+ for host, e := range p.dirs {

+ if _, again := absent[host]; again {

+ continue

+ }

+ done, named := outcomes[host]

+ switch {

+ case named && done, !named && complete:

+ delete(p.dirs, host)

+ case named:

+ // The count saturates once the delay reaches the cap, so the

+ // shift below never overflows however long a host keeps failing.

+ if 1<<e.failures <= pendingRetryCap {

+ e.failures++

+ }

+ e.waitTicks = min(1<<(e.failures-1), pendingRetryCap) - 1

+ }

+ for host, dir := range absent {

+ if e, ok := p.dirs[host]; ok {

+ e.dir = dir

+ continue

+ }

+ p.dirs[host] = &pendingDir{dir: dir}

}

- return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"), configAckFunc(ctx, store, "agent"), opts...)

+// appeared names a host whose instance directory a pass waited for, which

+// exists now, and whose retry is due; ok is false when there is none. Each

+// call is one tick of the backoff. Lstat only: the directory is the

+// activator's, and nothing here follows or opens it.

+func (p *pendingDirs) appeared() (host string, ok bool) {

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ for h, e := range p.dirs {

+ if _, err := os.Lstat(e.dir); err != nil {

+ continue

+ }

+ if e.waitTicks > 0 {

+ e.waitTicks--

+ continue

+ }

+ if !ok {

+ host, ok = h, true

+ }

+ return host, ok

}

// configAckFunc builds the callback config sync's two passes (the root

@@ -1906,33 +2008,47 @@ func configGenLoop(reg *registry.Registry, svc *activator.Service, dd string, wr

log.Printf("agent: config generation poll: list routes: %v", err)

continue

}

- execChanged, unitChanged, readErrs := genPollPass(ctx, routes, store, cache)

- if len(readErrs) > 0 {

- // ONE line per pass, never one per app: a bucket having a bad

- // minute would otherwise flood the journal at 20 lines a minute

- // per app. The count carries the scale, the first error carries

- // the cause; they are the same cause in practice.

- log.Printf("agent: config generation poll: %d apps unreadable: %v", len(readErrs), readErrs[0])

- }

- if unitChanged {

- // Requested first and never fatal: the write is instant while

- // syncExecSerialized can block on a serialized pass, so the root

- // run gets to start alongside it rather than behind it. A failed

- // marker write costs those apps the wait until

- // carlos-adopt.timer's next hourly pass — never correctness.

- if err := requestRootSync(dd); err != nil {

- log.Printf("agent: request root config sync: %v", err)

- } else {

- log.Printf("agent: config changed for unit-backed app(s), requested root config sync pass")

- }

- if !execChanged {

- continue

- }

- if err := syncExecSerialized(ctx, reg, store, priv, dd, svc, writes); err != nil {

- log.Printf("agent: config generation sync: %v", err)

+ configGenTick(ctx, routes, reg, store, priv, dd, svc, writes, cache)

+ }

+// configGenTick is one tick of configGenLoop over the routes it listed, split

+// out so a test can run the loop's own decision.

+func configGenTick(ctx context.Context, routes []registry.Route, reg *registry.Registry, store release.Store, priv *ecdh.PrivateKey, dd string, svc *activator.Service, writes hostwrite.Config, cache *genCache) {

+ execChanged, unitChanged, readErrs := genPollPass(ctx, routes, store, cache)

+ if len(readErrs) > 0 {

+ // ONE line per pass, never one per app: a bucket having a bad

+ // minute would otherwise flood the journal at 20 lines a minute

+ // per app. The count carries the scale, the first error carries

+ // the cause; they are the same cause in practice.

+ log.Printf("agent: config generation poll: %d apps unreadable: %v", len(readErrs), readErrs[0])

+ }

+ if unitChanged {

+ // Requested first and never fatal: the write is instant while

+ // syncExecSerialized can block on a serialized pass, so the root

+ // run gets to start alongside it rather than behind it. A failed

+ // marker write costs those apps the wait until

+ // carlos-adopt.timer's next hourly pass — never correctness.

+ if err := requestRootSync(dd); err != nil {

+ log.Printf("agent: request root config sync: %v", err)

+ } else {

+ log.Printf("agent: config changed for unit-backed app(s), requested root config sync pass")

}

+ // A nested row an earlier pass could not write for because its instance

+ // directory was not there yet: the first wake has made it, so its

+ // bundle goes in now (finding 6). The pass bounces the instance onto

+ // its settings once the wake it is in has finished (Service.Bounce).

+ if host, ok := configDirsPending.appeared(); ok && !execChanged {

+ log.Printf("agent: %s: its instance directory is there now, writing its config", host)

+ execChanged = true

+ }

+ if !execChanged {

+ return

+ }

+ if err := syncExecSerialized(ctx, reg, store, priv, dd, svc, writes); err != nil {

+ log.Printf("agent: config generation sync: %v", err)

+ }

}

// Fast-converge cadence and clamps (carlos-deploy spec §3-§4). The tick is

@@ -3257,6 +3373,9 @@ func instancesLoop(reg *registry.Registry, writes hostwrite.Config, bs *buildSta

// predicate, already covered by TestShouldSkipPublisherElection, answers

// both.

isFleetBox := shouldSkipPublisherElection(os.Getenv)

+ // An offline move's participants keep their reports current between the

+ // passes below (finding 7); idle, it reads nothing.

+ go offlineReassertLoop(ctx, store)

delay := time.Minute

for {

time.Sleep(delay)

@@ -3288,6 +3407,20 @@ func instancesLoop(reg *registry.Registry, writes hostwrite.Config, bs *buildSta

}

+// offlineReassertLoop runs the armed moves worker's offlineReassert every

+// offlineReassertEvery plus up to 5 s of jitter, for the life of the agent:

+// while this box takes part in an offline move, its source and sealer reports

+// stay current for the doors whatever the serial does. Thin and untested by

+// convention, like instancesLoop; the tick is offlineReassert.

+func offlineReassertLoop(ctx context.Context, s release.Store) {

+ for {

+ time.Sleep(offlineReassertEvery + rand.N(5*time.Second))

+ if w := armedMovesWorker.Load(); w != nil {

+ w.offlineReassert(ctx, s)

+ }

// retentionLoop runs the release-retention pass forever on the slow

// reconcile cadence (5min + jitter, first tick after 1 minute) — never the

// 2s fast pass: pruning is not urgent, and a policy written now is enforced

diff --git a/internal/host/configdirpending_test.go b/internal/host/configdirpending_test.go

new file mode 100644

index 000000000..a1605c1d3

--- /dev/null

+++ b/internal/host/configdirpending_test.go

@@ -0,0 +1,264 @@

+package host

+import (

+ "context"

+ "errors"

+ "os"

+ "path/filepath"

+ "testing"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/hostwrite"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// Harness Run 9, finding 6: a nested instance created with settings has no

+// instance directory until its first wake, so config sync's write of its

+// <host>.config.env failed, and only the hourly pass came back for it. The

+// instance woke and ran without its settings (its store binding included).

+//

+// Through the agent's own wiring: the boot or hourly pass (syncExecSerialized)

+// skips the row and names it, while a sibling whose directory exists gets its

+// file in the same pass; the generation loop's tick (configGenTick) does

+// nothing while the directory is absent, and on the first tick after the

+// first wake has made it, writes the file.

+func TestConfigGenTickWritesANestedRowsConfigOnceItsDirAppears(t *testing.T) {

+ configDirsPending = pendingDirs{}

+ t.Cleanup(func() { configDirsPending = pendingDirs{} })

+ ctx := context.Background()

+ s := release.NewDirStore(t.TempDir())

+ reg := restartTestRegistry(t)

+ dd := t.TempDir()

+ privText, _, err := appconfig.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, err := appconfig.ParsePrivateKey(privText, "test")

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := appconfig.SetEnv(ctx, s, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil {

+ t.Fatal(err)

+ }

+ fresh, woken := "shop.example.com", "older.example.com"

+ for _, h := range []string{fresh, woken} {

+ if err := reg.Upsert(registry.Route{Host: h, Kind: "instance", App: "shop", Account: "bab", Hibernate: true,

+ Status: "ready", DBPath: provision.NestedDBPath(dd, h), Socket: provision.NestedSocket(h)}); err != nil {

+ t.Fatal(err)

+ }

+ // The sibling has woken before: the activator's prepare made its

+ // instance directory (os.MkdirAll(dataDir, 0o700), activator/prepare.go).

+ firstWake := func(host string) {

+ t.Helper()

+ if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil {

+ t.Fatal(err)

+ }

+ firstWake(woken)

+ cfgPath := func(h string) string { return filepath.Join(provision.InstanceDir(dd, h), h+".config.env") }

+ // The boot pass (configSyncLoop's first, a minute after start).

+ if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil {

+ t.Fatalf("a nested row with no directory failed the pass: %v", err)

+ }

+ if b, err := os.ReadFile(cfgPath(woken)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {

+ t.Fatalf("the sibling's bundle = %q, %v", b, err)

+ }

+ if _, err := os.Lstat(provision.InstanceDir(dd, fresh)); !os.IsNotExist(err) {

+ t.Fatalf("config sync made the instance directory itself (err %v): that is the activator's", err)

+ }

+ // The generation loop: the first tick seeds its cache, and no tick

+ // writes while the directory is absent.

+ cache := newGenCache()

+ tick := func() {

+ t.Helper()

+ routes, err := reg.List()

+ if err != nil {

+ t.Fatal(err)

+ }

+ configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache)

+ }

+ for range 2 {

+ tick()

+ }

+ if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) {

+ t.Fatalf("a bundle with nowhere to go: %v", err)

+ }

+ // The first wake makes the directory; the next tick writes the bundle,

+ // with no config change and no hourly pass.

+ firstWake(fresh)

+ tick()

+ if b, err := os.ReadFile(cfgPath(fresh)); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {

+ t.Fatalf("after the first wake, the bundle = %q, %v", b, err)

+ }

+ if _, ok := configDirsPending.appeared(); ok {

+ t.Fatal("the written host is still waited for")

+ }

+ // Nothing more to wait for: a later tick does not run the pass again.

+ if err := os.Remove(cfgPath(fresh)); err != nil {

+ t.Fatal(err)

+ }

+ tick()

+ if _, err := os.Lstat(cfgPath(fresh)); !os.IsNotExist(err) {

+ t.Fatalf("a tick with nothing pending ran the pass (err %v)", err)

+ }

+// A pass that fails part way does not forget a host an earlier pass was

+// waiting for; a complete pass that no longer has the host in scope drops it.

+func TestConfigDirsPendingKeepsWhatAFailedPassDidNotReach(t *testing.T) {

+ var p pendingDirs

+ dir := filepath.Join(t.TempDir(), "instances", "shop.example.com")

+ p.record(map[string]string{"shop.example.com": dir}, nil, true)

+ p.record(map[string]string{}, nil, false)

+ if err := os.MkdirAll(dir, 0o700); err != nil {

+ t.Fatal(err)

+ }

+ if h, ok := p.appeared(); !ok || h != "shop.example.com" {

+ t.Fatalf("appeared = %q %v after a failed pass", h, ok)

+ }

+ p.record(map[string]string{}, map[string]bool{}, true)

+ if h, ok := p.appeared(); ok {

+ t.Fatalf("appeared = %q after a complete pass with the host out of scope", h)

+ }

+// blipStore fails reads of one key while fail is set: a bucket blip on

+// one app's bundle.

+type blipStore struct {

+ release.Store

+ key string

+ fail bool

+func (f *blipStore) Get(ctx context.Context, key string) ([]byte, string, error) {

+ if f.fail && key == f.key {

+ return nil, "", errors.New("transient: connection reset")

+ }

+ return f.Store.Get(ctx, key)

+// Fix round 1, Astra's finding 3: the first tick after the first wake makes

+// the directory meets a transient failure reading the app's bundle. The pass

+// returns nil, leaving the file unwritten; the host stays waited for, and the

+// next tick writes its config, with no config change and no hourly pass. A

+// bundle that keeps failing is retried on a backoff, not every tick.

+func TestConfigGenTickRetriesAHostAPassLeftUndone(t *testing.T) {

+ configDirsPending = pendingDirs{}

+ t.Cleanup(func() { configDirsPending = pendingDirs{} })

+ ctx := context.Background()

+ base := release.NewDirStore(t.TempDir())

+ s := &blipStore{Store: base, key: appconfig.Key("bab", "shop")}

+ reg := restartTestRegistry(t)

+ dd := t.TempDir()

+ privText, _, err := appconfig.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, err := appconfig.ParsePrivateKey(privText, "test")

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := appconfig.SetEnv(ctx, base, "bab", "shop", map[string]string{"STORE_BUCKET": "store-shop"}); err != nil {

+ t.Fatal(err)

+ }

+ host := "shop.example.com"

+ if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "shop", Account: "bab", Hibernate: true,

+ Status: "ready", DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host)}); err != nil {

+ t.Fatal(err)

+ }

+ cfgPath := filepath.Join(provision.InstanceDir(dd, host), host+".config.env")

+ cache := newGenCache()

+ passes := 0

+ tick := func() {

+ t.Helper()

+ routes, err := reg.List()

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, due := configDirsPending.peek(); due {

+ passes++

+ }

+ configGenTick(ctx, routes, reg, s, priv, dd, nil, hostwrite.Config{}, cache)

+ }

+ if err := syncExecSerialized(ctx, reg, s, priv, dd, nil, hostwrite.Config{}); err != nil {

+ t.Fatal(err)

+ }

+ tick() // seeds the cache

+ if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o700); err != nil {

+ t.Fatal(err)

+ }

+ s.fail = true

+ tick()

+ if _, err := os.Lstat(cfgPath); !os.IsNotExist(err) {

+ t.Fatalf("a bundle that did not read was written (err %v)", err)

+ }

+ if _, ok := configDirsPending.peek(); !ok {

+ t.Fatal("the pass that left the host undone dropped it")

+ }

+ s.fail = false

+ tick()

+ if b, err := os.ReadFile(cfgPath); err != nil || string(b) != "STORE_BUCKET=store-shop\n" {

+ t.Fatalf("the next tick's bundle = %q, %v", b, err)

+ }

+ if _, ok := configDirsPending.peek(); ok {

+ t.Fatal("the written host is still waited for")

+ }

+ // A bundle that keeps failing: retries back off (1, 2, 4 ... ticks apart).

+ if err := os.Remove(cfgPath); err != nil {

+ t.Fatal(err)

+ }

+ configDirsPending.record(map[string]string{host: provision.InstanceDir(dd, host)}, nil, false)

+ s.fail = true

+ passes = 0

+ for range 20 {

+ tick()

+ }

+ if passes == 0 || passes > 6 {

+ t.Fatalf("%d retries in 20 ticks of a failing bundle, want a backoff (1 to 6)", passes)

+ }

+// peek is appeared without the tick: whether a retry is due now.

+func (p *pendingDirs) peek() (string, bool) {

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ for h, e := range p.dirs {

+ if _, err := os.Lstat(e.dir); err == nil && e.waitTicks == 0 {

+ return h, true

+ }

+ return "", false

+// TestPendingDirsBackoffSaturates: a host that keeps failing backs off to

+// the cap and stays there; the failure count never grows far enough for the

+// shift to overflow into a negative or zero delay (Astra fix round 1).

+func TestPendingDirsBackoffSaturates(t *testing.T) {

+ var p pendingDirs

+ dir := t.TempDir()

+ p.record(map[string]string{"shop.example.com": dir}, nil, false)

+ for i := 0; i < 200; i++ {

+ p.record(nil, map[string]bool{"shop.example.com": false}, false)

+ e := p.dirs["shop.example.com"]

+ if e == nil {

+ t.Fatalf("pass %d: the waiting host was dropped", i)

+ }

+ if e.waitTicks < 0 || e.waitTicks > pendingRetryCap-1 {

+ t.Fatalf("pass %d: waitTicks %d outside [0,%d]", i, e.waitTicks, pendingRetryCap-1)

+ }

+ if got := p.dirs["shop.example.com"].waitTicks; got != pendingRetryCap-1 {

+ t.Fatalf("after 200 failures waitTicks = %d, want the cap %d", got, pendingRetryCap-1)

+ }

+ p.record(nil, map[string]bool{"shop.example.com": true}, false)

+ if _, ok := p.dirs["shop.example.com"]; ok {

+ t.Fatal("a delivered host stays pending")

+ }

diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go

index cb1019574..d272036c8 100644

--- a/internal/host/movesoffline.go

+++ b/internal/host/movesoffline.go

@@ -139,6 +139,7 @@ type offlineView struct {

type offlineSnapshot struct {

moves map[string]provision.MoveState // host -> its offline move

+ owners map[string][2]string // host -> the account and app its record lives under

failed bool // the listing failed: keep everything

// unreadable are the hosts whose record read but did not decode, or

// disagreed with its path (provision.ListAllRecordsStrict): their move

@@ -157,10 +158,11 @@ func newOfflineView(cfg moveCfg, written func(host string)) *offlineView {

// (fail closed for keep only; the hold never reads the view) and logged at

// most once a minute each.

func (v *offlineView) refresh(recs []provision.Record, unreadable map[string]bool) {

- next := &offlineSnapshot{moves: map[string]provision.MoveState{}, unreadable: map[string]bool{}}

+ next := &offlineSnapshot{moves: map[string]provision.MoveState{}, owners: map[string][2]string{}, unreadable: map[string]bool{}}

for _, rec := range recs {

if rec.Move.Offline() {

next.moves[rec.Host] = *rec.Move

+ next.owners[rec.Host] = [2]string{rec.Account, rec.App}

}

now := time.Now()

@@ -211,6 +213,28 @@ func (v *offlineView) keep(host string) bool {

return false

}

+// elsewhere reports whether host's offline move has its record name a region

+// other than this box's: the target's from landing to finishing, the source's

+// while reverting. Such a host is not this box's to touch whatever its row

+// says (scheduleSkip). A view with no listing yet, and a host with no

+// offline move, answer false: the row's own hold answers for those.

+func (v *offlineView) elsewhere(host string) bool {

+ snap := v.snap.Load()

+ if snap == nil || snap.failed || v.region == "" {

+ return false

+ }

+ m, ok := snap.moves[host]

+ if !ok {

+ return false

+ }

+ region := m.From

+ switch m.State {

+ case provision.OfflineLanding, provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing:

+ region = m.To

+ }

+ return !park.PlacementMatches(v.home, region, v.region)

// refreshOfflineView reads the record list once, outside a moves pass: at

// agent start, before the reconcile loops, next to refreshMovesHolds. A

// failed listing leaves keep failing closed until a listing is read.

@@ -319,6 +343,99 @@ func (w *movesWorker) offlineRound(ctx context.Context, s release.Store, rec pro

return errors.Join(errs...)

}

+// offlineReassertEvery is how often a box that takes part in an offline move

+// rewrites the reports a door judges by age (harness Run 9, finding 7). The

+// doors believe a report for two minutes (offlineReportFresh, internal/

+// console), and the moves pass runs on a serial move or every five to six

+// minutes. So a report written on the pass one door's bump started was stale

+// by the time an operator who waited over two minutes ran the next door, and

+// flip and open refused until the timed pass: most of Run 9's 366 s offline.

+// At this cadence a participant's report is never much more than 20 s old.

+const offlineReassertEvery = 15 * time.Second

+// offlineReassertRoles says which of this box's steps keep its report current

+// while a move is in m's state: a source box's stop from stopping to landed,

+// and the sealer's keep from sealed to landed. Those are the reports seal,

+// flip and open judge by age. The seal itself (sealing) and the target's

+// landing are not repeated here: they are long, and they run on the pass a

+// door's serial bump starts. From open on no door reads a report by age.

+// The sealer only re-reads and reports (offlineSealerRefresh): its ordinary

+// pass re-asserts the tombstone, and the refresh never does.

+func offlineReassertRoles(m *provision.MoveState, source, sealer bool) (asSource, asSealer bool) {

+ switch m.State {

+ case provision.OfflineStopping, provision.OfflineSealing:

+ return source, false

+ case provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:

+ return source, sealer

+ }

+ return false, false

+// offlineReassert is one tick of offlineReassertLoop: for each offline move

+// the view holds in which this box's report is judged by age, it reads the

+// record again and runs this box's source or sealer step for it, in

+// offlineRound's order, under the pass lock. Each step is an ensure that ends

+// by rewriting the box's report (offlineSourceStop, and offlineSealerRefresh,

+// which checks the tombstone but never re-asserts it). With no

+// such move in flight it reads nothing at all. It returns how many records it

+// acted on.

+func (w *movesWorker) offlineReassert(ctx context.Context, s release.Store) int {

+ cfg := w.cfg

+ v := cfg.Offline

+ if !cfg.OfflineCapable || v == nil || s == nil || cfg.Box == "" {

+ return 0

+ }

+ snap := v.snap.Load()

+ if snap == nil || snap.failed || len(snap.moves) == 0 {

+ return 0

+ }

+ here := cfg.parkRegion()

+ roles := func(m *provision.MoveState) (bool, bool) {

+ source := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.From, here)

+ return offlineReassertRoles(m, source, m.Sealer == cfg.Box)

+ }

+ var hosts []string

+ for host, m := range snap.moves {

+ if asSource, asSealer := roles(&m); asSource || asSealer {

+ hosts = append(hosts, host)

+ }

+ if len(hosts) == 0 {

+ return 0

+ }

+ sort.Strings(hosts)

+ w.pass.Lock()

+ defer w.pass.Unlock()

+ acted := 0

+ for _, host := range hosts {

+ owner := snap.owners[host]

+ rec, err := provision.GetRecord(ctx, s, owner[0], owner[1], host)

+ if err != nil {

+ w.logTrouble(host, fmt.Errorf("offline move: read the record to report again: %w", err))

+ continue

+ }

+ if !rec.Move.Offline() {

+ continue

+ }

+ asSource, asSealer := roles(rec.Move)

+ if !asSource && !asSealer {

+ continue

+ }

+ acted++

+ var errs []error

+ if asSource {

+ errs = append(errs, w.offlineSourceStep(ctx, s, *rec))

+ }

+ if asSealer {

+ errs = append(errs, w.offlineSealerRefresh(ctx, s, *rec))

+ }

+ if err := errors.Join(errs...); err != nil {

+ w.logTrouble(host, err)

+ }

+ return acted

// now is cfg.Now, or the wall clock when a test left it unset.

func (c moveCfg) now() time.Time {

if c.Now != nil {

diff --git a/internal/host/movesoffline_e2e_test.go b/internal/host/movesoffline_e2e_test.go

index 32f356db6..ee87816cb 100644

--- a/internal/host/movesoffline_e2e_test.go

+++ b/internal/host/movesoffline_e2e_test.go

@@ -8,6 +8,7 @@ import (

"encoding/base64"

"encoding/json"

"errors"

+ "fmt"

"io"

"net/http"

"net/http/httptest"

@@ -34,6 +35,7 @@ import (

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/schedule"

"github.com/carlosframework/platform/internal/status"

)

@@ -407,14 +409,24 @@ func (w *e2eWorld) publish(b *offBox) {

// report freshness by.

func (w *e2eWorld) tick(b *offBox) {

w.t.Helper()

- w.wd.mu.Lock()

- w.wd.now = time.Now()

- w.wd.mu.Unlock()

+ w.tickAt(b, time.Now())

+// tickAt is tick with the boxes' clock reading at.

+func (w *e2eWorld) tickAt(b *offBox, at time.Time) {

+ w.t.Helper()

+ w.setBoxClock(at)

b.reconcile()

w.publish(b)

b.pass()

}

+func (w *e2eWorld) setBoxClock(at time.Time) {

+ w.wd.mu.Lock()

+ w.wd.now = at

+ w.wd.mu.Unlock()

func (w *e2eWorld) tickAll() {

w.t.Helper()

w.tick(w.ie)

@@ -663,8 +675,15 @@ func (w *e2eWorld) begin() {

t.Fatal("ie-box still lists the host after its stop")

}

- // Config sync's render at rest, standing in: the source's config.env

- // from the bundle (store binding and secret).

+ w.renderSourceConfig()

+// renderSourceConfig stands in for config sync's render at rest: the

+// source's config.env from the bundle (store binding and secret), and the

+// source's bytes the move is judged against.

+func (w *e2eWorld) renderSourceConfig() {

+ t := w.t

+ t.Helper()

hr, err := w.ie.w.cfg.RenderHeld(context.Background(), w.wd.s, *w.ie.row(offHost), w.id, nil, func(string) error { return nil })

if err != nil {

t.Fatal(err)

@@ -1048,6 +1067,168 @@ func TestOfflineMoveE2E(t *testing.T) {

t.Run("nested source with launch files", func(t *testing.T) { offlineMoveDone(t, landOpts{nested: true}) })

t.Run("no files directory", func(t *testing.T) { offlineMoveDone(t, landOpts{noFiles: true}) })

t.Run("rollback after a tampered archive", offlineMoveRollback)

+ // Harness Run 9, finding 7: the doors driven with nothing between them

+ // but what production runs (each door's serial bump, the boxes' own

+ // nudges and the offline re-assert loop), first back to back, then with

+ // the operator three minutes behind at every step, past the doors' two

+ // minutes of belief in a report.

+ t.Run("doors back to back", func(t *testing.T) { offlineMoveDoorsApart(t, 0) })

+ t.Run("doors three minutes apart", func(t *testing.T) { offlineMoveDoorsApart(t, 3*time.Minute) })

+// e2eLoops are what acts on the two boxes between door calls in production,

+// and nothing else: each box's fast-converge tick (fastTick, the real serial

+// rule: a moved serial runs the pass set once per clamp, and a nudge made

+// during the pass runs it again), whose pass set is the agent's order

+// (reconcile, the edges file, the moves pass); and each box's offline

+// re-assert tick (offlineReassert). Nothing else ticks a box.

+type e2eLoops struct {

+ w *e2eWorld

+ fast map[*offBox]*fastDeps

+ tickAt time.Time // fastTick's own clock, stepped past the clamp each round

+ boxAt time.Time // what the boxes' clock reads during a serial pass

+ passes int // serial passes run

+ again int // re-assert ticks run

+func (w *e2eWorld) loops() *e2eLoops {

+ l := &e2eLoops{w: w, fast: map[*offBox]*fastDeps{}, tickAt: time.Now()}

+ for _, b := range []*offBox{w.ie, w.uk} {

+ // The sweep's registry is empty: the sweep stages releases and is

+ // not what this test is about. The serial leg is.

+ l.fast[b] = &fastDeps{reg: restartTestRegistry(w.t), store: w.rs, fullPass: func(context.Context) {

+ l.passes++

+ w.tickAt(b, l.boxAt)

+ }}

+ }

+ l.round() // the boot read seeds each box's serial

+ return l

+// round is one fastTick on each box; it reports whether either ran its pass.

+func (l *e2eLoops) round() bool {

+ before := l.passes

+ l.tickAt = l.tickAt.Add(fastFullClamp + time.Second)

+ for _, b := range []*offBox{l.w.ie, l.w.uk} {

+ fastTick(context.Background(), l.fast[b], l.tickAt)

+ }

+ return l.passes > before

+// after is what the boxes do once a door has bumped the serial, when the

+// operator runs the next door gap later: the serial passes at the time of

+// the bump, then the re-assert loop's ticks through the gap, up to now.

+func (l *e2eLoops) after(gap time.Duration) {

+ t := l.w.t

+ t.Helper()

+ l.boxAt = time.Now().Add(-gap)

+ rounds := 0

+ for l.round() {

+ if rounds++; rounds > 4 {

+ t.Fatal("the serial kept moving after four rounds of passes")

+ }

+ if gap > 0 {

+ // Without the re-assert, a door would read the reports as the bump's

+ // passes left them: older than it believes. The status view says

+ // so, and never as a process (MV-O31).

+ if v := l.w.status(); v.Next != "" || !strings.Contains(v.Waiting, "is not current") {

+ t.Fatalf("%s behind the bump, the view read %+v; want a report that is not current", gap, v)

+ }

+ for at := l.boxAt.Add(offlineReassertEvery); !at.After(time.Now()); at = at.Add(offlineReassertEvery) {

+ l.reassert(at)

+ }

+func (l *e2eLoops) reassert(at time.Time) {

+ l.w.setBoxClock(at)

+ for _, b := range []*offBox{l.w.ie, l.w.uk} {

+ b.w.offlineReassert(context.Background(), l.w.rs)

+ }

+ l.again++

+// offlineMoveDoorsApart drives start to open with the operator gap behind

+// each door's bump, and wants every door accepted on its first or second try,

+// the second after one more re-assert tick. It logs the time each door took

+// to its answer and from start to open.

+func offlineMoveDoorsApart(t *testing.T, gap time.Duration) {

+ w := newE2EWorld(t, landOpts{})

+ l := w.loops()

+ tries := map[string]int{}

+ door := func(verb, body, want string) e2eView {

+ t.Helper()

+ for try := 1; ; try++ {

+ code, out := w.door(verb, body)

+ if code == http.StatusAccepted {

+ tries[verbName(verb)] = try

+ v := w.decode(out)

+ if v.State != want {

+ t.Fatalf("%s: state %q, want %q", verbName(verb), v.State, want)

+ }

+ return v

+ }

+ if try == 2 {

+ t.Fatalf("%s refused twice, %s behind the previous step: %d %s", verbName(verb), gap, code, out)

+ }

+ l.reassert(time.Now())

+ }

+ next := func(step, verb string) {

+ t.Helper()

+ if v := w.status(); v.Next != verb {

+ t.Fatalf("after %s, %s behind: next %q, want %q (waiting %q)", step, gap, v.Next, verb, v.Waiting)

+ }

+ timers, err := w.ie.reg.TimersFor(offHost)

+ if err != nil {

+ t.Fatal(err)

+ }

+ w.srcTimers = map[string]registry.Timer{}

+ for _, tm := range timers {

+ w.srcTimers[tm.Name] = tm

+ }

+ began := time.Now()

+ var took []string

+ timed := func(name string, f func()) {

+ at := time.Now()

+ f()

+ took = append(took, fmt.Sprintf("%s %s", name, time.Since(at).Round(time.Millisecond)))

+ }

+ timed("start", func() {

+ w.id = door("", `{"region":"uk"}`, provision.OfflineStopping).ID

+ l.after(gap)

+ })

+ next("start", "seal")

+ w.renderSourceConfig()

+ timed("seal", func() {

+ door("seal", "", provision.OfflineSealing)

+ l.after(gap)

+ })

+ if st := w.wd.record().Move.State; st != provision.OfflineSealed {

+ t.Fatalf("state %s after the seal's passes", st)

+ }

+ next("seal", "flip")

+ timed("flip", func() {

+ door("flip", "", provision.OfflineLanding)

+ l.after(gap)

+ })

+ if st := w.wd.record().Move.State; st != provision.OfflineLanded {

+ t.Fatalf("state %s after the flip's passes", st)

+ }

+ next("flip", "open")

+ timed("open", func() {

+ door("open", "", provision.OfflineOpen)

+ l.after(0)

+ })

+ next("open", "finish")

+ total := time.Since(began).Round(time.Millisecond)

+ t.Logf("operator gap %s: start to open in %s of test time (%s); %d serial passes, %d re-assert ticks; tries %v",

+ gap, total, strings.Join(took, ", "), l.passes, l.again, tries)

+ // The move is the real one: the target holds what the source had.

+ w.requireTarget()

}

func offlineMoveDone(t *testing.T, opts landOpts) {

@@ -1090,6 +1271,7 @@ func offlineMoveDone(t *testing.T, opts landOpts) {

t.Fatal("ie-box's row went before finishing")

}

w.requireTombstoned("after open")

+ w.requireSourceSchedulerQuiet()

// The writes so far, before the first wake (which is not the tool's).

w.requireClosedWriteSet("through open")

w.firstWake()

@@ -1130,6 +1312,67 @@ func offlineMoveDone(t *testing.T, opts landOpts) {

held.Release(ctx)

}

+// requireSourceSchedulerQuiet is harness Run 9's defect 8. After open the

+// source keeps its row, held, until finishing, its timers with it, and open

+// has lifted the fence. The source's own schedule runner, built as the agent

+// builds it (newScheduleDeps over ie-box's registry, the deployment store,

+// its activator and data directory), ticks with every timer due: it writes

+// nothing for the host. No env file at either layout's path (the tick used

+// to mint one, a fresh admin token, in the parked instance directory), no

+// timer row moves, and no store write.

+func (w *e2eWorld) requireSourceSchedulerQuiet() {

+ t := w.t

+ t.Helper()

+ ie := w.ie

+ if ie.svc.Fenced(offHost) {

+ t.Fatal("fixture: ie-box is still fenced after open")

+ }

+ // Run 9's app declared a one-minute schedule; the row is the source's

+ // as the converger writes it, due now. (The fixture's own timers are

+ // the seal's, kept for the digest, and are not all ones the runner can

+ // fire.)

+ if err := ie.reg.UpsertTimer(registry.Timer{Host: offHost, Name: "every-minute", Kind: schedule.KindSchedule, Spec: "every 1m0s", Path: "/tick", NextAt: 1}); err != nil {

+ t.Fatal(err)

+ }

+ before, err := ie.reg.TimersFor(offHost)

+ if err != nil || len(before) == 0 {

+ t.Fatalf("fixture: ie-box keeps no timers for the host after open (%v)", err)

+ }

+ mark := w.rs.mark()

+ sd := newScheduleDeps(ie.reg, w.rs, ie.svc, nil, ie.dd)

+ sd.runner.Tick(context.Background())

+ // A delivery the tick started holds the host's claim until it ends.

+ for deadline := time.Now().Add(10 * time.Second); sd.runner.Busy(offHost); {

+ if time.Now().After(deadline) {

+ t.Fatal("the source's tick did not finish")

+ }

+ time.Sleep(5 * time.Millisecond)

+ }

+ for _, p := range []string{

+ filepath.Join(ie.dd, offHost+".env"),

+ filepath.Join(provision.InstanceDir(ie.dd, offHost), offHost+".env"),

+ filepath.Join(layout.RawInstanceDir(ie.dd, offHost), offHost+".env"),

+ } {

+ if lexists(p) {

+ t.Fatalf("the source's scheduler wrote %s after open", p)

+ }

+ after, err := ie.reg.TimersFor(offHost)

+ // The runner's daily sweep of finished once-rows is box-wide and is not

+ // a tick of the host (the timers digest leaves that history out, joint

+ // N14): only the live rows are compared.

+ live := func(ts []registry.Timer) []registry.Timer {

+ return slices.DeleteFunc(slices.Clone(ts), func(tm registry.Timer) bool { return tm.Kind == "once" && tm.NextAt == 0 })

+ }

+ before, after = live(before), live(after)

+ if err != nil || !reflect.DeepEqual(after, before) {

+ t.Fatalf("the source's tick moved the host's timers (%v):\nbefore %+v\nafter %+v", err, before, after)

+ }

+ if ops := w.rs.since(mark); len(ops) != 0 {

+ t.Fatalf("the source's tick wrote to the store: %v", ops)

+ }

// requireRetiredAfterFinish: finishing removed the row and nothing parked

// came back or went.

func (w *e2eWorld) requireRetiredAfterFinish(at moves.OfflineReport) {

diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go

index cbfb60cde..a0056e640 100644

--- a/internal/host/movesoffline_seal.go

+++ b/internal/host/movesoffline_seal.go

@@ -967,6 +967,57 @@ func (w *movesWorker) offlineKeepStop(ctx context.Context, s release.Store, rec

return writeOfflineReport(ctx, s, living)

}

+// offlineSealerRefresh is the sealer's half of the report refresh

+// (offlineReassert), from offline-sealed to offline-landed: offlineKeepStop's

+// checks without its write to the lease. It reads the seal and the lease, a

+// tombstone that is not this move's live one is the lost stop it always is,

+// it checks the source positions against the seal, and it rewrites the

+// living report with the tombstone's owner and expiry exactly as read. It

+// never re-asserts the tombstone (fix round 1, Astra's finding 2: a refresh

+// every 15 s through offlineKeepStop pushed the expiry out seven days each

+// time); the sealer's ordinary pass keeps doing that, once a pass.

+func (w *movesWorker) offlineSealerRefresh(ctx context.Context, s release.Store, rec provision.Record) error {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ if !m.Offline() || cfg.Box == "" || m.Sealer != cfg.Box {

+ return nil

+ }

+ switch m.State {

+ case provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:

+ default:

+ return nil

+ }

+ raw, err := w.offlineSealRecord(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ seal, err := decodeSeal(raw, host, m.ID)

+ if err != nil {

+ return fmt.Errorf("offline move %s: %s: %w; it is never rewritten, so abort the move", m.ID, host, err)

+ }

+ st, err := w.offlineLock(ctx, host)

+ if err != nil {

+ return err

+ }

+ if !st.mine(m.ID) {

+ return w.offlineLeaseLost(ctx, s, rec, seal, st)

+ }

+ if seal != nil {

+ if err := w.offlineCheckPositions(ctx, s, rec, seal); err != nil {

+ return err

+ }

+ living := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer}

+ if seal != nil {

+ living = *seal

+ }

+ living.Phase = moves.OfflinePhaseSealed

+ living.At = cfg.now().UTC()

+ living.TombstoneOwner = st.owner

+ living.TombstoneUntil = st.expires.UTC()

+ return writeOfflineReport(ctx, s, living)

// offlineCheckPositions is the backstop behind the holds and the fence:

// every source report written since the seal (at or after its At) must be

// at or below the sealed final position. One past it is a box that wrote

diff --git a/internal/host/movesoffline_seal_test.go b/internal/host/movesoffline_seal_test.go

index 02158838d..15e7d04ba 100644

--- a/internal/host/movesoffline_seal_test.go

+++ b/internal/host/movesoffline_seal_test.go

@@ -1403,3 +1403,75 @@ func TestOfflineSealLostStopOnASourcePastTheSealedPosition(t *testing.T) {

t.Fatalf("the tombstone is %s", owner)

}

+// Fix round 1, Astra's finding 2: the report refresh (offlineReassert) is

+// report-only for the sealer. Repeated refreshes rewrite the sealer's report

+// with the tombstone's expiry as read and leave the lock object byte for

+// byte; an expired, missing or replaced tombstone makes the refresh report

+// the lost stop, never a fresh report, and the lock is not written.

+func TestOfflineSealerRefreshNeverRenewsTheTombstone(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie) // seals, takes the tombstone, records sealed

+ if st := sw.record().Move.State; st != provision.OfflineSealed {

+ t.Fatalf("fixture: state %s", st)

+ }

+ sw.mustPass(ie) // the view learns sealed, as every pass refreshes it

+ ctx := context.Background()

+ key := sealPrefix + "/" + offHost + "/lock.json"

+ ours, ok := sw.fake.Current(sealIE, key)

+ if !ok {

+ t.Fatal("fixture: no tombstone")

+ }

+ _, expires, _ := sw.lock()

+ for i := range 3 {

+ time.Sleep(20 * time.Millisecond) // a renewal would write a later expiry

+ sw.advance(15 * time.Second)

+ if n := ie.w.offlineReassert(ctx, sw.s); n != 1 {

+ t.Fatalf("refresh %d acted on %d records", i, n)

+ }

+ if now, _ := sw.fake.Current(sealIE, key); !bytes.Equal(now, ours) {

+ t.Fatalf("refresh %d rewrote the tombstone:\nbefore %s\nafter %s", i, ours, now)

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseSealed || !r.At.Equal(sw.clock()) || !r.TombstoneUntil.Equal(expires.UTC()) || r.TombstoneOwner != moves.TombstoneOwner(offID) {

+ t.Fatalf("refresh %d: report %+v, want sealed at %v with the expiry %v as read", i, r, sw.clock(), expires)

+ }

+ lockBody := func(owner string, exp time.Time) []byte {

+ b, err := json.Marshal(litestream.Lease{Generation: 1, Owner: owner, ExpiresAt: exp})

+ if err != nil {

+ t.Fatal(err)

+ }

+ return b

+ }

+ for _, c := range []struct {

+ name string

+ damage func()

+ }{

+ {"expired", func() { sw.fake.Put(sealIE, key, lockBody(moves.TombstoneOwner(offID), time.Now().Add(-time.Second))) }},

+ {"missing", func() { sw.fake.BareDelete(sealIE, key) }},

+ {"replaced", func() { sw.fake.Put(sealIE, key, lockBody(moves.TombstoneOwner(offID2), time.Now().Add(time.Hour))) }},

+ } {

+ sw.fake.Put(sealIE, key, ours)

+ sw.advance(15 * time.Second)

+ ie.w.offlineReassert(ctx, sw.s)

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseSealed {

+ t.Fatalf("%s: before the damage the report is %s", c.name, r.Phase)

+ }

+ c.damage()

+ before := sw.fake.Snapshot()

+ sw.advance(15 * time.Second)

+ ie.w.offlineReassert(ctx, sw.s)

+ if sw.fake.Snapshot() != before {

+ t.Fatalf("%s: the refresh wrote the lease", c.name)

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["lease"].OK || !strings.Contains(r.Items["lease"].Reason, "lost stop") || r.TombstoneOwner != "" {

+ t.Fatalf("%s: report %+v, want the lost stop", c.name, r)

+ }

diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go

index b1b8da306..b44a184d9 100644

--- a/internal/host/movesoffline_test.go

+++ b/internal/host/movesoffline_test.go

@@ -1281,3 +1281,112 @@ func TestOfflineKeep(t *testing.T) {

}

+// readCountingStore counts the reads made through it.

+type readCountingStore struct {

+ release.Store

+ mu sync.Mutex

+ reads int

+func (c *readCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) {

+ c.mu.Lock()

+ c.reads++

+ c.mu.Unlock()

+ return c.Store.Get(ctx, key)

+func (c *readCountingStore) List(ctx context.Context, prefix string) ([]string, error) {

+ c.mu.Lock()

+ c.reads++

+ c.mu.Unlock()

+ return c.Store.List(ctx, prefix)

+// Harness Run 9, finding 7: between moves passes, a source box rewrites its

+// report on the re-assert tick while a door judges that report by age

+// (stopping to landed), and only then. It never runs the seal itself, it

+// reads nothing while no offline move is in flight, and a box that is

+// neither a source nor the sealer does nothing.

+func TestOfflineReassertRewritesOnlyTheReportsADoorReads(t *testing.T) {

+ wd := newOffWorld(t)

+ a := wd.box("ie-a", "", nil)

+ uk := wd.box("uk-box", "uk", nil)

+ a.tick()

+ uk.tick()

+ cs := &readCountingStore{Store: wd.s}

+ ctx := context.Background()

+ if n := a.w.offlineReassert(ctx, cs); n != 0 || cs.reads != 0 {

+ t.Fatalf("no move in flight: acted on %d, %d store reads", n, cs.reads)

+ }

+ wd.start()

+ a.tick() // the start door's bump

+ uk.tick()

+ wd.advance(3 * time.Minute)

+ for _, state := range []string{provision.OfflineStopping, provision.OfflineSealing} {

+ if state == provision.OfflineSealing {

+ wd.seal()

+ wd.advance(time.Minute)

+ }

+ if n := a.w.offlineReassert(ctx, cs); n != 1 {

+ t.Fatalf("%s: the source acted on %d records, want 1", state, n)

+ }

+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.At.Equal(wd.clock()) || !r.Held || !r.Fenced {

+ t.Fatalf("%s: the source's report %+v, want rewritten at %v", state, r, wd.clock())

+ }

+ if n := uk.w.offlineReassert(ctx, cs); n != 0 {

+ t.Fatalf("%s: the target acted on %d records", state, n)

+ }

+ if _, _, err := wd.s.Get(ctx, moves.OfflineSealKey(offHost, offID)); !errors.Is(err, os.ErrNotExist) {

+ t.Fatalf("the re-assert ran the seal (err %v)", err)

+ }

+ // From open on, no door reads a report by age: nothing is rewritten.

+ wd.sealed()

+ wd.flip()

+ wd.landed()

+ wd.open()

+ a.pass() // the view learns the state the way it always does

+ at := wd.report(moves.OfflineRoleSource, "ie-a").At

+ wd.advance(time.Minute)

+ if n := a.w.offlineReassert(ctx, cs); n != 0 {

+ t.Fatalf("open: the source acted on %d records", n)

+ }

+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.At.Equal(at) {

+ t.Fatalf("open: the source's report was rewritten at %v", r.At)

+ }

+// elsewhere, scheduleSkip's second half (harness Run 9, defect 8): the box

+// whose region the record does not name, through every state the record

+// moves through.

+func TestOfflineElsewhereFollowsTheRecordsRegion(t *testing.T) {

+ wd := newOffWorld(t)

+ a := wd.box("ie-a", "", nil)

+ uk := wd.box("uk-box", "uk", nil)

+ a.tick()

+ uk.tick()

+ check := func(when string, ieElsewhere, ukElsewhere bool) {

+ t.Helper()

+ a.pass()

+ uk.pass()

+ if got := a.w.cfg.Offline.elsewhere(offHost); got != ieElsewhere {

+ t.Fatalf("%s: ie-a elsewhere = %v", when, got)

+ }

+ if got := uk.w.cfg.Offline.elsewhere(offHost); got != ukElsewhere {

+ t.Fatalf("%s: uk-box elsewhere = %v", when, got)

+ }

+ check("no move", false, false)

+ wd.start()

+ check("stopping", false, true)

+ wd.seal()

+ wd.sealed()

+ check("sealed", false, true)

+ wd.flip()

+ check("landing", true, false)

+ wd.abort()

+ check("reverting", false, true)

diff --git a/internal/host/schedule_wiring.go b/internal/host/schedule_wiring.go

index b89e0dcf7..8bcd4dcd3 100644

--- a/internal/host/schedule_wiring.go

+++ b/internal/host/schedule_wiring.go

@@ -87,6 +87,7 @@ func newScheduleDeps(reg *registry.Registry, store release.Store, svc *activator

Reg: reg,

Deliver: &activatorDeliverer{svc: svc},

Token: scheduleToken(dd, svc, reg),

+ Skip: scheduleSkip,

Event: event,

Box: box,

Build: version.Build,

@@ -145,6 +146,29 @@ func armSchedule(reg *registry.Registry, store release.Store, storeErr error, sv

}

+// scheduleSkip is the runner's Skip (harness Run 9, defect 8): a host whose

+// row this box holds, and a host whose offline move's record names another

+// region. After open the source keeps its row, held, until finishing, and

+// open has lifted the fence that used to refuse the token; so the source's

+// tick minted a fresh <host>.env (an admin token) inside the instance

+// directory the move had just parked, a stray secret for a host served

+// elsewhere, and recorded a delivery the activator refuses for a held row

+// anyway (ServeScheduled). The token is the only instance file the runner

+// writes; delivery wakes nothing for a held row, and the converger writes

+// registry rows and the bucket status only. Any hold counts, not only an

+// offline move's: no held row is ever served a tick, and a hold whose move

+// has just ended is still a hold until reconcile removes or releases the

+// row.

+func scheduleSkip(rt registry.Route) bool {

+ if registry.IsHold(rt.Status) {

+ return true

+ }

+ if v := armedOfflineView(); v != nil {

+ return v.elsewhere(rt.Host)

+ }

+ return false

// scheduleToken is the runner's Token: ensure-then-read on the activator's

// own <dataDir>/<host>.env (spec §6.4).

diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go

index 6357fcd50..51b1f53fa 100644

--- a/internal/provision/plan14_root_reads_test.go

+++ b/internal/provision/plan14_root_reads_test.go

@@ -995,6 +995,22 @@ func TestPlan14RootReadCensus(t *testing.T) {

if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-install-console-upgrade.tsv"), &inv, true); err != nil {

t.Fatal(err)

}

+ // Census overlay moves-offline-fixes (harness Run 9's defects 6 to 8 in

+ // the offline operator move), loaded last, under

+ // census-ruling-moves-offline-fixes (platform-1c, 2026-10-20); the

+ // proposal names each unit's executing identity

+ // (docs/superpowers/notes/census-proposal-moves-offline-fixes.md). Every

+ // unit runs in the agent and makes no root decision: changed legacy

+ // units re-frozen at main's disposition, new ones proposed accepted

+ // legacy at their siblings'. Frozen declarations are added, so the seal

+ // moves; main's is checked first.

+ if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 {

+ t.Fatal(errs)

+ }

+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv"), &inv, true); err != nil {

+ t.Fatal(err)

+ }

+ seal = "826c54be9447f0a7c42386dba6be5888fc5a708bf1e5a57f2a6de6da4662a62f"

errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)

errs = append(errs, checkRootReadDeclarations(got, inv)...)

if len(errs) > 0 {

diff --git a/internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv b/internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv

new file mode 100644

index 000000000..0ab47efc1

--- /dev/null

+++ b/internal/provision/testdata/plan14-root-reads-moves-offline-fixes.tsv

@@ -0,0 +1,31 @@

+# Census overlay moves-offline-fixes (harness Run 9's defects 6 to 8 in the offline operator move, on main after 54ae7fb9), loaded LAST, after armed-box-updates. RULED as census-ruling-moves-offline-fixes (platform-1c, 2026-10-20, accepted as proposed, re-ruled for fix round 1); was PROPOSED-moves-offline-fixes: every row is rev3=census-ruling-moves-offline-fixes, and the ruling is asked of the census owner (platform-1c). The proposal, with each unit's executing identity: docs/superpowers/notes/census-proposal-moves-offline-fixes.md.

+# Every unit here runs in the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos) and makes no root decision, so each is proposed accepted legacy at main's disposition (J1 root-route for registry and route reads, J4 root-input for file reads), as census-ruling-moves-offline proposed its units.

+# Defect 6 (config.env for a nested row whose instance directory is not made yet): SyncExecWithCallbacks, renderExecConfigBefore and configGenLoop re-frozen at main's disposition; configGenTick and pendingDirs.appeared new. The seal moves.

+# Defect 7 (fresh reports at every door step): envelope only. The source and sealer re-assert (movesWorker.offlineReassert, offlineReassertLoop) reads no root input of its own: it reads the record from the deployment store and calls the steps census-ruling-moves-offline already classified. The console's stale-report refusal reads nothing new. internal/console and internal/host re-pinned; the seal stands.

+# Defect 8 (the source's scheduler wrote the moved host's env file after open): Runner.Tick and newScheduleDeps re-frozen at their dispositions, scheduleSkip new (J1), Runner.fire re-pinned at its class and task (hint, E2: overlay e's reclassification); internal/host and internal/schedule re-pinned. The seal moves.

+# Fix round 1, finding 1 (Astra: skipped timers could starve other hosts): Runner.Tick re-pinned at its re-frozen disposition (its due query moved into dueRows); Runner.dueRows new, accepted legacy (J1, as Runner.Tick): held hosts excluded in the query and hosts Skip names paged past before the limit; internal/schedule re-pinned. The seal moves.

+# Fix round 1, finding 2 (Astra: the report refresh extended the tombstone): envelope only. The refresh now calls movesWorker.offlineSealerRefresh, which reads the seal, the lease (offlineLock) and the source reports and writes only the sealer's report, never the lease; it has no root read of its own. internal/host re-pinned; the seal stands.

+# Fix round 1, finding 3 (Astra: a skipped config read cleared the pending retry): SyncExecWithCallbacks re-pinned (it names each in-scope host done or not, WithHostOutcome), pendingDirs.appeared re-pinned (the same Lstat, now with a retry backoff); internal/appconfig and internal/host re-pinned. The seal moves.

+@coverage internal/appconfig d2258ea641530870dc3b7600cda01896b0d647229a3b70b83addcd30b15210aa

+@coverage internal/console 7b015747c5ef8a3ee7473ad63f7f99200ce14c63c1b7a47945c212ad9239756b

+@coverage internal/host cdfe9b5c19793ca66ddfca7ea9cacf19166511a8927150e9b4f278a758c01473

+@coverage internal/schedule a76efa679b9089eff4f6dae8e6f0adc5d92de628e66e16d4a00c81ba70e086ff

+@legacy 378070928b39fce4bbb645a70cccc08421a0a8c025e73a90b3590e884286afb4 internal/appconfig/syncexec.go SyncExecWithCallbacks legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=route-and-input (J1,J4, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), through syncExecSerialized: main's body, with a nested row whose write found no instance directory named and skipped (errInstanceDirAbsent, WhenInstanceDirAbsent) instead of failing the pass, and each in-scope host reported done or not done (WithHostOutcome) once the pass has gone through every route; no new read site file:Stat@28417,registry:App@10156,registry:App@10371,registry:App@15917,registry:App@28302,registry:Environment@10164,registry:Environment@10379,registry:Environment@15925,registry:Environment@28310,registry:Host@1031,registry:Host@1460,registry:Host@15425,registry:Host@15752,registry:Host@15850,registry:Host@16682,registry:Host@17033,registry:Host@17968,registry:Host@18509,registry:Host@21436,registry:Host@21653,registry:Host@22211,registry:Host@22401,registry:Host@22410,registry:Host@22429,registry:Host@25330,registry:Host@25947,registry:Host@25999,registry:Host@26155,registry:Host@28811,registry:Host@831,registry:Host@902,registry:Host@996,whole:Route@10049,whole:Route@10110,whole:Route@10151,whole:Route@10268,whole:Route@10302,whole:Route@10366,whole:Route@1412,whole:Route@1441,whole:Route@15337,whole:Route@15387,whole:Route@15912,whole:Route@24926,whole:Route@25340,whole:Route@26044,whole:Route@28201,whole:Route@28234,whole:Route@28297,whole:Route@28398,whole:Route@29048,whole:Route@29121,whole:Route@693,whole:Route@721,whole:Route@750,whole:Route@776,whole:Route@9984 b523044621e10b4a702d5e1b8c5ecd843b645cf72b327487ed424f96956a3a7e

+internal/appconfig/syncexec.go SyncExecWithCallbacks legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=route-and-input (J1,J4, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), through syncExecSerialized: main's body, with a nested row whose write found no instance directory named and skipped (errInstanceDirAbsent, WhenInstanceDirAbsent) instead of failing the pass, and each in-scope host reported done or not done (WithHostOutcome) once the pass has gone through every route; no new read site file:Stat@28417,registry:App@10156,registry:App@10371,registry:App@15917,registry:App@28302,registry:Environment@10164,registry:Environment@10379,registry:Environment@15925,registry:Environment@28310,registry:Host@1031,registry:Host@1460,registry:Host@15425,registry:Host@15752,registry:Host@15850,registry:Host@16682,registry:Host@17033,registry:Host@17968,registry:Host@18509,registry:Host@21436,registry:Host@21653,registry:Host@22211,registry:Host@22401,registry:Host@22410,registry:Host@22429,registry:Host@25330,registry:Host@25947,registry:Host@25999,registry:Host@26155,registry:Host@28811,registry:Host@831,registry:Host@902,registry:Host@996,whole:Route@10049,whole:Route@10110,whole:Route@10151,whole:Route@10268,whole:Route@10302,whole:Route@10366,whole:Route@1412,whole:Route@1441,whole:Route@15337,whole:Route@15387,whole:Route@15912,whole:Route@24926,whole:Route@25340,whole:Route@26044,whole:Route@28201,whole:Route@28234,whole:Route@28297,whole:Route@28398,whole:Route@29048,whole:Route@29121,whole:Route@693,whole:Route@721,whole:Route@750,whole:Route@776,whole:Route@9984 b523044621e10b4a702d5e1b8c5ecd843b645cf72b327487ed424f96956a3a7e

+@legacy 062b7343a8fa6be7988022f17181fd4e7e17450b99c3b8c6f9b8ea764b14a22b internal/appconfig/syncexec.go renderExecConfigBefore legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=route-and-input (J1,J4, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from config sync and the moves worker's held render: main's body with one Lstat of a nested row's instance directory after its write failed, to tell a directory not made yet from any other failure; it follows no link and changes no outcome but the error's kind file:Lstat@3893,file:ReadFile@951,registry:Environment@3190,registry:Host@1070,registry:Host@2586,registry:Host@4088,registry:Host@4320,registry:Host@4576,registry:Host@4669,registry:Host@876,whole:Route@2522,whole:Route@257,whole:Route@3715,whole:Route@680 a60a9a2e2158f3a8dadf6fcf46810930dad89b67d1acc23fb57c324a7d9ac1f1

+internal/appconfig/syncexec.go renderExecConfigBefore legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=route-and-input (J1,J4, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from config sync and the moves worker's held render: main's body with one Lstat of a nested row's instance directory after its write failed, to tell a directory not made yet from any other failure; it follows no link and changes no outcome but the error's kind file:Lstat@3893,file:ReadFile@951,registry:Environment@3190,registry:Host@1070,registry:Host@2586,registry:Host@4088,registry:Host@4320,registry:Host@4576,registry:Host@4669,registry:Host@876,whole:Route@2522,whole:Route@257,whole:Route@3715,whole:Route@680 a60a9a2e2158f3a8dadf6fcf46810930dad89b67d1acc23fb57c324a7d9ac1f1

+@legacy 4f285d36e7c92102da6a29594cd2bf0de1cae4c88486cb09b59d502fdc670283 internal/host/agent.go configGenLoop legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): main's loop, its tick's body moved into configGenTick; the registry listing stays here whole:Route@1265 a0b36186f7c8abdd818c81c2651a75b202078ad7f197d93b6157bcab97604188

+internal/host/agent.go configGenLoop legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): main's loop, its tick's body moved into configGenTick; the registry listing stays here whole:Route@1265 a0b36186f7c8abdd818c81c2651a75b202078ad7f197d93b6157bcab97604188

+@legacy c9727345192047c534cb2c08fe70d75ed2787e0b37b262aa364dd66a11a1d121 internal/host/agent.go configGenTick legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as configGenLoop); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): new, configGenLoop's tick over the routes it listed; it also runs the agent's own config pass once an instance directory the last pass waited for exists whole:Route@268 751870d636c3ea7b3670a7e00c71d085e94ae9b013545e319047a4609f514943

+internal/host/agent.go configGenTick legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as configGenLoop); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): new, configGenLoop's tick over the routes it listed; it also runs the agent's own config pass once an instance directory the last pass waited for exists whole:Route@268 751870d636c3ea7b3670a7e00c71d085e94ae9b013545e319047a4609f514943

+@legacy c9582505e641f881ffcfb86213bf2ade68d7118eb11a521e0a45706cd6f3cf65 internal/host/agent.go pendingDirs.appeared legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-input (J4, as the moves units' isDir and lexists); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): new, an Lstat that never follows a link, of the instance directories config passes could not write into, with a retry backoff for a host a pass left undone; it decides only whether the agent runs its own config pass again file:Lstat@135 691589305b19ebe4e7311bb853ae68003676786ce0684e6e7133382d26c982c7

+internal/host/agent.go pendingDirs.appeared legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-input (J4, as the moves units' isDir and lexists); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): new, an Lstat that never follows a link, of the instance directories config passes could not write into, with a retry backoff for a host a pass left undone; it decides only whether the agent runs its own config pass again file:Lstat@135 691589305b19ebe4e7311bb853ae68003676786ce0684e6e7133382d26c982c7

+@legacy d46b09858688390f1b1f5efe86d3816dbdbe642720081ddbc8acc7e109f53ac4 internal/host/schedule_wiring.go newScheduleDeps legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from armSchedule: main's body with the runner's Skip set to scheduleSkip; no new read site registry:App@1479,registry:App@388,registry:Host@1546,registry:Host@396,whole:Route@1474,whole:Route@383 aa93bed2781d1005cf864ebaaf6ac4597d6f547c45d6a1063f10c4a483ba46c3

+internal/host/schedule_wiring.go newScheduleDeps legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, main's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from armSchedule: main's body with the runner's Skip set to scheduleSkip; no new read site registry:App@1479,registry:App@388,registry:Host@1546,registry:Host@396,whole:Route@1474,whole:Route@383 aa93bed2781d1005cf864ebaaf6ac4597d6f547c45d6a1063f10c4a483ba46c3

+@legacy e324727c3eaaa15ef864e46a26391bca939383e96f8c2fe9b1b4252478469481 internal/host/schedule_wiring.go scheduleSkip legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as the runner's own hold checks); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its schedule runner: new, reads a row's status (any hold) and host (the armed offline view: the record names another region) to tell the runner to leave the host alone; it only withholds a tick, never grants one registry:Host@155,registry:Status@64 4f471727429b5b79bfa909693a09e588051b9f490eafd8480ee9893581362d64

+internal/host/schedule_wiring.go scheduleSkip legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as the runner's own hold checks); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its schedule runner: new, reads a row's status (any hold) and host (the armed offline view: the record names another region) to tell the runner to leave the host alone; it only withholds a tick, never grants one registry:Host@155,registry:Status@64 4f471727429b5b79bfa909693a09e588051b9f490eafd8480ee9893581362d64

+@legacy db611291d2e3616ea5dc6a3dfdb540d850095a2a8e58bab324350e8aff53d789 internal/schedule/runner.go Runner.Tick legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its scheduler: census-ruling-seeded-instances' body with a host Skip names given no tick and no bookkeeping at the pre-filter, and its due query moved into Runner.dueRows; no root decision registry:Host@1352,registry:Host@1394,registry:Host@1415,registry:Host@1439,registry:Host@5457,registry:ImportGate@2969,registry:ImportGate@3884,registry:Status@2762,registry:Status@3870,whole:Route@1821,whole:Route@2111,whole:Route@2240,whole:Route@3650,whole:Route@3841,whole:Route@4974,whole:Route@5323,whole:Route@5498,whole:Route@5534,whole:Timer@1219,whole:Timer@1323,whole:Timer@1345,whole:Timer@1408,whole:Timer@1432,whole:Timer@1448,whole:Timer@4649,whole:Timer@4978,whole:Timer@5538 0c7ef6fbd18e1be821561b82bd059ffdb5e9ac3fe1a35d3b6e1cb8aada3a9edc

+internal/schedule/runner.go Runner.Tick legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its scheduler: census-ruling-seeded-instances' body with a host Skip names given no tick and no bookkeeping at the pre-filter, and its due query moved into Runner.dueRows; no root decision registry:Host@1352,registry:Host@1394,registry:Host@1415,registry:Host@1439,registry:Host@5457,registry:ImportGate@2969,registry:ImportGate@3884,registry:Status@2762,registry:Status@3870,whole:Route@1821,whole:Route@2111,whole:Route@2240,whole:Route@3650,whole:Route@3841,whole:Route@4974,whole:Route@5323,whole:Route@5498,whole:Route@5534,whole:Timer@1219,whole:Timer@1323,whole:Timer@1345,whole:Timer@1408,whole:Timer@1432,whole:Timer@1448,whole:Timer@4649,whole:Timer@4978,whole:Timer@5538 0c7ef6fbd18e1be821561b82bd059ffdb5e9ac3fe1a35d3b6e1cb8aada3a9edc

+internal/schedule/runner.go Runner.fire hint E2 accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (main: J1) PROPOSED E46: captured claim selection and root journal mutation; root RPC outside service host exclusion; SQL/route data never grants root authority. Executing identity: service controller; root for inventory/retirement callbacks. moves-offline-fixes: with Skip set (only the agent's runner sets it, scheduleSkip), the row is read again before the token and a skipped host gets no token and no record; class and task unchanged registry:Host@243,registry:Host@385,registry:Host@691,registry:Host@837,registry:Name@252,registry:Name@394,registry:Spec@148,registry:Spec@260,whole:Route@1005,whole:Route@1202,whole:Route@1504,whole:Route@1567,whole:Route@1735,whole:Route@662,whole:Route@742,whole:Route@768,whole:Route@879,whole:Timer@1009,whole:Timer@108,whole:Timer@1206,whole:Timer@1306,whole:Timer@1508,whole:Timer@1739,whole:Timer@317,whole:Timer@883 1aaf0e167a6852ab6958cb409aaac63523334b4595e1cb8a0e30ac96f4499e9d

+@legacy a1d5746862e64c4ea2a5b27cfe36565b7b2a11968923461eb6e4dbff72137b36 internal/schedule/runner.go Runner.dueRows legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as Runner.Tick); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its scheduler: new, Tick's due query moved out of Tick: held hosts left out by the query when the hold wiring is armed or Skip is set, and each host Skip names (one registry row read per host in the batch) added to the exclusion and the batch read again, so a skipped host's rows never fill the limit; it only withholds ticks registry:Host@489,registry:Host@525,registry:Host@567,registry:Host@647,whole:Route@590,whole:Route@611,whole:Timer@186,whole:Timer@264,whole:Timer@366,whole:Timer@388,whole:Timer@472,whole:Timer@699 6e0d3045eebd802a7d3936a84d7fcbd8a9edd2741e5a13ed8c595c7dcfdaa2c1

+internal/schedule/runner.go Runner.dueRows legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-fixes;basis=root-route (J1, as Runner.Tick); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), from its scheduler: new, Tick's due query moved out of Tick: held hosts left out by the query when the hold wiring is armed or Skip is set, and each host Skip names (one registry row read per host in the batch) added to the exclusion and the batch read again, so a skipped host's rows never fill the limit; it only withholds ticks registry:Host@489,registry:Host@525,registry:Host@567,registry:Host@647,whole:Route@590,whole:Route@611,whole:Timer@186,whole:Timer@264,whole:Timer@366,whole:Timer@388,whole:Timer@472,whole:Timer@699 6e0d3045eebd802a7d3936a84d7fcbd8a9edd2741e5a13ed8c595c7dcfdaa2c1

diff --git a/internal/schedule/runner.go b/internal/schedule/runner.go

index f340d5d24..24435450d 100644

--- a/internal/schedule/runner.go

+++ b/internal/schedule/runner.go

@@ -70,6 +70,15 @@ type Runner struct {

// inside fire for the rows that get that far, so a tick delivers only

// when the hold and the admission both allow it.

Gate *HostGate

+ // Skip names a host this box must not touch at all, whatever the hold

+ // wiring says: its due timers are neither delivered nor written, no

+ // token is minted for it and nothing is reported. The agent sets it for

+ // a held row and for a host whose offline move's record names another

+ // region (harness Run 9, defect 8: after open, the source's tick minted

+ // a fresh <host>.env in the instance directory the move had parked).

+ // Asked at the tick's route read and again just before the token, so a

+ // hold written in between is seen. Nil: none is skipped.

+ Skip func(rt registry.Route) bool

// beforeClaim is the tests' pause between a tick's route read and its

// claim (TestDrainRaceCannotAdvanceBookkeeping). Nil in the agent.

@@ -142,13 +151,13 @@ func (r *Runner) Tick(ctx context.Context) {

// the same reason: a held row is skipped below with no bookkeeping, so

// a held host's rows would sit at the front of every tick's batch and

// starve the rest. Off, the query is main's.

- var due []registry.Timer

- var err error

- if r.holdsArmed() {

- due, err = r.Reg.DueUnheldTimers(now.Unix(), dueBatch, r.busyHosts())

- } else {

- due, err = r.Reg.DueTimers(now.Unix(), dueBatch, r.busyHosts())

- }

+ //

+ // Skip set (the agent's scheduleSkip) is the same rule for the hosts it

+ // names, before the limit too (Astra, moves-offline-fixes review 1): a

+ // held host's rows are left out by the query, and a host Skip names on

+ // other grounds is added to the exclusion and the batch read again, so

+ // a skipped host's due rows can never fill the batch.

+ due, err := r.dueRows(now)

if err != nil {

log.Printf("schedule: due query: %v", err)

return

@@ -187,6 +196,9 @@ func (r *Runner) Tick(ctx context.Context) {

// Converge prunes these; until it does, they are not ours to fire.

continue

}

+ if r.Skip != nil && r.Skip(*rt) {

+ continue // not this box's host to touch: no bookkeeping either

+ }

// A held row (moves spec §5.2, §9.4 "frozen") is skipped with no

// bookkeeping at all: before this, its delivery was refused by the

// activator and recorded as a retry, so a held row's attempts

@@ -275,6 +287,43 @@ func (r *Runner) Tick(ctx context.Context) {

}

+// dueRows is Tick's due query: busy hosts excluded, held hosts too when the

+// hold wiring is armed or Skip is set, and, with Skip set, every host it

+// names, read again with each one found added to the exclusion. Each read

+// either has no skipped host or adds at least one host to the exclusion, so

+// the loop ends within one read per host with due rows. A host whose row

+// cannot be read is left in the batch: the tick's own read reports it.

+func (r *Runner) dueRows(now time.Time) ([]registry.Timer, error) {

+ exclude := r.busyHosts()

+ for {

+ var due []registry.Timer

+ var err error

+ if r.holdsArmed() || r.Skip != nil {

+ due, err = r.Reg.DueUnheldTimers(now.Unix(), dueBatch, exclude)

+ } else {

+ due, err = r.Reg.DueTimers(now.Unix(), dueBatch, exclude)

+ }

+ if err != nil || r.Skip == nil || len(due) == 0 {

+ return due, err

+ }

+ var skipped []string

+ seen := map[string]bool{}

+ for _, t := range due {

+ if seen[t.Host] {

+ continue

+ }

+ seen[t.Host] = true

+ if rt, err := r.Reg.Get(t.Host); err == nil && rt != nil && r.Skip(*rt) {

+ skipped = append(skipped, t.Host)

+ }

+ if len(skipped) == 0 {

+ return due, nil

+ }

+ exclude = append(exclude, skipped...)

+ }

// busyHosts is the in-flight set as DueTimers wants it. A snapshot: a host

// that goes busy after this read is caught by claim instead.

func (r *Runner) busyHosts() []string {

@@ -498,6 +547,19 @@ func (r *Runner) fire(ctx context.Context, rt registry.Route, t registry.Timer)

}

return

}

+ if r.Skip != nil {

+ // The token is the one write a tick makes on the instance's own

+ // ground, so the row is read again right before it.

+ var current *registry.Route

+ err := withTimerWork(ctx, func() error {

+ var readErr error

+ current, readErr = r.Reg.Get(rt.Host)

+ return readErr

+ })

+ if err != nil || current == nil || r.Skip(*current) {

+ return

+ }

start := r.now()

token, err := r.Token(rt.Host)

if err != nil {

diff --git a/internal/schedule/runner_test.go b/internal/schedule/runner_test.go

index 155e2cb39..d174327a9 100644

--- a/internal/schedule/runner_test.go

+++ b/internal/schedule/runner_test.go

@@ -1400,3 +1400,95 @@ func TestGatedTimersDoNotStarveAnotherHost(t *testing.T) {

})

}

+// Skip (harness Run 9, defect 8): a host the agent says not to touch, here a

+// held row with the move-hold wiring off (an offline move's source after

+// open), gets no token, no delivery and no bookkeeping. And a hold written

+// between the tick's route read and the fire is seen before the token, the

+// one write a tick makes on the instance's own ground.

+func TestSkipMintsNoTokenAndRecordsNothing(t *testing.T) {

+ d := &fakeDeliverer{status: 204}

+ reg, r := runnerFixture(t, d)

+ now := at("2026-10-20T17:27:04Z")

+ r.Now = func() time.Time { return now }

+ tokens := 0

+ r.Token = func(string) (string, error) { tokens++; return "tok", nil }

+ r.Skip = func(rt registry.Route) bool { return registry.IsHold(rt.Status) }

+ if err := reg.UpsertTimer(registry.Timer{Host: holdHost, Name: "every-minute", Kind: KindSchedule, Spec: "every 1m0s", Path: "/tick", NextAt: now.Unix() - 5}); err != nil {

+ t.Fatal(err)

+ }

+ if err := reg.SetHold(holdHost, "moving", "mv_a"); err != nil {

+ t.Fatal(err)

+ }

+ before := timersOf(t, reg, holdHost)

+ r.tick()

+ if tokens != 0 || len(d.calls) != 0 || !sameTimers(timersOf(t, reg, holdHost), before) || r.Busy(holdHost) {

+ t.Fatalf("a skipped host: %d tokens, %d deliveries, timers %+v (were %+v)", tokens, len(d.calls), timersOf(t, reg, holdHost), before)

+ }

+ // The row reads ready at the tick's read; the hold lands before the claim.

+ if ok, err := reg.ReleaseHold(holdHost, "mv_a"); err != nil || !ok {

+ t.Fatalf("release hold = %v %v", ok, err)

+ }

+ r.beforeClaim = func(host string) {

+ if err := reg.SetHold(host, "moving", "mv_b"); err != nil {

+ t.Error(err)

+ }

+ r.tick()

+ if tokens != 0 || len(d.calls) != 0 || !sameTimers(timersOf(t, reg, holdHost), before) {

+ t.Fatalf("a hold written before the fire: %d tokens, %d deliveries", tokens, len(d.calls))

+ }

+ // Ready again, nothing skipped: the tick fires as it always has.

+ if ok, err := reg.ReleaseHold(holdHost, "mv_b"); err != nil || !ok {

+ t.Fatalf("release hold = %v %v", ok, err)

+ }

+ r.beforeClaim = nil

+ r.tick()

+ if tokens != 1 || len(d.calls) != 1 {

+ t.Fatalf("a ready host: %d tokens, %d deliveries", tokens, len(d.calls))

+ }

+// Astra's review of moves-offline-fixes, finding 1: with the hold wiring off

+// (Gate nil, as on an offline move's boxes), a host Skip names must not fill

+// the due batch. 64 earlier due rows of a skipped host, held or not, and one

+// later row of an ordinary host: the ordinary one fires on the first tick.

+func TestSkippedHostsDoNotStarveAnotherHostsTimer(t *testing.T) {

+ for _, held := range []bool{true, false} {

+ d := &fakeDeliverer{status: 204}

+ reg, r := runnerFixture(t, d)

+ now := at("2026-10-20T17:27:04Z")

+ r.Now = func() time.Time { return now }

+ const other = "b.bab.oncarlos.com"

+ if err := reg.Upsert(registry.Route{Host: other, Kind: "instance", Account: "bab", App: "app", Status: "ready", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ if held {

+ if err := reg.SetHold(holdHost, "moving", "mv_a"); err != nil {

+ t.Fatal(err)

+ }

+ r.Skip = func(rt registry.Route) bool { return registry.IsHold(rt.Status) }

+ } else {

+ // Ready, but named elsewhere (an offline move's record names

+ // another region): only Skip knows.

+ r.Skip = func(rt registry.Route) bool { return rt.Host == holdHost }

+ }

+ for i := range dueBatch {

+ if err := reg.UpsertTimer(registry.Timer{Host: holdHost, Name: fmt.Sprintf("t%02d", i), Kind: KindSchedule, Spec: "every 1m0s", Path: "/tick", NextAt: now.Unix() - 1000 + int64(i)}); err != nil {

+ t.Fatal(err)

+ }

+ if err := reg.UpsertTimer(registry.Timer{Host: other, Name: "later", Kind: KindSchedule, Spec: "every 1m0s", Path: "/tick", NextAt: now.Unix() - 5}); err != nil {

+ t.Fatal(err)

+ }

+ if r.Gate != nil {

+ t.Fatal("fixture: the hold wiring is armed")

+ }

+ r.tick()

+ if len(d.calls) != 1 || d.hosts[0] != other {

+ t.Fatalf("held %v: deliveries to %v, want exactly one to %s", held, d.hosts, other)

+ }