offline-move: record shape, transitions, manifest and copy rows
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
7 files changed,
+1768
−2
approved_moves_copy_test.go+6 −2docs/superpowers/plans/2026-10-06-moves-phase1-copy.md+36 −0internal/moves/offline.go+316 −0internal/moves/offline_test.go+335 −0internal/provision/offline_move.go+293 −0internal/provision/offline_move_test.go+761 −0internal/provision/provision.go+21 −0
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 312.547s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.728s
ok github.com/carlosframework/platform/cmd/moveprobe 93.236s
ok github.com/carlosframework/platform/cmd/zddprobe 0.705s
ok github.com/carlosframework/platform/internal/activator 251.151s
ok github.com/carlosframework/platform/internal/agentsock 0.016s
ok github.com/carlosframework/platform/internal/alerting 0.003s
ok github.com/carlosframework/platform/internal/analytics 0.020s
ok github.com/carlosframework/platform/internal/appconfig 2.486s
ok github.com/carlosframework/platform/internal/apperr 0.091s
ok github.com/carlosframework/platform/internal/applog 0.146s
ok github.com/carlosframework/platform/internal/awsinv 0.004s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.892s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.015s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.697s
ok github.com/carlosframework/platform/internal/changelog 0.322s
ok github.com/carlosframework/platform/internal/claimactivation 0.813s
ok github.com/carlosframework/platform/internal/clamd 0.055s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 406.673s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.199s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.005s
ok github.com/carlosframework/platform/internal/dnsrec 0.005s
ok github.com/carlosframework/platform/internal/docsite 0.004s
ok github.com/carlosframework/platform/internal/domains 2.403s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 45.015s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.130s
ok github.com/carlosframework/platform/internal/gate 1.654s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.016s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.154s
ok github.com/carlosframework/platform/internal/host 182.442s
ok github.com/carlosframework/platform/internal/hostactors 5.477s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.016s
ok github.com/carlosframework/platform/internal/hostbox 2.612s
ok github.com/carlosframework/platform/internal/hostclaim 0.581s
ok github.com/carlosframework/platform/internal/hostenv 0.016s
ok github.com/carlosframework/platform/internal/hostevidence 0.016s
ok github.com/carlosframework/platform/internal/hostexec 4.872s
ok github.com/carlosframework/platform/internal/hostfence 0.030s
ok github.com/carlosframework/platform/internal/hostfiles 10.501s
ok github.com/carlosframework/platform/internal/hostfloor 0.016s
ok github.com/carlosframework/platform/internal/hostguard 1.000s
ok github.com/carlosframework/platform/internal/hostops 1.581s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.015s
ok github.com/carlosframework/platform/internal/hostquarantine 0.016s
ok github.com/carlosframework/platform/internal/hostrelocate 0.468s
ok github.com/carlosframework/platform/internal/hostresolve 0.137s
ok github.com/carlosframework/platform/internal/hostwrite 0.015s
ok github.com/carlosframework/platform/internal/infraview 0.015s
ok github.com/carlosframework/platform/internal/instancectl 5.235s
ok github.com/carlosframework/platform/internal/instanceexec 2.742s
ok github.com/carlosframework/platform/internal/instancefs 0.020s
ok github.com/carlosframework/platform/internal/instancekey 0.017s
ok github.com/carlosframework/platform/internal/instanceupdates 0.023s
ok github.com/carlosframework/platform/internal/layout 1.563s
ok github.com/carlosframework/platform/internal/lbhealth 0.002s
ok github.com/carlosframework/platform/internal/lease 9.815s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.013s
ok github.com/carlosframework/platform/internal/ledger 0.014s
ok github.com/carlosframework/platform/internal/mail 0.161s
ok github.com/carlosframework/platform/internal/maintenance 0.086s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.515s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.078s
ok github.com/carlosframework/platform/internal/metrics 0.004s
ok github.com/carlosframework/platform/internal/moves 21.037s
ok github.com/carlosframework/platform/internal/moves/archive 0.803s
ok github.com/carlosframework/platform/internal/objstore 0.187s
ok github.com/carlosframework/platform/internal/ops 0.187s
ok github.com/carlosframework/platform/internal/park 0.208s
ok github.com/carlosframework/platform/internal/privatefile 0.004s
2026/10/20 00:09:59 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/20 00:09:59 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:09:59 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:09:59 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/20 00:09:59 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:09:59 provision: an app's records could not be listed: they are skipped
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:09:59 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:09:59 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:09:59 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:09:59 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:09:59 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:09:59 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:09:59 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:09:59 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:09:59 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:09:59 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:09:59 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:09:59 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:09:59 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:09:59 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:09:59 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:09:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:09:59 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:09:59 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:09:59 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:09:59 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:09:59 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:00 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:10:00 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:10:00 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/20 00:10:00 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/20 00:10:00 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:00 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:00 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (4.63s)
plan14_root_reads_test.go:940: root read census: 3 discrepancies
changed root source envelope internal/moves
changed root source envelope internal/provision
unclassified root read internal/provision/offline_move.go:OfflineStart file:Open@184
--- FAIL: TestPlan9B8RecordWriteCensus (0.54s)
plan9_b8_census_test.go:372: record-write census rejected 1 site(s); update the explicit inventory only after reviewing the source:
unlisted site internal/moves/offline.go:var OfflineManifest [lit:instances]: review it, classify it, and update its explicit inventory entry
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:10:28 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/20 00:10:28 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/20 00:10:28 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/20 00:10:28 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io hibernate false -> true
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged acme.go.tito.io status-named false -> true
2026/10/20 00:10:28 provision: converged acme.go.tito.io status-named true -> false
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:10:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:10:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:10:28 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:10:28 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/20 00:10:28 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/20 00:10:28 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:10:28 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:10:28 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:10:28 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:10:28 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:28 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:10:28 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/20 00:10:28 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/20 00:10:28 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:28 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:10:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:28 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/20 00:10:28 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:10:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:28 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:10:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:10:28 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:10:28 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:10:28 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:10:28 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:10:28 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/20 00:10:28 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:10:28 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:10:28 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/20 00:10:29 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:10:29 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:10:29 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/20 00:10:29 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/20 00:10:29 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/20 00:10:29 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:10:29 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/20 00:10:29 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/20 00:10:29 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/20 00:10:29 provision: dry-run: would converge owned.go.tito.io seed
2026/10/20 00:10:29 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:10:29 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/20 00:10:29 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:10:29 provision: converged owned.go.tito.io channel stable -> edge
2026/10/20 00:10:29 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/20 00:10:29 provision: converged owned.go.tito.io hibernate false -> true
2026/10/20 00:10:29 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/20 00:10:29 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/20 00:10:29 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: removed retired static route site.go.tito.io (record gone)
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/20 00:10:29 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/20 00:10:29 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:29 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:10:29 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:10:29 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:10:29 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:10:29 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/20 00:10:29 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:10:29 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:10:29 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:10:29 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:10:29 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/20 00:10:29 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/20 00:10:29 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:10:29 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/20 00:10:29 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/20 00:10:29 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:10:29 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/20 00:10:29 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:10:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:29 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/20 00:10:30 provision: a.shop.example.com: swapped for import sd_772s4oiqjqjiivpz35vpi5euwe (the original route is journaled)
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_772s4oiqjqjiivpz35vpi5euwe committed: its gate is cleared and its journal row dropped
2026/10/20 00:10:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_kfxn2xqmhgmktqnw7cdsamp7kq rolled back to its original route
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_kfxn2xqmhgmktqnw7cdsamp7kq: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_kfxn2xqmhgmktqnw7cdsamp7kq is cleaned up and its journal row retired
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_nvme37pso7wtnvrqyltsijthom rolled back to its original route
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_nvme37pso7wtnvrqyltsijthom: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_nvme37pso7wtnvrqyltsijthom is cleaned up and its journal row retired
2026/10/20 00:10:30 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_gweg6is6xmeegdibw6apqfdrmu
2026/10/20 00:10:30 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/20 00:10:30 provision: dry-run: would converge a.shop.example.com seed source
2026/10/20 00:10:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/20 00:10:30 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/20 00:10:30 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_udf5gxm2iveooauy4iidk2hm2a cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_a5p5lcvyr2udcldsteuqve2aqq cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_gihyim22mtudfanwkx5xifuwwe is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_k553hj7ieoas725xleiqjmkxhq is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_t232hp7e7er2x3ye5776bzuvaa is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:10:30 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/20 00:10:30 provision: b.shop.example.com: import sd_5gpncxyikbong4elw3ffhfqtg4 left alone this pass: the import journal could not be read
2026/10/20 00:10:30 provision: c.shop.example.com: import sd_gj6xy526qqivpvubrjkw4azogi left alone this pass: the import journal could not be read
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_6zbrujnaoocwhtooqypgnyhove committed: its gate is cleared and its journal row dropped
2026/10/20 00:10:30 provision: a.shop.example.com: not converging import sd_igtlixutvmgpsyd5pe3el6w7w4: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:10:30 provision: a.shop.example.com: not converging import sd_vos7tmaylfdrof74cnifbwf2gu: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:10:30 provision: a.shop.example.com: not converging import sd_jqruguf6lqgo6ric5v2ue7ldo4: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:10:30 provision: a.shop.example.com: not converging import sd_w6sttnh6lcoazeoeai7osu6qea: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:10:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/20 00:10:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: a.shop.example.com: import sd_ajmafrs34vvlev22usqwkuinqi committed: its gate is cleared and its journal row dropped
2026/10/20 00:10:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:30 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:30 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/20 00:10:30 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/20 00:10:30 provision: acme.go.tito.io: aborted seed sd_fmigj3sy43hvpo3uxvdsxlgfba: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:10:30 provision: acme.go.tito.io: aborted seed sd_fmigj3sy43hvpo3uxvdsxlgfba rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/20 00:10:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:31 provision: removed instance route acme.go.tito.io (its seed sd_ekekzfeppmwmtaxhx3vomxe7ku was aborted); the next pass rolls it back
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_ekekzfeppmwmtaxhx3vomxe7ku: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_ekekzfeppmwmtaxhx3vomxe7ku rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_62p647uwyv4utqqhyutgzjr4du has a row this pass may not remove; leaving the seed aborted
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_c5v7dxr5r7t34fchfdadc4muga rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:31 provision: removed instance route acme.go.tito.io (its seed sd_yedspn7apmdxs5ktvqu2ztf7au was aborted); the next pass rolls it back
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_yedspn7apmdxs5ktvqu2ztf7au has no route on this box: its local data is removed
2026/10/20 00:10:31 provision: acme.go.tito.io: aborted seed sd_yedspn7apmdxs5ktvqu2ztf7au rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_5bycf6ruyctrqipzw6gzch3gsy (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_5bycf6ruyctrqipzw6gzch3gsy rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_5bycf6ruyctrqipzw6gzch3gsy is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_ynoew7g7josu4iohmsf5erxazu (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_ynoew7g7josu4iohmsf5erxazu rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_ynoew7g7josu4iohmsf5erxazu is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_xrhhe6qdsrj2n7o5bw3u5zgzke (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_xrhhe6qdsrj2n7o5bw3u5zgzke rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_xrhhe6qdsrj2n7o5bw3u5zgzke: the host's replica is left alone: its record no longer names this seed
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_xrhhe6qdsrj2n7o5bw3u5zgzke is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_chlxvfi3anhnibgni64gcrubz4 (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_chlxvfi3anhnibgni64gcrubz4 rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_chlxvfi3anhnibgni64gcrubz4 rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_chlxvfi3anhnibgni64gcrubz4 is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_jri372lrxlkt5guvz3xyoaybaa (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_jri372lrxlkt5guvz3xyoaybaa rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_6mzcjetar3rnkty24y4jt4uflm (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_6mzcjetar3rnkty24y4jt4uflm is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_w7gpfxbgfju6w7k6xlkvgxghli (the original route is journaled)
2026/10/20 00:10:31 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_w7gpfxbgfju6w7k6xlkvgxghli left alone this pass: its record could not be read
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_nm3cpj3aee6zxruqhl6f7hbp5e (the original route is journaled)
2026/10/20 00:10:31 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_nm3cpj3aee6zxruqhl6f7hbp5e left alone this pass: its record could not be read
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_bkgyn4eijavxvws5bzfjwlt5ny (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkgyn4eijavxvws5bzfjwlt5ny: close the import gate: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkgyn4eijavxvws5bzfjwlt5ny rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkgyn4eijavxvws5bzfjwlt5ny is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_7iajjzmfpvvb6zqcegpnr7vtzm (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_7iajjzmfpvvb6zqcegpnr7vtzm: close the import gate: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_7iajjzmfpvvb6zqcegpnr7vtzm rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_7iajjzmfpvvb6zqcegpnr7vtzm is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_tym3o3vgj5xcqsgg6uk2w3wdla (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_tym3o3vgj5xcqsgg6uk2w3wdla rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_tym3o3vgj5xcqsgg6uk2w3wdla rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_tym3o3vgj5xcqsgg6uk2w3wdla is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_7bg3ce3x4t6nklbkhzphk2jnd4 (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_7bg3ce3x4t6nklbkhzphk2jnd4 rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_7bg3ce3x4t6nklbkhzphk2jnd4 is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_zxzgtnlq7ftuc4mnbksa63asbm (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_zxzgtnlq7ftuc4mnbksa63asbm: retire its instance key: injected failure at retire-key
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_zxzgtnlq7ftuc4mnbksa63asbm rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_zxzgtnlq7ftuc4mnbksa63asbm is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_ft2c3wlu7ka7prmww7sglzmuia (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_ft2c3wlu7ka7prmww7sglzmuia rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_ft2c3wlu7ka7prmww7sglzmuia is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_rwuxtac7ruhfkihw3vohs7z7hy (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_rwuxtac7ruhfkihw3vohs7z7hy: remove its local data: injected failure at remove-local
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_rwuxtac7ruhfkihw3vohs7z7hy rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_rwuxtac7ruhfkihw3vohs7z7hy is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_33cwtkyfw3vwj7jlq5ueo4qu5m (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_33cwtkyfw3vwj7jlq5ueo4qu5m rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_33cwtkyfw3vwj7jlq5ueo4qu5m is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_shhix7irrf6z35uku7wrviq7du (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_shhix7irrf6z35uku7wrviq7du: remove its local data: injected failure at remove-local-done
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_shhix7irrf6z35uku7wrviq7du rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_shhix7irrf6z35uku7wrviq7du is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_mh72tnfzisziucghl3iqinzege (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_mh72tnfzisziucghl3iqinzege rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_mh72tnfzisziucghl3iqinzege is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_twtvygmzv43zcgqzt6fbyxn4ye (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_twtvygmzv43zcgqzt6fbyxn4ye: restore the original route: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_twtvygmzv43zcgqzt6fbyxn4ye rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_twtvygmzv43zcgqzt6fbyxn4ye is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_bkwttmztzujsakqu6lybivnuue (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkwttmztzujsakqu6lybivnuue: restore the original route: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkwttmztzujsakqu6lybivnuue rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bkwttmztzujsakqu6lybivnuue is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_2xgrgeq4lcomp6skessk4m2w3i (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_2xgrgeq4lcomp6skessk4m2w3i is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_acrqpkiei7iiwg6ffqgobgv5fa (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_acrqpkiei7iiwg6ffqgobgv5fa rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_acrqpkiei7iiwg6ffqgobgv5fa: delete its snapshot: injected failure at delete-snapshot
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_acrqpkiei7iiwg6ffqgobgv5fa is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_sjgfoosqz2lm3hjkizm2eu3awi (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_sjgfoosqz2lm3hjkizm2eu3awi rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_sjgfoosqz2lm3hjkizm2eu3awi is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_uzmoi3chp7hkmlfun6ilpms4im (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_uzmoi3chp7hkmlfun6ilpms4im rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_uzmoi3chp7hkmlfun6ilpms4im: delete the host's replica: injected failure at delete-replica
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_uzmoi3chp7hkmlfun6ilpms4im is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_6fxn6ygbzdxsw5rm2r47hzcyie (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_6fxn6ygbzdxsw5rm2r47hzcyie rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_6fxn6ygbzdxsw5rm2r47hzcyie is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_2kisxl52auiuzaufbynxqsdere (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_2kisxl52auiuzaufbynxqsdere rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_2kisxl52auiuzaufbynxqsdere: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_2kisxl52auiuzaufbynxqsdere is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_mu3oorgdbtdkmlfnwdxlyqryaa (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_mu3oorgdbtdkmlfnwdxlyqryaa rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_mu3oorgdbtdkmlfnwdxlyqryaa: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_mu3oorgdbtdkmlfnwdxlyqryaa is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_726rtqqqga7ghischvcdecamo4 (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_726rtqqqga7ghischvcdecamo4 rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_726rtqqqga7ghischvcdecamo4 is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: mark import sd_726rtqqqga7ghischvcdecamo4 rolled back: injected failure at cas
2026/10/20 00:10:31 provision: a.shop.example.com: aborted seed sd_726rtqqqga7ghischvcdecamo4 rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_auxzgfpby6lgiff4wttbqsuxce (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_auxzgfpby6lgiff4wttbqsuxce rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_auxzgfpby6lgiff4wttbqsuxce is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: aborted seed sd_auxzgfpby6lgiff4wttbqsuxce rolled back (no box had swapped it)
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_3xobo6dptzmvrkf6wbvai2i3ba (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_3xobo6dptzmvrkf6wbvai2i3ba rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_3xobo6dptzmvrkf6wbvai2i3ba rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_3xobo6dptzmvrkf6wbvai2i3ba rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_3xobo6dptzmvrkf6wbvai2i3ba is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_icnbt6gbzja6ogvhbymwat4ytm (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_icnbt6gbzja6ogvhbymwat4ytm rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_icnbt6gbzja6ogvhbymwat4ytm is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_spvqr5bwabgdlkyvq6dpkpdz2q (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_spvqr5bwabgdlkyvq6dpkpdz2q rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_spvqr5bwabgdlkyvq6dpkpdz2q is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_jnaknmj6uki2m6l567xgtxb4ba (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_jnaknmj6uki2m6l567xgtxb4ba rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_jnaknmj6uki2m6l567xgtxb4ba is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_23wcqcokwsq64wi3bp2df3ajpq (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_23wcqcokwsq64wi3bp2df3ajpq rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_23wcqcokwsq64wi3bp2df3ajpq is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_t7iqyx5n5gmyos5xamv5vyws6a (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_t7iqyx5n5gmyos5xamv5vyws6a rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_t7iqyx5n5gmyos5xamv5vyws6a is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_j7zm3f2k5xiaaycnt42xk6nq2i (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_j7zm3f2k5xiaaycnt42xk6nq2i rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_j7zm3f2k5xiaaycnt42xk6nq2i is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_bzjicifon3chaszwfeettnr6qq (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bzjicifon3chaszwfeettnr6qq rolled back to its original route
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_bzjicifon3chaszwfeettnr6qq is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_qyjcnvvppc4hikqk26xeb3wm3m (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_qyjcnvvppc4hikqk26xeb3wm3m has no route on this box: its local data is removed and no route is put back
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_qyjcnvvppc4hikqk26xeb3wm3m is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_z4bmg5tlc5bbdvxwliwlldqif4 (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_7owgoevyw27vjl5a5uzjhdoxym (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_owtwkehyudc454hc2ww4vaxmiu (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_owtwkehyudc454hc2ww4vaxmiu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_ys7ukmao6bqmv2k664p2cg4jwy (the original route is journaled)
2026/10/20 00:10:31 provision: a.shop.example.com: import sd_ys7ukmao6bqmv2k664p2cg4jwy is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/20 00:10:31 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/20 00:10:31 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/20 00:10:31 provision: a.shop.example.com: swapped for import sd_75via6iestvdrec2svr7xfj7yu (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_c6cnkby53tqjd65ues4kw2iv3m (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_3x3ewe5rbpgzmw2nk4slyv7oou (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_46mbrioy32j4n7szf7r6bfpj3a (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_pmlho4lcqkkrx6rtb6nfffa3g4 (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_pmlho4lcqkkrx6rtb6nfffa3g4 rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_reeqnzgek2dvqpzdikpiq2luai (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_reeqnzgek2dvqpzdikpiq2luai has no route on this box: its local data is removed and no route is put back
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_reeqnzgek2dvqpzdikpiq2luai is cleaned up and its journal row retired
2026/10/20 00:10:32 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_ksuetzjn3wjvy7nueuhdp3qqxm (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_ksuetzjn3wjvy7nueuhdp3qqxm: remove its local data: injected failure at remove-local
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_ksuetzjn3wjvy7nueuhdp3qqxm is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_7othhlfj5xhzv3tbuti2ojzaeq (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_7othhlfj5xhzv3tbuti2ojzaeq: a route appeared at the host, so its local database is left and the rollback waits
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_azos62yo6petdhq4ugcah7obry (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_azos62yo6petdhq4ugcah7obry rollback deferred: no fence generation: registry busy
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_fymcvks3fpshj5umpi6w3hno4q (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_fymcvks3fpshj5umpi6w3hno4q rollback deferred: no fence generation: the generation source answered 0
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_hpisytc4wwzawa2s6ccb6l3uci (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_4fuqhb325r2ntmdvpn3drpq4ri (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_lb6xeojptppi56r6yyqwkq7hma (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_5qdwrs2hlrhtidnb2ykbm4jt44 (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_5qdwrs2hlrhtidnb2ykbm4jt44 rolled back to its original route
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_5qdwrs2hlrhtidnb2ykbm4jt44 is cleaned up and its journal row retired
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_2d5hikydvrz35rl4ljqdvp5u5y (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_2d5hikydvrz35rl4ljqdvp5u5y rolled back to its original route
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_2d5hikydvrz35rl4ljqdvp5u5y is cleaned up and its journal row retired
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_34ie5rjcbw6fky6eynxr2emhoe (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_34ie5rjcbw6fky6eynxr2emhoe rolled back to its original route
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_34ie5rjcbw6fky6eynxr2emhoe is cleaned up and its journal row retired
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_sy5gdieltsrzzdjoz4e4kpx2d4 (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_sy5gdieltsrzzdjoz4e4kpx2d4: retire its instance key: the env file is a symlink
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_sy5gdieltsrzzdjoz4e4kpx2d4 rolled back to its original route
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_sy5gdieltsrzzdjoz4e4kpx2d4 is cleaned up and its journal row retired
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_ii6dtigcmwizfqor46jbrm4j5q (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_fbmdhxedm64njzysag5l7mlu4q (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_lw4mcpiwjkafkj4m2h5gqjtsqe refused: there is no route at a.shop.example.com to replace
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_notcfua2ggulz64x2gk5ssmlf4 refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_fmu2lm7blea5os3lgiz3c6zboe refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_w2ppkxeurvrcxqvpshclcye3re refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_i7oyla3t2bq57g5s3xntmvnrxi refused: the route belongs to app other
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_kaw3bqvdxv5f7ga4eqsmpzc2hu refused: the route is filed under another account
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_w3beeijgttjjejubllcjqrk67i refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_y63ts2xdri7ctba7zqzd4aswxq refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_uparievtfj3it2wl4s5mbpu6lm refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_jj5nb2cixfhogu7annmy32xgta refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_gsan6ebx4f4r75zsj2zki5myme refused: the route is not an address route this import can take over (socket)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_dylet7fkb7vc54ygeg2ia4ta2a refused: the route is not an address route this import can take over (no address)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_f7pnhqs7niwzvlxyyqmedk5ok4 refused: the route is not an address route this import can take over (provisioned)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_sxeb5lttphymmyw4kydxyipcii refused: the route is not an address route this import can take over (kind static)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_yof5r3fqbq6u25hihbkj7u2cvi refused: the route is not an address route this import can take over (backing)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_bved4pkfc7mepo7jg7gwm2uiby refused: the route is not an address route this import can take over (tunnel)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_zyxtoxko5tyxdggpodovva2d2e refused: the route is not an address route this import can take over (caps)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_mjxhwtomko5cojw6kkur5w66gi refused: the route is not an address route this import can take over (platform)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_rsxrte54fpczisujllvb2xbydq refused: the route is not an address route this import can take over (canary)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_2erzm5wo7n2leq2sgn4rbdctie refused: the route is not an address route this import can take over (kind sidecar)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_5flxdj2c242nhn3bpopgktadki refused: the route is not an address route this import can take over (region)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_axbu4ypor5omdbs3pjygrh4sne refused: the route is not an address route this import can take over (fleet_box)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_lqxafxeh3sp2gwrz4dtlise3ia refused: the route is not an address route this import can take over (hibernate)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_2pixrj7uuf4ozc6bvwk7dbhjn4 refused: the route is not an address route this import can take over (claim_owner)
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_upuybxktwtvv44naef66v3d2ka refused: this box does not run instances nested
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_lsp75rpsd5q7eatiuqsuvi4vpi refused: this box does not run instances nested
2026/10/20 00:10:32 provision: a.shop.example.com: import sd_24m32g4pcufepbf6e2ggsfrdcm refused: claim control is on for this box
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_nerhqknoylwhzbsbe5tip6yf7i (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_imbkjptezlmqbpsnlvqcqajc4a (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_vbha4sy6ayhkddqs7n53iqx36m: its verify hash is not a sha256
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_h22aedidt6q4k3upg5dnfravz4: its verify hash is not a sha256
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_dnwxrwgna2ymmxexccgqjcnxja: its verify hash is not a sha256
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/20 00:10:32 provision: A.shop.example.com/..: not swapping for import sd_m3s7jlfmjwwqpuotwudosv5vum: its host or seed id is malformed (seeds: not a host)
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_xn2uosgpb5e3g76yppsw2vdn4u: its record does not describe a nested, hibernating instance
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_rpybsjjkspnrdmsfr5itqaaqai: its record does not describe a nested, hibernating instance
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_trchuqw237567q6srdkmwm45zi: its record does not describe a nested, hibernating instance
2026/10/20 00:10:32 provision: a.shop.example.com: not swapping for import sd_eqaa4epcj3f73sl22sidhvsna4: its record does not describe a nested, hibernating instance
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_ghwrdxlydquvjm7jdrehfxm7ru (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_zngnxlewr6ejspjn4snypkifsq (the original route is journaled)
2026/10/20 00:10:32 provision: a.shop.example.com: swapped for import sd_zqe73huwvwmd2siu277ltpul4a (the original route is journaled)
2026/10/20 00:10:33 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/20 00:10:33 provision: dry-run: would consider swapping a.shop.example.com for import sd_mf3lc5tfgcszckxo7lui5za7cu
2026/10/20 00:10:33 provision: a.shop.example.com: import sd_fprbz7iasaa6alde6j6hg2ukwq rolled back to its original route
2026/10/20 00:10:33 provision: a.shop.example.com: import sd_fprbz7iasaa6alde6j6hg2ukwq: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:10:33 provision: a.shop.example.com: import sd_fprbz7iasaa6alde6j6hg2ukwq is cleaned up and its journal row retired
2026/10/20 00:10:33 provision: a.shop.example.com: import sd_uwv2tdywnfsi5ogj3umnkcdjki left alone: the import journal holds seed sd_j4mzzbcccaoyesyqphbxhdiwlm at swapped
2026/10/20 00:10:33 provision: a.shop.example.com: not minting import sd_eiy6e5t4eaifqyvghx3opikpxu: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:10:33 provision: a.shop.example.com: not minting import sd_nzrorinnqybqdl3byzab7opj4q: it is placed but not committed, and only its swap makes its row
2026/10/20 00:10:33 provision: a.shop.example.com: swapped for import sd_rkq5vphnutuoareikebbakrete (the original route is journaled)
2026/10/20 00:10:33 provision: a.shop.example.com: not minting import sd_rkq5vphnutuoareikebbakrete: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:10:33 provision: a.shop.example.com: not minting import sd_rkq5vphnutuoareikebbakrete: it is placed but not committed, and only its swap makes its row
2026/10/20 00:10:33 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:10:33 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/20 00:10:33 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/20 00:10:33 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/20 00:10:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/20 00:10:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/20 00:10:33 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:10:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/20 00:10:33 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/20 00:10:33 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:10:33 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:10:33 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/20 00:10:33 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/20 00:10:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:10:33 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/20 00:10:33 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:10:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:10:33 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/20 00:10:33 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:10:33 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:10:33 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:33 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/20 00:10:33 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:33 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:33 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:10:33 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:10:33 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:10:33 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:10:33 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:10:33 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:10:33 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL github.com/carlosframework/platform/internal/provision 34.199s
ok github.com/carlosframework/platform/internal/publicrelease 0.007s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.433s
ok github.com/carlosframework/platform/internal/relay 7.239s
ok github.com/carlosframework/platform/internal/release 24.608s
ok github.com/carlosframework/platform/internal/rollout 3.760s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.011s
ok github.com/carlosframework/platform/internal/rootparticipation 0.058s
ok github.com/carlosframework/platform/internal/rootstate 0.026s
ok github.com/carlosframework/platform/internal/runs 0.579s
ok github.com/carlosframework/platform/internal/s3compat 0.007s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.490s
ok github.com/carlosframework/platform/internal/seeds 0.010s
ok github.com/carlosframework/platform/internal/sim/hostlife 9.435s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.171s
ok github.com/carlosframework/platform/internal/status 0.069s
ok github.com/carlosframework/platform/internal/steering 0.005s
ok github.com/carlosframework/platform/internal/stopevidence 0.272s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.554s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go| index 71c38f7c3..73c8186e4 100644 |
| --- a/approved_moves_copy_test.go |
| +++ b/approved_moves_copy_test.go |
| @@ -91,6 +91,10 @@ var movesCopyIDs = []string{ |
| "MV-N500", |
| "MV-N501", "MV-N502", |
| "MV-N510", "MV-N511", "MV-N512", "MV-N513", |
| + // The offline operator move (plans/2026-10-20-offline-operator-move.md). |
| + "MV-O1", "MV-O2", "MV-O3", "MV-O4", "MV-O5", "MV-O6", "MV-O7", "MV-O8", "MV-O9", "MV-O10", |
| + "MV-O20", "MV-O21", "MV-O22", "MV-O23", "MV-O24", "MV-O25", "MV-O26", "MV-O27", "MV-O28", "MV-O29", "MV-O30", "MV-O31", |
| + "MV-O32", "MV-O33", |
| } |
| // copyRow is one row of the record. The File cell is the file the sentence is |
| @@ -108,7 +112,7 @@ func (r copyRow) Path() string { |
| return r.File |
| } |
| -var movesCopyIDPattern = regexp.MustCompile(`^(M-C|MV-N|HC5-N|RUN-C)[0-9]+$`) |
| +var movesCopyIDPattern = regexp.MustCompile(`^(M-C|MV-N|MV-O|HC5-N|RUN-C)[0-9]+$`) |
| // parseMovesCopyRecord reads the record's tables. A data row is a line that |
| // follows a `| id | file | text | status |` header; the separator line is |
| @@ -123,7 +127,7 @@ func parseMovesCopyRecord(path string) ([]copyRow, error) { |
| } |
| func parseMovesCopyTable(src, name string) ([]copyRow, error) { |
| - return parseCopyTable(src, name, movesCopyIDPattern, "an M-C<n>, MV-N<n>, HC5-N<n> or RUN-C<n>") |
| + return parseCopyTable(src, name, movesCopyIDPattern, "an M-C<n>, MV-N<n>, MV-O<n>, HC5-N<n> or RUN-C<n>") |
| } |
| // parseCopyTable reads a copy record's tables: the moves record, the run |
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md| index c6f42a60b..d1547ff60 100644 |
| --- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| +++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| @@ -484,3 +484,39 @@ Task 5 withdraws MV-N21 ("moves to the home region are not supported yet. the ho |
| |---|---|---|---| |
| | MV-N510 | internal/console/apiinstancemove.go | moves into the home region need host claims. this console runs without them | approved | |
| | MV-N511 | internal/console/apiinstancemove.go | <box> sets CARLOS_REGION, so it cannot take a move into the home region | approved | |
| + |
| +## Offline operator move |
| + |
| +Plan `docs/superpowers/plans/2026-10-20-offline-operator-move.md`. The rows take MV-O1 to MV-O33, each `pending` until Paul reviews them, and merge waits for that. MV-O1 to MV-O10 are `carlos move offline` output (`move.go`, Task 7); MV-O20 to MV-O31 are the console doors' refusals (`internal/console/apiinstancemove_offline.go`, Task 6); MV-O32 and MV-O33 are the two paragraphs of the docs page (Task 7). MV-O11 to MV-O19 are not used. The sentences the offline verbs share with the move already reviewed keep their ids and approved text, and only their fills change: M-C1 (Moving <host> from <from> to <to>.), M-C10 with `<from>` filled in (Waiting for <box> in <from> to confirm it has stopped <host>. <host> is offline.), M-C6, M-C7, and M-C9. `<from>` is the home's configured name, or the word `home` when none is configured. Door sentences are lower case, as the move door's are. |
| + |
| +| id | file | text | status | |
| +|---|---|---|---| |
| +| MV-O1 | move.go | <host> is offline from now until the move opens or is aborted. | pending | |
| +| MV-O2 | move.go | Every box in <from> has stopped <host>. Run carlos move offline seal --host <host> to copy it. | pending | |
| +| MV-O3 | move.go | Copying <host> to <to>. | pending | |
| +| MV-O4 | move.go | Copied and checked. Run carlos move offline flip --host <host> to start it in <to>. | pending | |
| +| MV-O5 | move.go | <to> holds everything <host> needs. Nothing is served yet. Run carlos move offline open --host <host> to serve it from <to>. | pending | |
| +| MV-O6 | move.go | <to> could not take <host>: <reason>. Run carlos move offline abort --host <host> to move it back. | pending | |
| +| MV-O7 | move.go | <host> is served from <to>. Run carlos move offline finish --host <host> once <from> has cleaned up. | pending | |
| +| MV-O8 | move.go | Moving <host> back to <from>. | pending | |
| +| MV-O9 | move.go | The move could not finish a step: <reason>. Run carlos move offline status --host <host> for each item. | pending | |
| +| MV-O10 | move.go | Waiting for <box> to clean up <host>. | pending | |
| +| MV-O20 | internal/console/apiinstancemove_offline.go | offline moves are not available while host claims or Phase 1 moves are on | pending | |
| +| MV-O21 | internal/console/apiinstancemove_offline.go | <box> cannot take part in an offline move. it runs an older release or has host claims on | pending | |
| +| MV-O22 | internal/console/apiinstancemove_offline.go | <region> has <n> boxes. an offline move needs exactly one box in the region it moves to | pending | |
| +| MV-O23 | internal/console/apiinstancemove_offline.go | <box> does not hold this deployment's secrets key | pending | |
| +| MV-O24 | internal/console/apiinstancemove_offline.go | <host> has no offline move in flight | pending | |
| +| MV-O25 | internal/console/apiinstancemove_offline.go | this move is <state>. run carlos move offline status --host <host> to see what it waits for | pending | |
| +| MV-O26 | internal/console/apiinstancemove_offline.go | <host> keeps local data on more than one box: <boxes>. remove the stale copy first | pending | |
| +| MV-O27 | internal/console/apiinstancemove_offline.go | <host> is already served from <to>. move it back with a new move | pending | |
| +| MV-O28 | internal/console/apiinstancemove_offline.go | <box> keeps a sidecar file for <host>. instances with sidecars cannot move yet | pending | |
| +| MV-O29 | internal/console/apiinstancemove_offline.go | <host> has a host claim. offline moves are only for hosts without one | pending | |
| +| MV-O30 | internal/console/apiinstancemove_offline.go | <other> is moving offline. move one instance at a time | pending | |
| +| MV-O31 | internal/console/apiinstancemove_offline.go | <box> still runs a process for <host>. wait for it to stop, then flip | pending | |
| + |
| +The docs page (`docs/site/cli.md`, under `### carlos move`) gets two paragraphs and one block of commands. The block is commands, not prose, and is not a row. The docsite gate reads it. |
| + |
| +| id | file | text | status | |
| +|---|---|---|---| |
| +| MV-O32 | docs/site/cli.md | `carlos move offline` moves one instance with planned downtime, one step at a time. It is for operators, works with host claims and Phase 1 moves off, and never acts on its own: each verb does one step and says which verb comes next. | pending | |
| +| MV-O33 | docs/site/cli.md | The instance is offline from `start` until `open`. Until `open`, `abort` puts it back where it was with nothing lost. After `open`, moving it back is a new move. | pending | |
diff --git a/internal/moves/offline.go b/internal/moves/offline.go| new file mode 100644 |
| index 000000000..c95790aaf |
| --- /dev/null |
| +++ b/internal/moves/offline.go |
| @@ -0,0 +1,316 @@ |
| +package moves |
| + |
| +import ( |
| + "fmt" |
| + "path/filepath" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/provision" |
| +) |
| + |
| +// The offline operator move's contract (docs/superpowers/plans/2026-10-20- |
| +// offline-operator-move.md, Task 1): the dependency manifest, the report each |
| +// box writes at each step, and the keys, owners and paths every task shares. |
| +// Nothing here writes anything. The record side is provision/offline_move.go. |
| + |
| +// OfflineItem is one row of the dependency list: a thing the instance needs |
| +// that the move must carry, rebuild or consciously leave behind. ID names the |
| +// item in a report's Items. Disposition says what the move does with it. |
| +// Owner says who decides it is in place: |
| +// |
| +// door a door refuses the move up front (nothing to check later) |
| +// seal the sealer's report carries its Check; flip needs it OK |
| +// landing the target's landing report carries its Check; open needs it OK |
| +// none not checked by a report; the Disposition says why |
| +type OfflineItem struct{ ID, Item, Disposition, Owner string } |
| + |
| +// The owners. |
| +const ( |
| + OfflineOwnerDoor = "door" |
| + OfflineOwnerSeal = "seal" |
| + OfflineOwnerLanding = "landing" |
| + OfflineOwnerNone = "none" |
| +) |
| + |
| +// OfflineManifest is the plan's dependency list, one row per item. It is the |
| +// checklist: the doors refuse flip and open unless every item their stage |
| +// owns is OK, a report may carry only items listed here, and |
| +// TestOfflineManifestIsCovered fails if a row has no owner. A Disposition |
| +// begins with its kind (carried, rebuilt, refused, parked, not carried); an |
| +// item no report checks (owner none) says why after the first colon. |
| +var OfflineManifest = []OfflineItem{ |
| + {"db", "SQLite DB at its final position", "carried: instances/<host>/ is mirrored to the target's moves/<host>/ and promoted by the target's first wake after open", OfflineOwnerSeal}, |
| + {"db-tail", "writes the source's stop failed to checkpoint", "carried: the sealer checkpoints first when a source report is ahead of the replica", OfflineOwnerSeal}, |
| + {"build-state", "instances/<host>/build-state.json", "carried: inside the replica prefix, after CarryRequest", OfflineOwnerSeal}, |
| + {"lease", "instances/<host>/lock.json", "not carried: the source's lock becomes the tombstone until finish; a stale tombstone of another move in the target bucket is lifted at landing", OfflineOwnerSeal}, |
| + {"files", "flat <DataDir>/<host>/ and the nested instance directory minus platform artifacts", "carried: sealed archive at moves/@offline/<host>/<id>/files.sealed in the target bucket", OfflineOwnerSeal}, |
| + {"env", "<host>.env: instance key, admin token and overrides", "carried: byte for byte inside the archive", OfflineOwnerSeal}, |
| + {"key-adoption", "<host>.instance-key-adoption.json", "not carried: written only for claim launches; a copy on the target is parked", OfflineOwnerLanding}, |
| + {"config", "<host>.config.env: the app's settings, store binding and secrets", "rebuilt: rendered on the target from the bucket bundle", OfflineOwnerLanding}, |
| + {"env-pin", "the composed launch environment", "rebuilt: app settings must be equal on both rows; box settings are not compared", OfflineOwnerLanding}, |
| + {"release", "the build the source runs", "rebuilt: not copied, ordinary adoption stages the channel's build for the ready row after open", OfflineOwnerLanding}, |
| + {"store", "object store bucket and its credentials", "not carried: the bucket stays where it is and its binding travels in config, so env-pin covers it", OfflineOwnerNone}, |
| + {"timers", "timer rows, once-timers the app armed included", "carried: a timers snapshot beside the archive, replaced on the held row", OfflineOwnerSeal}, |
| + {"next-due", "the row's NextDue", "carried: in the journal's source baselines", OfflineOwnerSeal}, |
| + {"replication", "the row's replication rung", "carried: in the journal's source baselines", OfflineOwnerSeal}, |
| + {"row", "the registry row", "rebuilt: reconcile mints it from the record, held at flat paths", OfflineOwnerLanding}, |
| + {"routing", "edges file, backhaul, certificates, derived aliases", "not carried: produced by the existing passes once the row is ready, and the runbook confirms a certificate path before the move", OfflineOwnerNone}, |
| + {"source-retire", "everything the source box keeps for the host after open", "parked: renamed, never deleted, and listed path by path in the source's parked report", OfflineOwnerNone}, |
| + {"sidecar", "<host>-sidecar.env", "refused: a source report that sees one fails the seal", OfflineOwnerSeal}, |
| + {"restore-verify", "control/restore-verify.json entry", "not carried: the next drill writes a fresh verdict, and until then the old one describes the source copy", OfflineOwnerNone}, |
| + {"warm-cache", "local DB, meta dirs, files and env a box holds for the host", "parked: renamed under .offline-moves on the source after open and on a target that already has them", OfflineOwnerLanding}, |
| + {"receipts", "move-receipts/<host>.json in the source bucket", "not carried: cleared at finish, and self-checking", OfflineOwnerNone}, |
| + {"class", "kind, addr, fleet, pool, hibernate, replica, replicas, steering, unit, seed and open run", "refused: the start door and the start transition refuse every class the move cannot carry", OfflineOwnerDoor}, |
| +} |
| + |
| +// OfflineItemOwner is the owner of manifest item id. |
| +func OfflineItemOwner(id string) (owner string, ok bool) { |
| + for _, it := range OfflineManifest { |
| + if it.ID == id { |
| + return it.Owner, true |
| + } |
| + } |
| + return "", false |
| +} |
| + |
| +// OfflineOwned lists the manifest items an owner decides, in manifest order. |
| +func OfflineOwned(owner string) []OfflineItem { |
| + var out []OfflineItem |
| + for _, it := range OfflineManifest { |
| + if it.Owner == owner { |
| + out = append(out, it) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// The roles a report names: a source-region box that stopped the host, the |
| +// one source box that seals, and the target box that lands. |
| +const ( |
| + OfflineRoleSource = "source" |
| + OfflineRoleSealer = "sealer" |
| + OfflineRoleTarget = "target" |
| +) |
| + |
| +// The phases a report names. |
| +const ( |
| + OfflinePhaseStopped = "stopped" |
| + OfflinePhaseSealed = "sealed" |
| + OfflinePhaseLanded = "landed" |
| + OfflinePhaseOpened = "opened" |
| + OfflinePhaseParked = "parked" |
| + OfflinePhaseGone = "gone" |
| + OfflinePhaseLifted = "lifted" |
| + OfflinePhaseReverted = "reverted" |
| + OfflinePhaseFailed = "failed" |
| +) |
| + |
| +// OfflineReportVersion is OfflineReport.V. |
| +const OfflineReportVersion = 1 |
| + |
| +// StagedObject is one object of the staged replica prefix: its size and ETag, |
| +// keyed in OfflineReport.Staged by the key relative to the prefix. |
| +type StagedObject struct { |
| + Size int64 `json:"size"` |
| + ETag string `json:"etag"` |
| +} |
| + |
| +// RetiredPath is one line of a source's parked report: a path the box kept |
| +// for the host, what was done to it (parked, absent or left) and why. |
| +type RetiredPath struct { |
| + Path string `json:"path"` |
| + Action string `json:"action"` |
| + Reason string `json:"reason,omitempty"` |
| +} |
| + |
| +// The actions a RetiredPath may name. |
| +const ( |
| + RetiredParked = "parked" |
| + RetiredAbsent = "absent" |
| + RetiredLeft = "left" |
| +) |
| + |
| +// OfflineReport is what a box writes at control/offline-moves/ at each step |
| +// of an offline move (OfflineReportKey). It is evidence, never a decision: the |
| +// doors read it and decide. One struct serves every role, and a role fills |
| +// only the fields it observed. |
| +type OfflineReport struct { |
| + V int `json:"v"` |
| + Op string `json:"op"` |
| + Host string `json:"host"` |
| + Box string `json:"box"` |
| + // Role is source, sealer or target; Phase is stopped, sealed, landed, |
| + // opened, parked, gone, lifted, reverted or failed. |
| + Role string `json:"role"` |
| + Phase string `json:"phase"` |
| + Items map[string]Check `json:"items,omitempty"` |
| + At time.Time `json:"at"` |
| + |
| + Held bool `json:"held"` |
| + Fenced bool `json:"fenced"` |
| + Quiesced bool `json:"quiesced"` |
| + // LocalDB: the DB file or the activator's litestream meta dir is present. |
| + LocalDB bool `json:"local_db"` |
| + Files bool `json:"files"` |
| + Env bool `json:"env"` |
| + Sidecar bool `json:"sidecar"` |
| + RowPresent bool `json:"row_present"` |
| + LocalTXID uint64 `json:"local_txid,omitempty"` |
| + |
| + FinalTXID uint64 `json:"final_txid,omitempty"` |
| + Staged map[string]StagedObject `json:"staged,omitempty"` |
| + FilesPlain string `json:"files_plain,omitempty"` |
| + FilesSealed string `json:"files_sealed,omitempty"` |
| + FilesBytes int64 `json:"files_bytes,omitempty"` |
| + FilesEntries int `json:"files_entries,omitempty"` |
| + |
| + EnvSHA256 string `json:"env_sha256,omitempty"` |
| + KeySHA256 string `json:"key_sha256,omitempty"` |
| + EnvDigest string `json:"env_digest,omitempty"` |
| + Release string `json:"release,omitempty"` |
| + TimersDigest string `json:"timers_digest,omitempty"` |
| + Replication string `json:"replication,omitempty"` |
| + NextDue int64 `json:"next_due,omitempty"` |
| + |
| + TombstoneOwner string `json:"tombstone_owner,omitempty"` |
| + TombstoneUntil time.Time `json:"tombstone_until"` |
| + |
| + Procs int `json:"procs,omitempty"` |
| + LayoutJournal bool `json:"layout_journal,omitempty"` |
| + |
| + // Retired is the parked report's inventory of what the source kept. |
| + Retired []RetiredPath `json:"retired,omitempty"` |
| +} |
| + |
| +var ( |
| + offlineRoles = map[string]bool{OfflineRoleSource: true, OfflineRoleSealer: true, OfflineRoleTarget: true} |
| + offlinePhases = map[string]bool{OfflinePhaseStopped: true, OfflinePhaseSealed: true, OfflinePhaseLanded: true, OfflinePhaseOpened: true, |
| + OfflinePhaseParked: true, OfflinePhaseGone: true, OfflinePhaseLifted: true, OfflinePhaseReverted: true, OfflinePhaseFailed: true} |
| + // The owners whose Checks a role's report may carry. The target also |
| + // re-verifies what the seal recorded, so it may carry the seal's items. |
| + offlineRoleOwners = map[string][]string{ |
| + OfflineRoleSource: {OfflineOwnerSeal}, |
| + OfflineRoleSealer: {OfflineOwnerSeal}, |
| + OfflineRoleTarget: {OfflineOwnerLanding, OfflineOwnerSeal}, |
| + } |
| +) |
| + |
| +// ValidateOfflineReport reports why a report cannot be trusted as a report: a |
| +// wrong version, an unknown role or phase, an unusable key part, or an item |
| +// the manifest does not list or that this role does not carry. A reader |
| +// calls it before it reads anything else. |
| +func ValidateOfflineReport(r OfflineReport) error { |
| + if r.V != OfflineReportVersion { |
| + return fmt.Errorf("offline report: version %d, want %d", r.V, OfflineReportVersion) |
| + } |
| + if OfflineReportKey(r.Host, r.Op, r.Role, r.Box) == "" { |
| + return fmt.Errorf("offline report: host, op, role or box is not usable") |
| + } |
| + if !offlinePhases[r.Phase] { |
| + return fmt.Errorf("offline report: unknown phase %q", r.Phase) |
| + } |
| + for id := range r.Items { |
| + owner, ok := OfflineItemOwner(id) |
| + if !ok { |
| + return fmt.Errorf("offline report: item %q is not in the manifest", id) |
| + } |
| + allowed := false |
| + for _, o := range offlineRoleOwners[r.Role] { |
| + if o == owner { |
| + allowed = true |
| + } |
| + } |
| + if !allowed { |
| + return fmt.Errorf("offline report: item %q (owner %s) is not carried by a %s report", id, owner, r.Role) |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// ItemsOK says whether the report holds an OK Check for every manifest |
| +// item the owner decides, and names the ones that are missing or not OK, in |
| +// manifest order. The doors use it: flip needs the sealer's report OK for |
| +// owner seal, open the target's for owner landing. |
| +func (r OfflineReport) ItemsOK(owner string) (ok bool, bad []string) { |
| + for _, it := range OfflineOwned(owner) { |
| + if c, found := r.Items[it.ID]; !found || !c.OK { |
| + bad = append(bad, it.ID) |
| + } |
| + } |
| + return len(bad) == 0, bad |
| +} |
| + |
| +// offlineSegment reports a string fit to be one path or key segment: not |
| +// empty, not a dot name, and made only of letters, digits and _ . - so it can |
| +// never escape the directory or prefix it is joined under. |
| +func offlineSegment(s string) bool { |
| + if s == "" || s == "." || s == ".." { |
| + return false |
| + } |
| + for _, c := range s { |
| + switch { |
| + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '_', c == '.', c == '-': |
| + default: |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| +// OfflineReportKey is where a box writes its report: |
| +// control/offline-moves/<host>/<op>/<role>/<box>.json. It is "" when host |
| +// fails provision.ValidHostShape, role is not source, sealer or target, or op |
| +// or box is not a plain name; a caller that gets "" has a bug and writes |
| +// nothing. The prefix is outside every create-only and no-delete prefix, so |
| +// a box may rewrite its own report. |
| +func OfflineReportKey(host, op, role, box string) string { |
| + if !provision.ValidHostShape(host) || !offlineSegment(op) || !offlineRoles[role] || !offlineSegment(box) { |
| + return "" |
| + } |
| + return "control/offline-moves/" + host + "/" + op + "/" + role + "/" + box + ".json" |
| +} |
| + |
| +// OfflineReportPrefix is the prefix a door lists to read every report of one |
| +// move, "" for a host that fails provision.ValidHostShape or an unusable op. |
| +func OfflineReportPrefix(host, op string) string { |
| + if !provision.ValidHostShape(host) || !offlineSegment(op) { |
| + return "" |
| + } |
| + return "control/offline-moves/" + host + "/" + op + "/" |
| +} |
| + |
| +// OfflineArchivePrefix is where the move's sealed files archive and its |
| +// timers snapshot live in the target bucket: moves/@offline/<host>/<op>/. The |
| +// "@" cannot start a host, so it never collides with a host's own staging |
| +// prefix. "" for a host that fails provision.ValidHostShape or an unusable op. |
| +func OfflineArchivePrefix(host, op string) string { |
| + if !provision.ValidHostShape(host) || !offlineSegment(op) { |
| + return "" |
| + } |
| + return "moves/@offline/" + host + "/" + op + "/" |
| +} |
| + |
| +// TombstoneOwner is the owner string of the source lease's tombstone for a |
| +// move: "moved:" + op. The lease package acts only on this owner. |
| +func TombstoneOwner(op string) string { return "moved:" + op } |
| + |
| +// TombstoneTTL is the longest a tombstone is written for. The sealer |
| +// re-asserts it every pass and finish lifts it. |
| +const TombstoneTTL = 7 * 24 * time.Hour |
| + |
| +// OfflineParkDir is where a box parks what it renames for the host: |
| +// <dataDir>/.offline-moves/<host>/<op>/<role>/. "" when dataDir is empty, |
| +// host fails provision.ValidHostShape, role is not source, sealer or target, |
| +// or op is not a plain name; a caller must not rename into "". |
| +func OfflineParkDir(dataDir, host, op, role string) string { |
| + if dataDir == "" || !provision.ValidHostShape(host) || !offlineSegment(op) || !offlineRoles[role] { |
| + return "" |
| + } |
| + return filepath.Join(dataDir, ".offline-moves", host, op, role) |
| +} |
| + |
| +// OfflineNamesOK is the validity the key helpers share, for a caller that |
| +// wants to ask before it builds anything. |
| +func OfflineNamesOK(host, op, role, box string) bool { |
| + return OfflineReportKey(host, op, role, box) != "" |
| +} |
diff --git a/internal/moves/offline_test.go b/internal/moves/offline_test.go| new file mode 100644 |
| index 000000000..d7ac75a52 |
| --- /dev/null |
| +++ b/internal/moves/offline_test.go |
| @@ -0,0 +1,335 @@ |
| +package moves |
| + |
| +import ( |
| + "encoding/json" |
| + "fmt" |
| + "reflect" |
| + "sort" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/provision" |
| +) |
| + |
| +const ( |
| + offTestHost = "a.go.example.com" |
| + offTestOp = "mv_abcdefghijklmnopqrstuvwxyz" |
| +) |
| + |
| +// planManifestIDs are the rows of the plan's "The dependency list" table, in |
| +// its order. A row added to or dropped from OfflineManifest without the plan |
| +// (and this list) fails TestOfflineManifestIsCovered. |
| +var planManifestIDs = []string{ |
| + "db", "db-tail", "build-state", "lease", "files", "env", "key-adoption", "config", "env-pin", "release", |
| + "store", "timers", "next-due", "replication", "row", "routing", "source-retire", "sidecar", "restore-verify", |
| + "warm-cache", "receipts", "class", |
| +} |
| + |
| +var offlineDispositionKinds = []string{"carried", "rebuilt", "refused", "parked", "not carried"} |
| + |
| +// validateOfflineManifest is the checklist rule: every row has an id, a thing |
| +// it names, a disposition that begins with its kind and gives its reason, and |
| +// exactly one owner; no id repeats. |
| +func validateOfflineManifest(rows []OfflineItem) error { |
| + seen := map[string]bool{} |
| + owners := map[string]bool{OfflineOwnerDoor: true, OfflineOwnerSeal: true, OfflineOwnerLanding: true, OfflineOwnerNone: true} |
| + for i, it := range rows { |
| + switch { |
| + case it.ID == "": |
| + return fmt.Errorf("row %d has no id", i) |
| + case seen[it.ID]: |
| + return fmt.Errorf("row %q is listed twice", it.ID) |
| + case it.Item == "": |
| + return fmt.Errorf("row %q names no item", it.ID) |
| + case !owners[it.Owner]: |
| + return fmt.Errorf("row %q has owner %q, which is not door, seal, landing or none", it.ID, it.Owner) |
| + } |
| + seen[it.ID] = true |
| + kind := "" |
| + for _, k := range offlineDispositionKinds { |
| + if strings.HasPrefix(it.Disposition, k+": ") && len(it.Disposition) > len(k)+2 { |
| + kind = k |
| + } |
| + } |
| + if kind == "" { |
| + return fmt.Errorf("row %q has disposition %q, which does not begin with a kind and a reason", it.ID, it.Disposition) |
| + } |
| + if strings.ContainsRune(it.Disposition, '\u2014') || strings.ContainsRune(it.Item, '\u2014') { |
| + return fmt.Errorf("row %q has an em dash", it.ID) |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +func TestOfflineManifestIsCovered(t *testing.T) { |
| + if err := validateOfflineManifest(OfflineManifest); err != nil { |
| + t.Fatal(err) |
| + } |
| + var ids []string |
| + for _, it := range OfflineManifest { |
| + ids = append(ids, it.ID) |
| + } |
| + if !reflect.DeepEqual(ids, planManifestIDs) { |
| + t.Fatalf("the manifest's rows are not the plan's:\n got %v\nwant %v", ids, planManifestIDs) |
| + } |
| + |
| + // Every row is owned by exactly one of the four: OfflineOwned partitions |
| + // the manifest. |
| + total := 0 |
| + for _, owner := range []string{OfflineOwnerDoor, OfflineOwnerSeal, OfflineOwnerLanding, OfflineOwnerNone} { |
| + rows := OfflineOwned(owner) |
| + total += len(rows) |
| + for _, it := range rows { |
| + if got, ok := OfflineItemOwner(it.ID); !ok || got != owner { |
| + t.Errorf("OfflineItemOwner(%q) = %q, %v; OfflineOwned(%q) lists it", it.ID, got, ok, owner) |
| + } |
| + } |
| + } |
| + if total != len(OfflineManifest) { |
| + t.Fatalf("the owners cover %d of %d rows", total, len(OfflineManifest)) |
| + } |
| + if len(OfflineOwned(OfflineOwnerSeal)) == 0 || len(OfflineOwned(OfflineOwnerLanding)) == 0 || len(OfflineOwned(OfflineOwnerDoor)) == 0 { |
| + t.Fatal("a stage owns nothing: the doors would have nothing to wait for") |
| + } |
| + if _, ok := OfflineItemOwner("nope"); ok { |
| + t.Fatal("an unlisted item has an owner") |
| + } |
| + // An item nothing checks must say why it is not checked. |
| + for _, it := range OfflineOwned(OfflineOwnerNone) { |
| + if !strings.Contains(it.Disposition, ": ") { |
| + t.Errorf("%s: owner none with no reason", it.ID) |
| + } |
| + } |
| +} |
| + |
| +// The checklist rule must bite: a row added with no owner, a duplicate, an |
| +// unknown owner or a disposition with no reason fails. |
| +func TestOfflineManifestRuleBites(t *testing.T) { |
| + good := OfflineItem{"extra", "something", "carried: because", OfflineOwnerSeal} |
| + if err := validateOfflineManifest(append(append([]OfflineItem(nil), OfflineManifest...), good)); err != nil { |
| + t.Fatalf("a well-formed extra row: %v", err) |
| + } |
| + for name, bad := range map[string]OfflineItem{ |
| + "no owner": {"extra", "something", "carried: because", ""}, |
| + "unknown owner": {"extra", "something", "carried: because", "operator"}, |
| + "no id": {"", "something", "carried: because", OfflineOwnerSeal}, |
| + "duplicate id": {"db", "something", "carried: because", OfflineOwnerSeal}, |
| + "no item": {"extra", "", "carried: because", OfflineOwnerSeal}, |
| + "no reason": {"extra", "something", "carried", OfflineOwnerSeal}, |
| + "empty reason": {"extra", "something", "carried: ", OfflineOwnerSeal}, |
| + "unknown kind": {"extra", "something", "forgotten: because", OfflineOwnerSeal}, |
| + "em dash in a row": {"extra", "something", "carried: because \u2014 reasons", OfflineOwnerSeal}, |
| + } { |
| + if err := validateOfflineManifest(append(append([]OfflineItem(nil), OfflineManifest...), bad)); err == nil { |
| + t.Errorf("%s: the rule accepted the row", name) |
| + } |
| + } |
| +} |
| + |
| +// A report may carry only items the manifest lists, and only the ones its |
| +// role carries. |
| +func TestOfflineReportItemsAreTheManifests(t *testing.T) { |
| + base := func(role string) OfflineReport { |
| + return OfflineReport{V: OfflineReportVersion, Op: offTestOp, Host: offTestHost, Box: "box-a", Role: role, Phase: OfflinePhaseSealed, |
| + Items: map[string]Check{}, At: time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)} |
| + } |
| + // Every owned item, in a report of the right role, validates. |
| + sealer := base(OfflineRoleSealer) |
| + for _, it := range OfflineOwned(OfflineOwnerSeal) { |
| + sealer.Items[it.ID] = Check{OK: true} |
| + } |
| + if err := ValidateOfflineReport(sealer); err != nil { |
| + t.Fatalf("a full sealer report: %v", err) |
| + } |
| + if ok, bad := sealer.ItemsOK(OfflineOwnerSeal); !ok || len(bad) != 0 { |
| + t.Fatalf("seal items: %v %v", ok, bad) |
| + } |
| + if ok, bad := sealer.ItemsOK(OfflineOwnerLanding); ok || len(bad) != len(OfflineOwned(OfflineOwnerLanding)) { |
| + t.Fatalf("a sealer report satisfies the landing's items: %v %v", ok, bad) |
| + } |
| + target := base(OfflineRoleTarget) |
| + target.Phase = OfflinePhaseLanded |
| + for _, it := range OfflineOwned(OfflineOwnerLanding) { |
| + target.Items[it.ID] = Check{OK: true} |
| + } |
| + target.Items["db"] = Check{OK: true} // the target re-verifies what the seal recorded |
| + if err := ValidateOfflineReport(target); err != nil { |
| + t.Fatalf("a full target report: %v", err) |
| + } |
| + // A not-OK item, and a missing one, are named in manifest order. |
| + target.Items["row"] = Check{OK: false, Reason: "no row"} |
| + delete(target.Items, "config") |
| + ok, bad := target.ItemsOK(OfflineOwnerLanding) |
| + if ok || !reflect.DeepEqual(bad, []string{"config", "row"}) { |
| + t.Fatalf("ItemsOK: %v %v", ok, bad) |
| + } |
| + |
| + // An item the manifest does not list fails the report. |
| + rogue := base(OfflineRoleSealer) |
| + rogue.Items["made-up"] = Check{OK: true} |
| + if err := ValidateOfflineReport(rogue); err == nil || !strings.Contains(err.Error(), "made-up") { |
| + t.Fatalf("an unlisted item: %v", err) |
| + } |
| + // An item whose owner is not the report's role. |
| + wrongRole := base(OfflineRoleSealer) |
| + wrongRole.Items["row"] = Check{OK: true} // landing's |
| + if err := ValidateOfflineReport(wrongRole); err == nil { |
| + t.Fatal("a sealer report carried a landing item") |
| + } |
| + for _, owner := range []string{OfflineOwnerDoor, OfflineOwnerNone} { |
| + for _, it := range OfflineOwned(owner) { |
| + r := base(OfflineRoleTarget) |
| + r.Items[it.ID] = Check{OK: true} |
| + if err := ValidateOfflineReport(r); err == nil { |
| + t.Errorf("a report carried %s, which is %s-owned and checked by no report", it.ID, owner) |
| + } |
| + } |
| + } |
| + // Version, phase, role and names. |
| + for name, mut := range map[string]func(*OfflineReport){ |
| + "version": func(r *OfflineReport) { r.V = 0 }, |
| + "phase": func(r *OfflineReport) { r.Phase = "dancing" }, |
| + "role": func(r *OfflineReport) { r.Role = "bystander" }, |
| + "host": func(r *OfflineReport) { r.Host = "../x" }, |
| + "no op": func(r *OfflineReport) { r.Op = "" }, |
| + "box path": func(r *OfflineReport) { r.Box = "a/b" }, |
| + } { |
| + r := base(OfflineRoleSealer) |
| + mut(&r) |
| + if err := ValidateOfflineReport(r); err == nil { |
| + t.Errorf("%s: accepted", name) |
| + } |
| + } |
| +} |
| + |
| +// Every field a report carries survives the wire. |
| +func TestOfflineReportRoundTrips(t *testing.T) { |
| + at := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + in := OfflineReport{ |
| + V: 1, Op: offTestOp, Host: offTestHost, Box: "box-a", Role: OfflineRoleSealer, Phase: OfflinePhaseSealed, |
| + Items: map[string]Check{"db": {OK: true}, "files": {OK: false, Reason: "no archive"}}, At: at, |
| + Held: true, Fenced: true, Quiesced: true, LocalDB: true, Files: true, Env: true, Sidecar: true, RowPresent: true, LocalTXID: 41, |
| + FinalTXID: 42, Staged: map[string]StagedObject{"replica/0001.ltx": {Size: 9, ETag: "e1"}}, |
| + FilesPlain: "p", FilesSealed: "s", FilesBytes: 7, FilesEntries: 2, |
| + EnvSHA256: "e", KeySHA256: "k", EnvDigest: "d", Release: "v0.28.4", TimersDigest: "t", Replication: "cold", NextDue: 99, |
| + TombstoneOwner: TombstoneOwner(offTestOp), TombstoneUntil: at.Add(TombstoneTTL), |
| + Procs: 2, LayoutJournal: true, |
| + Retired: []RetiredPath{{Path: "/data/a.go.example.com.db", Action: RetiredParked}, {Path: "/opt/carlos/live/a.go.example.com", Action: RetiredLeft, Reason: "root-owned"}}, |
| + } |
| + raw, err := json.Marshal(in) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var out OfflineReport |
| + if err := json.Unmarshal(raw, &out); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !reflect.DeepEqual(in, out) { |
| + t.Fatalf("round trip:\n in %+v\nout %+v", in, out) |
| + } |
| + // A report's booleans are always written: an absent held means not held, |
| + // and a reader can see the box said so. |
| + empty, _ := json.Marshal(OfflineReport{}) |
| + for _, k := range []string{`"held":false`, `"fenced":false`, `"quiesced":false`} { |
| + if !strings.Contains(string(empty), k) { |
| + t.Errorf("a zero report omits %s: %s", k, empty) |
| + } |
| + } |
| +} |
| + |
| +func TestOfflineKeyHelpers(t *testing.T) { |
| + if got := OfflineReportKey(offTestHost, offTestOp, OfflineRoleSealer, "box-a"); got != "control/offline-moves/"+offTestHost+"/"+offTestOp+"/sealer/box-a.json" { |
| + t.Fatalf("report key %q", got) |
| + } |
| + if got := OfflineReportPrefix(offTestHost, offTestOp); got != "control/offline-moves/"+offTestHost+"/"+offTestOp+"/" { |
| + t.Fatalf("report prefix %q", got) |
| + } |
| + if got := OfflineArchivePrefix(offTestHost, offTestOp); got != "moves/@offline/"+offTestHost+"/"+offTestOp+"/" { |
| + t.Fatalf("archive prefix %q", got) |
| + } |
| + if got := TombstoneOwner(offTestOp); got != "moved:"+offTestOp { |
| + t.Fatalf("tombstone owner %q", got) |
| + } |
| + if got := OfflineParkDir("/data", offTestHost, offTestOp, OfflineRoleSource); got != "/data/.offline-moves/"+offTestHost+"/"+offTestOp+"/source" { |
| + t.Fatalf("park dir %q", got) |
| + } |
| + if TombstoneTTL != 7*24*time.Hour { |
| + t.Fatalf("tombstone ttl %v", TombstoneTTL) |
| + } |
| + |
| + // Hosts that fail ValidHostShape are rejected by every helper that takes |
| + // one. The precondition keeps this honest if the shape rule moves. |
| + for _, host := range []string{"", "../x", "a/b.example.com", "a b.example.com", "UPPER.EXAMPLE.COM", ".leading.example.com", "a.b.example.com/", "a..b"} { |
| + if provision.ValidHostShape(host) { |
| + t.Fatalf("test precondition: %q is a valid host shape", host) |
| + } |
| + if k := OfflineReportKey(host, offTestOp, OfflineRoleSealer, "box-a"); k != "" { |
| + t.Errorf("report key for %q: %q", host, k) |
| + } |
| + if k := OfflineReportPrefix(host, offTestOp); k != "" { |
| + t.Errorf("report prefix for %q: %q", host, k) |
| + } |
| + if k := OfflineArchivePrefix(host, offTestOp); k != "" { |
| + t.Errorf("archive prefix for %q: %q", host, k) |
| + } |
| + if k := OfflineParkDir("/data", host, offTestOp, OfflineRoleSource); k != "" { |
| + t.Errorf("park dir for %q: %q", host, k) |
| + } |
| + if OfflineNamesOK(host, offTestOp, OfflineRoleSealer, "box-a") { |
| + t.Errorf("names ok for %q", host) |
| + } |
| + } |
| + // And the other parts cannot climb out of the prefix they are joined to. |
| + for _, op := range []string{"", ".", "..", "a/b", "../mv_x", "mv x"} { |
| + if OfflineReportKey(offTestHost, op, OfflineRoleSealer, "box-a") != "" || OfflineArchivePrefix(offTestHost, op) != "" || |
| + OfflineParkDir("/data", offTestHost, op, OfflineRoleSource) != "" || OfflineReportPrefix(offTestHost, op) != "" { |
| + t.Errorf("op %q accepted", op) |
| + } |
| + } |
| + for _, box := range []string{"", "..", "a/b", "box a"} { |
| + if OfflineReportKey(offTestHost, offTestOp, OfflineRoleSealer, box) != "" { |
| + t.Errorf("box %q accepted", box) |
| + } |
| + } |
| + for _, role := range []string{"", "Source", "operator", "../source"} { |
| + if OfflineReportKey(offTestHost, offTestOp, role, "box-a") != "" || OfflineParkDir("/data", offTestHost, offTestOp, role) != "" { |
| + t.Errorf("role %q accepted", role) |
| + } |
| + } |
| + if OfflineParkDir("", offTestHost, offTestOp, OfflineRoleSource) != "" { |
| + t.Error("a park dir with no data dir") |
| + } |
| +} |
| + |
| +// The report key sits outside every prefix the bucket makes create-only or |
| +// no-delete, so a box may rewrite its report. |
| +func TestOfflineReportKeyIsOutsideTheProtectedPrefixes(t *testing.T) { |
| + key := OfflineReportKey(offTestHost, offTestOp, OfflineRoleTarget, "box-a") |
| + for _, p := range []string{"control/boxes", "control/box-volumes", "control/box-enrollment", "control/box-fences"} { |
| + if strings.HasPrefix(key, p) { |
| + t.Errorf("%s is under %s", key, p) |
| + } |
| + } |
| +} |
| + |
| +// Sanity for the inventory the other tasks pin their wiring to: the exported |
| +// state constants of the record side are the ones a report's phases pair with. |
| +func TestOfflineStateAndPhaseNamesAreDistinct(t *testing.T) { |
| + states := []string{provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, |
| + provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting} |
| + sort.Strings(states) |
| + for i := 1; i < len(states); i++ { |
| + if states[i] == states[i-1] { |
| + t.Fatalf("state %q is declared twice", states[i]) |
| + } |
| + } |
| + for _, s := range states { |
| + if !strings.HasPrefix(s, "offline-") { |
| + t.Errorf("state %q lacks the offline- prefix", s) |
| + } |
| + } |
| + if provision.MoveKindOffline != "offline" { |
| + t.Fatalf("kind %q", provision.MoveKindOffline) |
| + } |
| +} |
diff --git a/internal/provision/offline_move.go b/internal/provision/offline_move.go| new file mode 100644 |
| index 000000000..238c33f76 |
| --- /dev/null |
| +++ b/internal/provision/offline_move.go |
| @@ -0,0 +1,293 @@ |
| +package provision |
| + |
| +import ( |
| + "fmt" |
| + "time" |
| +) |
| + |
| +// The offline operator move (docs/superpowers/plans/2026-10-20-offline- |
| +// operator-move.md, Task 1). One hibernating instance moves between two |
| +// regions with planned downtime: the operator stops it, the move copies a |
| +// checked list of everything it depends on, and only then does the target |
| +// open it. The record carries the move as move.kind "offline" through the |
| +// states below, each advanced by exactly one transition in this file. |
| +// |
| +// Kinds never cross. Nothing here touches a legacy or Phase 1 move (every |
| +// transition refuses a move whose kind is not offline), and no legacy or |
| +// Phase 1 transition accepts one (the legacy transitions name the states |
| +// they admit, none of which is an offline state; MoveGuard.check refuses the |
| +// kind outright). |
| +// |
| +// No code in this file Puts a record. Each transition is a mut for |
| +// SetRecordMove, so the write is provision's own read-CAS-write, which |
| +// advances record_seq. Across a whole move the record changes only in |
| +// region, move, last_move, record_seq and move.updated_at. |
| + |
| +// MoveKindOffline is MoveState.Kind for an offline operator move. Every other |
| +// move leaves Kind empty. |
| +const MoveKindOffline = "offline" |
| + |
| +// The offline move's states, in the order a move takes them: |
| +// |
| +// offline-stopping -> offline-sealing -> offline-sealed -> offline-landing |
| +// -> offline-landed -> offline-open -> offline-finishing -> (cleared) |
| +// |
| +// offline-reverting is abort's state, reached from stopping through landed. |
| +const ( |
| + OfflineStopping = "offline-stopping" |
| + OfflineSealing = "offline-sealing" |
| + OfflineSealed = "offline-sealed" |
| + OfflineLanding = "offline-landing" |
| + OfflineLanded = "offline-landed" |
| + OfflineOpen = "offline-open" |
| + OfflineFinishing = "offline-finishing" |
| + OfflineReverting = "offline-reverting" |
| +) |
| + |
| +// The outcomes OfflineFinish leaves in LastMove. |
| +const ( |
| + OfflineOutcomeDone = "done" |
| + OfflineOutcomeAborted = "aborted" |
| +) |
| + |
| +// refusedOfflineKind is MoveGuard.check's refusal of an offline move. |
| +const refusedOfflineKind = "an offline move has its own transitions" |
| + |
| +// Offline reports whether this is an offline operator move. Nil-safe, so a |
| +// record at rest answers false. |
| +func (m *MoveState) Offline() bool { return m != nil && m.Kind == MoveKindOffline } |
| + |
| +// offlineAt checks what every offline transition checks first: the record |
| +// carries an offline move, it is the move the caller means, and it is in the |
| +// one state the transition leaves. Every failure is ErrMoveConflict. |
| +func offlineAt(rec *Record, id, from string) error { |
| + if err := offlineIs(rec, id); err != nil { |
| + return err |
| + } |
| + if rec.Move.State != from { |
| + return fmt.Errorf("%w: the offline move is %s, this step follows %s", ErrMoveConflict, rec.Move.State, from) |
| + } |
| + return nil |
| +} |
| + |
| +// offlineIs is offlineAt without the state. |
| +func offlineIs(rec *Record, id string) error { |
| + if rec.Move == nil { |
| + return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNoMove) |
| + } |
| + if !rec.Move.Offline() { |
| + return fmt.Errorf("%w: the record's move is not an offline move", ErrMoveConflict) |
| + } |
| + if rec.Move.ID != id { |
| + return fmt.Errorf("%w: the record's offline move is %s", ErrMoveConflict, rec.Move.ID) |
| + } |
| + return nil |
| +} |
| + |
| +// OfflineStart begins an offline move from region `from` to region `to` |
| +// (either may be "", the home placement). It carries MoveRequest's refusals: |
| +// a seeded record, an open run, a move already in flight, the same region and |
| +// a replica. It also refuses a `from` that is not the record's region, and a |
| +// malformed id. It stamps kind, id, from, to, state, requested_by and |
| +// requested_at; it does not touch the region. |
| +func OfflineStart(id, from, to, by string, now time.Time) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := seedMoveRefusal(rec); err != nil { |
| + return err |
| + } |
| + if rec.ActiveRun.Open() { |
| + return &MoveRunOpenError{Run: *rec.ActiveRun} |
| + } |
| + if rec.Move != nil { |
| + return fmt.Errorf("%w: a move to %s is already %s", ErrMoveConflict, rec.Move.To, rec.Move.State) |
| + } |
| + if !moveIDShape.MatchString(id) || now.IsZero() { |
| + return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart) |
| + } |
| + if rec.Region == to || from != rec.Region { |
| + return fmt.Errorf("%w: the instance is already in %s", ErrMoveConflict, rec.Region) |
| + } |
| + if rec.Role != "" { |
| + return fmt.Errorf("%w: a %s does not move. delete it and create one in the new region", ErrMoveConflict, rec.Role) |
| + } |
| + rec.Move = &MoveState{ |
| + Kind: MoveKindOffline, |
| + ID: id, |
| + From: from, |
| + To: to, |
| + State: OfflineStopping, |
| + RequestedBy: by, |
| + RequestedAt: now.UTC(), |
| + } |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineSeal is stopping to sealing: every source box has confirmed the stop |
| +// and the operator asked for the copy. It names the sealer (the one box that |
| +// copies) and the source boxes whose reports the later doors wait for. |
| +func OfflineSeal(id, sealer string, boxes []string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineStopping); err != nil { |
| + return err |
| + } |
| + if sealer == "" || len(boxes) == 0 { |
| + return fmt.Errorf("%w: a seal names the sealer and the boxes that stopped the instance", ErrMoveConflict) |
| + } |
| + rec.Move.State = OfflineSealing |
| + rec.Move.Sealer = sealer |
| + rec.Move.Boxes = append([]string(nil), boxes...) |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineSealedBy is sealing to sealed: the sealer recorded what it copied. |
| +// Only the box the seal named may write it. src is copied, so the journal |
| +// never aliases the caller's value. |
| +func OfflineSealedBy(id, sealer string, src MoveSource) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineSealing); err != nil { |
| + return err |
| + } |
| + if sealer == "" || rec.Move.Sealer != sealer { |
| + return fmt.Errorf("%w: the offline move is sealed by %s", ErrMoveConflict, rec.Move.Sealer) |
| + } |
| + s := src |
| + if src.Files != nil { |
| + f := *src.Files |
| + s.Files = &f |
| + } |
| + rec.Move.Source = &s |
| + rec.Move.State = OfflineSealed |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineFlip is sealed to landing, and the one record write that moves the |
| +// region: region = to. The record's region must still be the move's source, |
| +// so a record that was changed under the move is refused rather than flipped. |
| +func OfflineFlip(id string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineSealed); err != nil { |
| + return err |
| + } |
| + if rec.Region != rec.Move.From { |
| + return fmt.Errorf("%w: the record's region is %s, not the move's source", ErrMoveConflict, rec.Region) |
| + } |
| + rec.Region = rec.Move.To |
| + rec.Move.State = OfflineLanding |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineLandedBy is landing to landed: the target box reports that every |
| +// item is in place. Guarded by the record's region, so a landing that raced an |
| +// abort (which put the region back) is refused. |
| +func OfflineLandedBy(id, to string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineLanding); err != nil { |
| + return err |
| + } |
| + if rec.Region != to || rec.Move.To != to { |
| + return fmt.Errorf("%w: the record's region is %s, not %s", ErrMoveConflict, rec.Region, to) |
| + } |
| + rec.Move.State = OfflineLanded |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineOpenTransition is landed to open: the operator opens the target. The |
| +// region must be the target's. It stamps opened_at, which is when the instance |
| +// came back and so what the finished move's offline time is counted to. |
| +// |
| +// The brief named this OfflineOpen, which is also the name of the state |
| +// constant; Go has one namespace, so the two transitions that clash with a |
| +// state (this one and OfflineFinishingTransition) carry a suffix. |
| +func OfflineOpenTransition(id string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineLanded); err != nil { |
| + return err |
| + } |
| + if rec.Region != rec.Move.To { |
| + return fmt.Errorf("%w: the record's region is %s, not the move's target", ErrMoveConflict, rec.Region) |
| + } |
| + t := time.Now().UTC() |
| + rec.Move.State, rec.Move.OpenedAt = OfflineOpen, &t |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineFinishingTransition is open to finishing: the operator asked the |
| +// source to clean up. After open the move only goes forward. |
| +func OfflineFinishingTransition(id string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineAt(rec, id, OfflineOpen); err != nil { |
| + return err |
| + } |
| + rec.Move.State = OfflineFinishing |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineAbort puts the move on its way back: any of stopping, sealing, |
| +// sealed, landing or landed becomes reverting. From landing and landed the |
| +// region goes back to the source in the same write (the flip is undone with |
| +// the state, never separately). From open and finishing it refuses, because |
| +// the target already serves; from reverting it is a no-op success, so a |
| +// second abort is not an error. |
| +func OfflineAbort(id string) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineIs(rec, id); err != nil { |
| + return err |
| + } |
| + switch rec.Move.State { |
| + case OfflineReverting: |
| + return nil |
| + case OfflineStopping, OfflineSealing, OfflineSealed: |
| + case OfflineLanding, OfflineLanded: |
| + rec.Region = rec.Move.From |
| + default: |
| + return fmt.Errorf("%w: the offline move is %s. it cannot be aborted once it is open", ErrMoveConflict, rec.Move.State) |
| + } |
| + rec.Move.State = OfflineReverting |
| + return nil |
| + } |
| +} |
| + |
| +// OfflineFinish clears the move and leaves its outcome in last_move. It is |
| +// legal from finishing (outcome done, region must be the target's) and from |
| +// reverting (outcome aborted, region must be the source's); anything else is |
| +// refused. The outcome's offline time runs from the request to the open for a |
| +// done move and to now for an aborted one. |
| +func OfflineFinish(id string, now time.Time) func(*Record) error { |
| + return func(rec *Record) error { |
| + if err := offlineIs(rec, id); err != nil { |
| + return err |
| + } |
| + m := rec.Move |
| + out := &MoveOutcome{ID: id, At: now.UTC()} |
| + end := now |
| + switch m.State { |
| + case OfflineFinishing: |
| + if rec.Region != m.To { |
| + return fmt.Errorf("%w: the record's region is %s, not the move's target", ErrMoveConflict, rec.Region) |
| + } |
| + out.Outcome = OfflineOutcomeDone |
| + if m.OpenedAt != nil { |
| + end = *m.OpenedAt |
| + } |
| + case OfflineReverting: |
| + if rec.Region != m.From { |
| + return fmt.Errorf("%w: the record's region is %s, not the move's source", ErrMoveConflict, rec.Region) |
| + } |
| + out.Outcome = OfflineOutcomeAborted |
| + default: |
| + return fmt.Errorf("%w: the offline move is %s. it finishes from finishing or reverting", ErrMoveConflict, m.State) |
| + } |
| + if s := end.Sub(m.RequestedAt).Seconds(); s > 0 { |
| + out.OfflineS = s |
| + } |
| + rec.LastMove, rec.Move = out, nil |
| + return nil |
| + } |
| +} |
diff --git a/internal/provision/offline_move_test.go b/internal/provision/offline_move_test.go| new file mode 100644 |
| index 000000000..e0da0f0cf |
| --- /dev/null |
| +++ b/internal/provision/offline_move_test.go |
| @@ -0,0 +1,761 @@ |
| +package provision |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/seeds" |
| +) |
| + |
| +const ( |
| + offID = "mv_abcdefghijklmnopqrstuvwxyz" |
| + otherOff = "mv_zyxwvutsrqponmlkjihgfedcba" |
| + offSealer = "box-a" |
| +) |
| + |
| +var offBoxes = []string{"box-a", "box-b"} |
| + |
| +// offlineRecord creates a record that carries everything an offline move must |
| +// leave alone: an acquisition marker, a pool, an environment, a pinned |
| +// channel, a hibernate flag and a steering rule. Its region is "" (home). |
| +func offlineRecord(t *testing.T, s release.Store) Record { |
| + t.Helper() |
| + hib := true |
| + rec := Record{ |
| + Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", |
| + Pool: "main", Region: "", Channel: "stable", Environment: "shop", |
| + Acquisition: "acq_0123456789abcdef", Hibernate: &hib, Steering: "nearest", |
| + CreatedAt: time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC), CreatedBy: "ops@example.com", |
| + } |
| + if err := CreateRecord(context.Background(), s, rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + return rec |
| +} |
| + |
| +// offlineStep is one transition by name, so a test can walk the path and try |
| +// every step from every state. |
| +type offlineStep struct { |
| + name string |
| + mut func() func(*Record) error |
| + from string // the one state it leaves |
| +} |
| + |
| +func offlineSteps(now time.Time) []offlineStep { |
| + src := MoveSource{Replication: "cold", NextDue: 99, FinalTXID: 42, Release: "v0.28.4", EnvDigest: "d1", Files: &MoveFilesDigest{Plain: "p", Sealed: "s", Bytes: 7, Entries: 2}} |
| + return []offlineStep{ |
| + {"start", func() func(*Record) error { return OfflineStart(offID, "", "uk", "ops@example.com", now) }, ""}, |
| + {"seal", func() func(*Record) error { return OfflineSeal(offID, offSealer, offBoxes) }, OfflineStopping}, |
| + {"sealed", func() func(*Record) error { return OfflineSealedBy(offID, offSealer, src) }, OfflineSealing}, |
| + {"flip", func() func(*Record) error { return OfflineFlip(offID) }, OfflineSealed}, |
| + {"landed", func() func(*Record) error { return OfflineLandedBy(offID, "uk") }, OfflineLanding}, |
| + {"open", func() func(*Record) error { return OfflineOpenTransition(offID) }, OfflineLanded}, |
| + {"finishing", func() func(*Record) error { return OfflineFinishingTransition(offID) }, OfflineOpen}, |
| + {"finish", func() func(*Record) error { return OfflineFinish(offID, now.Add(time.Hour)) }, OfflineFinishing}, |
| + } |
| +} |
| + |
| +func offlineWalk(t *testing.T, s release.Store, rec Record, steps []offlineStep) *Record { |
| + t.Helper() |
| + var got *Record |
| + for _, st := range steps { |
| + var err error |
| + got, err = SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| + if err != nil { |
| + t.Fatalf("%s: %v", st.name, err) |
| + } |
| + } |
| + return got |
| +} |
| + |
| +func offlineGet(t *testing.T, s release.Store, rec Record) Record { |
| + t.Helper() |
| + got, err := GetRecord(context.Background(), s, rec.Account, rec.App, rec.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return *got |
| +} |
| + |
| +// asMap is a record as the generic JSON object every other reader sees. |
| +func asMap(t *testing.T, rec Record) map[string]any { |
| + t.Helper() |
| + raw, err := json.Marshal(rec) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var m map[string]any |
| + if err := json.Unmarshal(raw, &m); err != nil { |
| + t.Fatal(err) |
| + } |
| + return m |
| +} |
| + |
| +func TestOfflineFullLegalPathThroughTheWriter(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + wantState := []string{OfflineStopping, OfflineSealing, OfflineSealed, OfflineLanding, OfflineLanded, OfflineOpen, OfflineFinishing} |
| + wantRegion := []string{"", "", "", "uk", "uk", "uk", "uk"} |
| + |
| + for i, st := range steps[:len(steps)-1] { |
| + got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| + if err != nil { |
| + t.Fatalf("%s: %v", st.name, err) |
| + } |
| + if got.Move == nil || !got.Move.Offline() || got.Move.State != wantState[i] || got.Region != wantRegion[i] { |
| + t.Fatalf("%s: move %+v region %q", st.name, got.Move, got.Region) |
| + } |
| + if got.Move.Legacy() { |
| + t.Fatalf("%s: an offline move reads as legacy", st.name) |
| + } |
| + if got.Move.UpdatedAt.IsZero() { |
| + t.Fatalf("%s: updated_at not stamped", st.name) |
| + } |
| + // What the box and the console read back is what the writer wrote. |
| + if again := offlineGet(t, s, rec); !reflect.DeepEqual(again.Move, got.Move) || again.Region != got.Region { |
| + t.Fatalf("%s: stored %+v, returned %+v", st.name, again.Move, got.Move) |
| + } |
| + } |
| + mid := offlineGet(t, s, rec) |
| + m := mid.Move |
| + if m.ID != offID || m.From != "" || m.To != "uk" || m.RequestedBy != "ops@example.com" || !m.RequestedAt.Equal(now) || |
| + m.Sealer != offSealer || !reflect.DeepEqual(m.Boxes, offBoxes) || m.Incarnation != "" || m.Cut != "" || |
| + m.Source == nil || m.Source.Release != "v0.28.4" || m.Source.EnvDigest != "d1" || m.Source.FinalTXID != 42 || |
| + m.Source.Files == nil || m.Source.Files.Entries != 2 || m.OpenedAt == nil { |
| + t.Fatalf("the journal at finishing: %+v source %+v", m, m.Source) |
| + } |
| + |
| + got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, steps[len(steps)-1].mut()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got.Move != nil || got.LastMove == nil || got.LastMove.ID != offID || got.LastMove.Outcome != "done" || !got.LastMove.At.Equal(now.Add(time.Hour)) || got.LastMove.OfflineS < 0 { |
| + t.Fatalf("finish: move %+v last %+v", got.Move, got.LastMove) |
| + } |
| + if got.Region != "uk" { |
| + t.Fatalf("finish: region %q", got.Region) |
| + } |
| +} |
| + |
| +// Review Focus 8 (§9.2): across a whole move the record differs only in |
| +// region, move, last_move, record_seq (advanced) and move.updated_at. The |
| +// record here carries an acquisition marker, a pool, an environment and the |
| +// rest of what ordinary writers must leave alone; the comparison is of the |
| +// whole JSON object, so a field a transition touched by accident shows. |
| +func TestOfflineRecordDiffersOnlyWhereTheMoveSaysSo(t *testing.T) { |
| + for _, via := range []string{"done", "aborted after the flip", "aborted before the flip"} { |
| + t.Run(via, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + // Advance record_seq into presence first, as it is on any record a |
| + // writer has touched under the sequence rule. |
| + ctx = WithRecordSequence(ctx) |
| + before := offlineGet(t, s, rec) |
| + beforeMap := asMap(t, before) |
| + |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + upto := len(steps) |
| + switch via { |
| + case "aborted after the flip": |
| + upto = 5 // through landed |
| + case "aborted before the flip": |
| + upto = 3 // through sealed |
| + } |
| + seqs := []uint64{} |
| + for _, st := range steps[:upto] { |
| + got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, st.mut()) |
| + if err != nil { |
| + t.Fatalf("%s: %v", st.name, err) |
| + } |
| + seqs = append(seqs, got.RecordSeq) |
| + } |
| + if via != "done" { |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Hour))); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + after := offlineGet(t, s, rec) |
| + if after.Move != nil || after.LastMove == nil { |
| + t.Fatalf("the move did not end: %+v %+v", after.Move, after.LastMove) |
| + } |
| + for i := 1; i < len(seqs); i++ { |
| + if seqs[i] <= seqs[i-1] { |
| + t.Fatalf("record_seq did not advance across transitions: %v", seqs) |
| + } |
| + } |
| + if after.RecordSeq <= before.RecordSeq { |
| + t.Fatalf("record_seq %d -> %d", before.RecordSeq, after.RecordSeq) |
| + } |
| + |
| + afterMap := asMap(t, after) |
| + allowed := map[string]bool{"region": true, "move": true, "last_move": true, "record_seq": true} |
| + keys := map[string]bool{} |
| + for k := range beforeMap { |
| + keys[k] = true |
| + } |
| + for k := range afterMap { |
| + keys[k] = true |
| + } |
| + for k := range keys { |
| + if reflect.DeepEqual(beforeMap[k], afterMap[k]) { |
| + continue |
| + } |
| + if !allowed[k] { |
| + t.Errorf("field %q changed: %v -> %v", k, beforeMap[k], afterMap[k]) |
| + } |
| + } |
| + if _, ok := afterMap["move"]; ok { |
| + t.Errorf("move left on the record at rest: %v", afterMap["move"]) |
| + } |
| + wantRegion := "" |
| + if via == "done" { |
| + wantRegion = "uk" |
| + } |
| + if after.Region != wantRegion { |
| + t.Errorf("region %q, want %q", after.Region, wantRegion) |
| + } |
| + for _, k := range []string{"acq", "pool", "environment", "channel", "hibernate", "steering", "created_at", "created_by", "slug"} { |
| + if _, ok := afterMap[k]; !ok { |
| + t.Errorf("field %q is gone", k) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Every transition, tried from every state but its own, is refused with |
| +// ErrMoveConflict and leaves the record as it was. |
| +func TestOfflineEveryIllegalPairIsRefused(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + // Each state is reached by walking the first n steps; reverting by an |
| + // abort from stopping. |
| + type at struct { |
| + state string |
| + walk []offlineStep |
| + abort bool |
| + } |
| + states := []at{ |
| + {OfflineStopping, steps[:1], false}, {OfflineSealing, steps[:2], false}, {OfflineSealed, steps[:3], false}, |
| + {OfflineLanding, steps[:4], false}, {OfflineLanded, steps[:5], false}, {OfflineOpen, steps[:6], false}, |
| + {OfflineFinishing, steps[:7], false}, {OfflineReverting, steps[:1], true}, |
| + } |
| + |
| + for _, at := range states { |
| + for _, st := range steps[1:] { // start has its own test |
| + legal := st.from == at.state |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, at.walk) |
| + if at.abort { |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + before := offlineGet(t, s, rec) |
| + _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| + switch { |
| + case legal && err != nil: |
| + t.Errorf("%s from %s: %v", st.name, at.state, err) |
| + case !legal && st.name == "finish" && at.state == OfflineReverting: |
| + if err != nil { |
| + t.Errorf("finish from reverting: %v", err) |
| + } |
| + case !legal && !errors.Is(err, ErrMoveConflict): |
| + t.Errorf("%s from %s: err %v, want ErrMoveConflict", st.name, at.state, err) |
| + } |
| + if err != nil { |
| + if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| + t.Errorf("%s from %s: a refusal changed the record", st.name, at.state) |
| + } |
| + } |
| + } |
| + } |
| +} |
| + |
| +func TestOfflineWrongIDIsRefusedEverywhere(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + wrong := map[string]func() func(*Record) error{ |
| + "seal": func() func(*Record) error { return OfflineSeal(otherOff, offSealer, offBoxes) }, |
| + "sealed": func() func(*Record) error { return OfflineSealedBy(otherOff, offSealer, MoveSource{}) }, |
| + "flip": func() func(*Record) error { return OfflineFlip(otherOff) }, |
| + "landed": func() func(*Record) error { return OfflineLandedBy(otherOff, "uk") }, |
| + "open": func() func(*Record) error { return OfflineOpenTransition(otherOff) }, |
| + "finishing": func() func(*Record) error { return OfflineFinishingTransition(otherOff) }, |
| + "finish": func() func(*Record) error { return OfflineFinish(otherOff, now) }, |
| + "abort": func() func(*Record) error { return OfflineAbort(otherOff) }, |
| + } |
| + // Each wrong-id step is tried in the state where its right-id twin is legal. |
| + for i, st := range steps[1:] { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:i+1]) |
| + before := offlineGet(t, s, rec) |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, wrong[st.name]()); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s with another move's id: %v", st.name, err) |
| + } |
| + if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| + t.Errorf("%s with another move's id changed the record", st.name) |
| + } |
| + } |
| + // Abort from a legal state with the wrong id, too. |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:1]) |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, wrong["abort"]()); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("abort with another move's id: %v", err) |
| + } |
| + // No move at all. |
| + s = release.NewDirStore(t.TempDir()) |
| + rec = offlineRecord(t, s) |
| + for name, mk := range wrong { |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, mk()); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s on a record with no move: %v", name, err) |
| + } |
| + } |
| +} |
| + |
| +func TestOfflineStartRefusals(t *testing.T) { |
| + ctx := context.Background() |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + |
| + start := func(t *testing.T, s release.Store, rec Record, id, from, to string) (*Record, error) { |
| + t.Helper() |
| + return SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStart(id, from, to, "ops@example.com", now)) |
| + } |
| + t.Run("stamps the head and leaves the region", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + got, err := start(t, s, rec, offID, "", "uk") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + m := got.Move |
| + if m.Kind != MoveKindOffline || m.ID != offID || m.From != "" || m.To != "uk" || m.State != OfflineStopping || m.RequestedBy != "ops@example.com" || !m.RequestedAt.Equal(now) || got.Region != "" { |
| + t.Fatalf("head %+v region %q", m, got.Region) |
| + } |
| + if m.Cut != "" || m.Incarnation != "" || m.Inputs != nil || m.Source != nil || m.CutPhase != "" || m.Sealer != "" || m.Boxes != nil { |
| + t.Fatalf("start stamped more than the head: %+v", m) |
| + } |
| + }) |
| + t.Run("in flight", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + if _, err := start(t, s, rec, offID, "", "uk"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := start(t, s, rec, otherOff, "", "us"); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("second start: %v", err) |
| + } |
| + if got := offlineGet(t, s, rec); got.Move.ID != offID || got.Move.To != "uk" { |
| + t.Fatalf("a refused start replaced the move: %+v", got.Move) |
| + } |
| + }) |
| + t.Run("a legacy or Phase 1 move in flight", func(t *testing.T) { |
| + for name, mut := range map[string]func(*Record) error{ |
| + "legacy": MoveRequest("uk", now), |
| + "phase1": MoveStartTransition(MoveStart{ID: offID, Incarnation: strings.Repeat("ab", 32), From: "", To: "uk", Cut: "auto", Now: now}), |
| + } { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + if _, err := start(t, s, rec, otherOff, "", "us"); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("%s: offline start over it: %v", name, err) |
| + } |
| + } |
| + }) |
| + t.Run("same region, wrong from, bad id", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + for name, c := range map[string][3]string{ |
| + "same region": {offID, "", ""}, |
| + "wrong from": {offID, "us", "uk"}, |
| + "short id": {"mv_short", "", "uk"}, |
| + "empty id": {"", "", "uk"}, |
| + "rollback id": {"rb_abcdefghijklmnopqrstuvwxyz", "", "uk"}, |
| + } { |
| + if _, err := start(t, s, rec, c[0], c[1], c[2]); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s: %v", name, err) |
| + } |
| + } |
| + if got := offlineGet(t, s, rec); got.Move != nil { |
| + t.Fatalf("a refused start left a journal: %+v", got.Move) |
| + } |
| + }) |
| + t.Run("replica", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + r := Record{Schema: 1, Host: "r.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "", Channel: "stable", CreatedAt: now, Role: "replica"} |
| + if err := CreateRecord(ctx, s, r); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := start(t, s, r, offID, "", "uk"); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("replica: %v", err) |
| + } |
| + }) |
| + t.Run("seed", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + // A seeded record that is not placed: the writer under test refuses |
| + // before any journal exists, as MoveRequest does. |
| + unplaced := rec |
| + unplaced.Seed = &SeedSource{ID: "sd_1", Kind: seeds.KindReplica, State: seeds.StateReady, By: "ops@example.com", At: now} |
| + err := OfflineStart(offID, "", "uk", "ops@example.com", now)(&unplaced) |
| + var refusal *SeedMoveError |
| + if !errors.Is(err, ErrMoveConflict) || !errors.As(err, &refusal) { |
| + t.Fatalf("seed: %v", err) |
| + } |
| + if unplaced.Move != nil { |
| + t.Fatal("a refused start left a journal") |
| + } |
| + }) |
| + t.Run("open run", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + running := rec |
| + running.ActiveRun = &RunState{ID: "run_1", State: RunPicked, Deadline: now.Add(time.Hour)} |
| + err := OfflineStart(offID, "", "uk", "ops@example.com", now)(&running) |
| + var open *MoveRunOpenError |
| + if !errors.As(err, &open) { |
| + t.Fatalf("open run: %v", err) |
| + } |
| + if running.Move != nil { |
| + t.Fatal("a refused start left a journal") |
| + } |
| + }) |
| +} |
| + |
| +func TestOfflineSealAndSealedByChecks(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + ctx := context.Background() |
| + |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:1]) |
| + for name, mut := range map[string]func(*Record) error{ |
| + "no sealer": OfflineSeal(offID, "", offBoxes), |
| + "no boxes": OfflineSeal(offID, offSealer, nil), |
| + } { |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s: %v", name, err) |
| + } |
| + } |
| + // The seal copies the caller's slice. |
| + boxes := []string{"box-a", "box-b"} |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSeal(offID, offSealer, boxes)); err != nil { |
| + t.Fatal(err) |
| + } |
| + boxes[0] = "changed" |
| + if got := offlineGet(t, s, rec); got.Move.Boxes[0] != "box-a" { |
| + t.Fatalf("the journal aliases the caller's slice: %v", got.Move.Boxes) |
| + } |
| + |
| + // Only the named sealer records what it sealed, and the source is copied. |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, "box-b", MoveSource{})); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("another box sealing: %v", err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, "", MoveSource{})); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("no sealer: %v", err) |
| + } |
| + files := &MoveFilesDigest{Plain: "p", Bytes: 1} |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, offSealer, MoveSource{Files: files, Release: "v1"})); err != nil { |
| + t.Fatal(err) |
| + } |
| + files.Plain = "changed" |
| + if got := offlineGet(t, s, rec); got.Move.Source.Files.Plain != "p" || got.Move.Source.Release != "v1" { |
| + t.Fatalf("the journal aliases the caller's source: %+v", got.Move.Source) |
| + } |
| +} |
| + |
| +func TestOfflineFlipAndLandedGuardTheRegion(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + ctx := context.Background() |
| + |
| + // A record whose region moved under the move is not flipped. |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:3]) |
| + if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFlip(offID)); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("flip over a changed region: %v", err) |
| + } |
| + |
| + // landed is guarded by the record's region == to. |
| + s = release.NewDirStore(t.TempDir()) |
| + rec = offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:4]) // through flip: region uk, landing |
| + for _, to := range []string{"", "us"} { |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineLandedBy(offID, to)); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("landed by %q: %v", to, err) |
| + } |
| + } |
| + // An abort puts the region back; a landing that raced it is refused. |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineLandedBy(offID, "uk")); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("landed after abort: %v", err) |
| + } |
| +} |
| + |
| +func TestOfflineAbort(t *testing.T) { |
| + ctx := context.Background() |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + |
| + // Legal from stopping (1 step walked) through landed (5), region back |
| + // from landing and landed only. |
| + for walked, c := range map[int]struct { |
| + state string |
| + region string |
| + }{ |
| + 1: {OfflineStopping, ""}, 2: {OfflineSealing, ""}, 3: {OfflineSealed, ""}, |
| + 4: {OfflineLanding, "uk"}, 5: {OfflineLanded, "uk"}, |
| + } { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + got := offlineWalk(t, s, rec, steps[:walked]) |
| + if got.Move.State != c.state || got.Region != c.region { |
| + t.Fatalf("setup %s: %s %q", c.state, got.Move.State, got.Region) |
| + } |
| + got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)) |
| + if err != nil { |
| + t.Fatalf("abort from %s: %v", c.state, err) |
| + } |
| + if got.Move.State != OfflineReverting || got.Region != "" || !got.Move.Offline() || got.Move.ID != offID { |
| + t.Fatalf("abort from %s: %+v region %q", c.state, got.Move, got.Region) |
| + } |
| + // A second abort is a no-op success. |
| + again, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)) |
| + if err != nil || again.Move.State != OfflineReverting || again.Region != "" { |
| + t.Fatalf("second abort from %s: %v %+v", c.state, err, again) |
| + } |
| + // The way back to rest is finish. |
| + done, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Minute))) |
| + if err != nil || done.Move != nil || done.LastMove == nil || done.LastMove.Outcome != "aborted" || done.LastMove.ID != offID || done.Region != "" { |
| + t.Fatalf("finish after abort from %s: %v %+v", c.state, err, done) |
| + } |
| + } |
| + |
| + // Refused from open and finishing, and nothing changes. |
| + for walked, state := range map[int]string{6: OfflineOpen, 7: OfflineFinishing} { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:walked]) |
| + before := offlineGet(t, s, rec) |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("abort from %s: %v", state, err) |
| + } |
| + if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| + t.Fatalf("abort from %s changed the record", state) |
| + } |
| + } |
| +} |
| + |
| +func TestOfflineFinishFromBothAndTheRegionGuard(t *testing.T) { |
| + ctx := context.Background() |
| + // The request was 90 minutes ago; open stamps opened_at from the clock. |
| + now := time.Now().UTC().Add(-90 * time.Minute) |
| + steps := offlineSteps(now) |
| + |
| + // Done: offline time runs from the request to the open. |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + got := offlineWalk(t, s, rec, steps) |
| + if got.LastMove.Outcome != "done" { |
| + t.Fatalf("outcome %q", got.LastMove.Outcome) |
| + } |
| + if got.LastMove.OfflineS < 5400 || got.LastMove.OfflineS > 5400+120 { |
| + t.Fatalf("offline_s %v, want about 5400", got.LastMove.OfflineS) |
| + } |
| + |
| + // finish refuses a region that is not where the outcome says. |
| + s = release.NewDirStore(t.TempDir()) |
| + rec = offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:7]) |
| + if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("done finish with the wrong region: %v", err) |
| + } |
| + s = release.NewDirStore(t.TempDir()) |
| + rec = offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:5]) |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "uk"; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| + t.Fatalf("aborted finish with the target's region: %v", err) |
| + } |
| +} |
| + |
| +// A record written before this change must parse and re-encode byte for byte. |
| +// The bytes are written out by hand, in the order the struct has always |
| +// encoded them: a record at rest, one with a legacy journal, and one with a |
| +// Phase 1 journal carrying the source baselines. |
| +func TestOfflineFieldsAreAdditiveOnTheWire(t *testing.T) { |
| + const atRest = `{"record_seq":4,"acq":"acq_0123456789abcdef","schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","pool":"main","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","created_by":"ops@example.com","last_move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","outcome":"done","at":"2026-10-06T10:00:00Z","offline_s":61.5},"hibernate":true,"environment":"shop","steering":"nearest"}` |
| + const legacy = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"to":"uk","state":"ready","requested_at":"2026-10-06T09:00:00Z","ready_at":"2026-10-06T09:01:00Z","updated_at":"2026-10-06T09:01:00Z"}}` |
| + const phase1 = `{"record_seq":9,"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","incarnation":"abababababababababababababababababababababababababababababababab","from":"ie","to":"uk","cut":"auto","state":"cutting","requested_by":"ops@example.com","requested_at":"2026-10-06T09:00:00Z","updated_at":"2026-10-06T09:05:00Z","cut_phase":"stopped","cut_at":"2026-10-06T09:04:00Z","cut_budget_s":600,"source":{"replication":"cold","next_due":99,"final_txid":42,"db_sha256":"abc","files":{"plain":"p","sealed":"s","bytes":7,"entries":2},"timers_sha256":"t","evidence":"e"}}}` |
| + for name, raw := range map[string]string{"at rest": atRest, "legacy move": legacy, "phase 1 move": phase1} { |
| + rec, err := DecodeRecord([]byte(raw)) |
| + if err != nil { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + if rec.Move.Offline() { |
| + t.Fatalf("%s: a move written before offline moves reads as offline", name) |
| + } |
| + if rec.Move != nil && (rec.Move.Kind != "" || rec.Move.Sealer != "" || rec.Move.Boxes != nil) { |
| + t.Fatalf("%s: the new fields are not empty: %+v", name, rec.Move) |
| + } |
| + if rec.Move != nil && rec.Move.Source != nil && (rec.Move.Source.Release != "" || rec.Move.Source.EnvDigest != "") { |
| + t.Fatalf("%s: the new source fields are not empty", name) |
| + } |
| + out, err := json.Marshal(rec) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if string(out) != raw { |
| + t.Errorf("%s did not round-trip byte for byte:\n got %s\nwant %s", name, out, raw) |
| + } |
| + } |
| + |
| + // And an offline journal round-trips too, with its fields in the journal. |
| + const offline = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"uk","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","to":"uk","state":"offline-landing","requested_by":"ops@example.com","requested_at":"2026-10-21T09:00:00Z","updated_at":"2026-10-21T09:05:00Z","source":{"final_txid":42,"release":"v0.28.4","env_digest":"d1"},"kind":"offline","sealer":"box-a","boxes":["box-a","box-b"]}}` |
| + rec, err := DecodeRecord([]byte(offline)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !rec.Move.Offline() || rec.Move.Sealer != "box-a" || len(rec.Move.Boxes) != 2 || rec.Move.Source.Release != "v0.28.4" { |
| + t.Fatalf("offline journal: %+v", rec.Move) |
| + } |
| + if out, _ := json.Marshal(rec); string(out) != offline { |
| + t.Errorf("offline journal did not round-trip:\n got %s\nwant %s", out, offline) |
| + } |
| +} |
| + |
| +// Kinds never cross: no legacy and no Phase 1 transition accepts an offline |
| +// move, in any of its states, and none changes the record. |
| +func TestEveryLegacyAndPhase1TransitionRefusesAnOfflineMove(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + steps := offlineSteps(now) |
| + // An empty incarnation and the offline move's own id: the guard that would |
| + // match if the kind were not checked. |
| + g := MoveGuard{ID: offID} |
| + gInc := MoveGuard{ID: offID, Incarnation: strings.Repeat("ab", 32)} |
| + in := MoveInputs{} |
| + refused := map[string]func() func(*Record) error{ |
| + "MoveRequest": func() func(*Record) error { return MoveRequest("us", now) }, |
| + "MoveStartTransition": func() func(*Record) error { |
| + return MoveStartTransition(MoveStart{ID: otherOff, Incarnation: strings.Repeat("ab", 32), From: "", To: "us", Cut: "auto", Now: now}) |
| + }, |
| + "MoveAbort": func() func(*Record) error { return MoveAbort() }, |
| + "MoveMarkCutting": func() func(*Record) error { return MoveMarkCutting() }, |
| + "MoveComplete": func() func(*Record) error { return MoveComplete() }, |
| + "MoveAbortComplete": func() func(*Record) error { return MoveAbortComplete() }, |
| + "MovePromoteReady": func() func(*Record) error { return MovePromoteReady(now) }, |
| + "MoveGuardedPromoteReady": func() func(*Record) error { return MoveGuardedPromoteReady(g, now, in) }, |
| + "MoveGuardedCut": func() func(*Record) error { return MoveGuardedCut(g) }, |
| + "MoveGuardedRepin": func() func(*Record) error { return MoveGuardedRepin(g, in) }, |
| + "MoveGuardedAbort": func() func(*Record) error { return MoveGuardedAbort(g) }, |
| + "MoveGuardedAbortComplete": func() func(*Record) error { return MoveGuardedAbortComplete(g, now) }, |
| + "MoveAdmit": func() func(*Record) error { return MoveAdmit(g, now, 600, now.Add(time.Hour), 3) }, |
| + "MoveBackToReady": func() func(*Record) error { return MoveBackToReady(g) }, |
| + "MoveRecordSource": func() func(*Record) error { return MoveRecordSource(g, MoveSource{}) }, |
| + "MoveFlip": func() func(*Record) error { return MoveFlip(g, now, now.Add(time.Hour)) }, |
| + "MoveCommit": func() func(*Record) error { return MoveCommit(g, now) }, |
| + "MoveClose": func() func(*Record) error { return MoveClose(g, now) }, |
| + "MoveRevert": func() func(*Record) error { return MoveRevert(g, "r") }, |
| + "MoveMarkStuck": func() func(*Record) error { return MoveMarkStuck(g, "r") }, |
| + "MoveRetry": func() func(*Record) error { return MoveRetry(g) }, |
| + "MoveRollBack": func() func(*Record) error { return MoveRollBack(g, "rb_abcdefghijklmnopqrstuvwxyz", "r", now) }, |
| + "guarded, real incarnation": func() func(*Record) error { return MoveGuardedAbort(gInc) }, |
| + } |
| + // Walk one record through every offline state and try everything on it. |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + check := func(label string) { |
| + t.Helper() |
| + before := offlineGet(t, s, rec) |
| + for name, mk := range refused { |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, mk()); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s: %s: err %v, want ErrMoveConflict", label, name, err) |
| + } |
| + } |
| + if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| + t.Errorf("%s: a refused transition changed the record: %+v -> %+v", label, before.Move, after.Move) |
| + } |
| + } |
| + for _, st := range steps[:len(steps)-1] { |
| + offlineWalk(t, s, rec, []offlineStep{st}) |
| + check(st.name) |
| + } |
| + // The reverting state, reached by abort from sealed. |
| + s = release.NewDirStore(t.TempDir()) |
| + rec = offlineRecord(t, s) |
| + offlineWalk(t, s, rec, steps[:3]) |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + check("reverting") |
| +} |
| + |
| +// And the other way: no offline transition accepts a legacy or Phase 1 move. |
| +func TestOfflineTransitionsRefuseOtherKinds(t *testing.T) { |
| + now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| + for name, start := range map[string]func(*Record) error{ |
| + "legacy": MoveRequest("uk", now), |
| + "phase1": MoveStartTransition(MoveStart{ID: offID, Incarnation: strings.Repeat("ab", 32), From: "", To: "uk", Cut: "auto", Now: now}), |
| + } { |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := offlineRecord(t, s) |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, start); err != nil { |
| + t.Fatal(err) |
| + } |
| + before := offlineGet(t, s, rec) |
| + for _, st := range offlineSteps(now)[1:] { |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s move: offline %s: %v", name, st.name, err) |
| + } |
| + } |
| + if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); !errors.Is(err, ErrMoveConflict) { |
| + t.Errorf("%s move: offline abort: %v", name, err) |
| + } |
| + if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| + t.Errorf("%s move: an offline transition changed it", name) |
| + } |
| + } |
| +} |
| + |
| +func TestMoveStateOfflineIsNilSafe(t *testing.T) { |
| + var m *MoveState |
| + if m.Offline() { |
| + t.Fatal("nil move is offline") |
| + } |
| + if (&MoveState{}).Offline() || (&MoveState{Kind: "other"}).Offline() || !(&MoveState{Kind: MoveKindOffline}).Offline() { |
| + t.Fatal("Offline reads the wrong field") |
| + } |
| +} |
diff --git a/internal/provision/provision.go b/internal/provision/provision.go| index eddeb22ba..91155922b 100644 |
| --- a/internal/provision/provision.go |
| +++ b/internal/provision/provision.go |
| @@ -1051,6 +1051,17 @@ type MoveState struct { |
| // Relocation is the durable pre-flip conversion decision |
| // (move_relocation.go). It never installs anything. |
| Relocation *MoveRelocationDecision `json:"relocation,omitempty"` |
| + |
| + // Kind, Sealer and Boxes belong to the offline operator move |
| + // (offline_move.go). All three are additive and omitempty: a journal |
| + // written before them parses and re-encodes byte for byte. Kind is |
| + // "offline" for that move and empty for every other. |
| + Kind string `json:"kind,omitempty"` |
| + // Sealer is the one box that seals the move, named at seal. |
| + Sealer string `json:"sealer,omitempty"` |
| + // Boxes are the source-region boxes that had to confirm the stop, named |
| + // at seal. |
| + Boxes []string `json:"boxes,omitempty"` |
| } |
| // MoveInputs (what the target was checked against, §7.3 T3) is NOT |
| @@ -1067,6 +1078,11 @@ type MoveSource struct { |
| Files *MoveFilesDigest `json:"files,omitempty"` |
| TimersSHA256 string `json:"timers_sha256,omitempty"` |
| Evidence string `json:"evidence,omitempty"` |
| + // Release and EnvDigest are the offline move's: the build the source |
| + // ran and the portable digest of its launch environment, recorded at |
| + // seal. Additive and omitempty. |
| + Release string `json:"release,omitempty"` |
| + EnvDigest string `json:"env_digest,omitempty"` |
| } |
| type MoveFilesDigest struct { |
| @@ -1335,6 +1351,11 @@ func (g MoveGuard) check(rec *Record) error { |
| if rec.Move == nil || rec.Move.Legacy() { |
| return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNoJournaledMove) |
| } |
| + // An offline move has its own transitions and no incarnation. A guard |
| + // with an empty incarnation must not reach it: kinds never cross. |
| + if rec.Move.Offline() { |
| + return fmt.Errorf("%w: %s", ErrMoveConflict, refusedOfflineKind) |
| + } |
| if rec.Move.ID != g.ID || rec.Move.Incarnation != g.Incarnation { |
| return fmt.Errorf("%w: %s", ErrMoveConflict, fmt.Sprintf("the record's move is %s for incarnation %s", rec.Move.ID, rec.Move.Incarnation)) |
| } |