amadan
host: root opens the relocation attempt journal beneath the data root, a later attempt is never empty, a mismatched or unreadable release is a claim defect found before an attempt is counted, and every relocation guard has a test that fails without it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
12 files changed,
+999
−163
internal/host/claimowners.go+6 −0internal/host/movesprobeowners.go+9 −1internal/host/relocationassembly.go+7 −0internal/host/relocationassembly_test.go+1 −1internal/host/relocationattempts.go+64 −2internal/host/relocationattempts_linux_test.go+258 −0internal/host/relocationattempts_test.go+18 −0internal/host/relocationowners.go+56 −26internal/host/relocationowners_test.go+458 −55internal/host/relocationwake.go+33 −70internal/hostrelocate/participant.go+38 −8internal/hostrelocate/participant_gate_test.go+51 −0
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 307.699s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.850s
ok github.com/carlosframework/platform/internal/activator 138.625s
ok github.com/carlosframework/platform/internal/agentsock 0.014s
ok github.com/carlosframework/platform/internal/alerting 0.004s
ok github.com/carlosframework/platform/internal/analytics 0.018s
ok github.com/carlosframework/platform/internal/appconfig 2.149s
ok github.com/carlosframework/platform/internal/apperr 0.076s
ok github.com/carlosframework/platform/internal/applog 0.130s
ok github.com/carlosframework/platform/internal/awsinv 0.005s
ok github.com/carlosframework/platform/internal/boxchoice 0.005s
ok github.com/carlosframework/platform/internal/boxid 0.560s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.011s
ok github.com/carlosframework/platform/internal/certwarm 0.003s
ok github.com/carlosframework/platform/internal/cgroup 0.188s
ok github.com/carlosframework/platform/internal/changelog 0.250s
ok github.com/carlosframework/platform/internal/clamd 0.054s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 316.446s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.220s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.010s
ok github.com/carlosframework/platform/internal/dbverify 0.004s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.004s
ok github.com/carlosframework/platform/internal/domains 1.966s
ok github.com/carlosframework/platform/internal/doorkey 0.009s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 25.561s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.007s
ok github.com/carlosframework/platform/internal/fleetapi 0.134s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.017s
ok github.com/carlosframework/platform/internal/geo 0.020s
ok github.com/carlosframework/platform/internal/geoip 0.033s
ok github.com/carlosframework/platform/internal/host 47.660s
ok github.com/carlosframework/platform/internal/hostactors 0.305s
ok github.com/carlosframework/platform/internal/hostbox 1.315s
ok github.com/carlosframework/platform/internal/hostclaim 0.393s
ok github.com/carlosframework/platform/internal/hostevidence 0.020s
ok github.com/carlosframework/platform/internal/hostexec 0.311s
ok github.com/carlosframework/platform/internal/hostfence 0.042s
ok github.com/carlosframework/platform/internal/hostfiles 4.090s
ok github.com/carlosframework/platform/internal/hostfloor 0.021s
ok github.com/carlosframework/platform/internal/hostguard 0.159s
ok github.com/carlosframework/platform/internal/hostops 1.257s
ok github.com/carlosframework/platform/internal/hostquarantine 0.003s
ok github.com/carlosframework/platform/internal/hostrelocate 0.402s
ok github.com/carlosframework/platform/internal/hostresolve 0.135s
ok github.com/carlosframework/platform/internal/hostwrite 0.002s
ok github.com/carlosframework/platform/internal/infraview 0.002s
ok github.com/carlosframework/platform/internal/instancectl 1.698s
ok github.com/carlosframework/platform/internal/instanceexec 2.681s
ok github.com/carlosframework/platform/internal/instancefs 0.004s
ok github.com/carlosframework/platform/internal/instancekey 0.002s
ok github.com/carlosframework/platform/internal/layout 1.288s
ok github.com/carlosframework/platform/internal/lease 9.430s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.006s
ok github.com/carlosframework/platform/internal/ledger 0.014s
ok github.com/carlosframework/platform/internal/mail 0.155s
ok github.com/carlosframework/platform/internal/maintenance 0.077s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.491s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.070s
ok github.com/carlosframework/platform/internal/metrics 0.008s
ok github.com/carlosframework/platform/internal/moves 20.860s
ok github.com/carlosframework/platform/internal/moves/archive 0.725s
ok github.com/carlosframework/platform/internal/objstore 0.153s
ok github.com/carlosframework/platform/internal/ops 0.154s
ok github.com/carlosframework/platform/internal/park 0.172s
ok github.com/carlosframework/platform/internal/privatefile 0.004s
2026/10/12 00:14:21 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/12 00:14:21 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/12 00:14:21 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:21 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/12 00:14:21 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/12 00:14:21 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/12 00:14:21 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/12 00:14:21 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/12 00:14:21 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/12 00:14:21 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/12 00:14:21 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/12 00:14:21 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/12 00:14:21 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/12 00:14:21 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/12 00:14:21 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/12 00:14:21 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/12 00:14:21 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:21 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:21 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/12 00:14:21 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/12 00:14:21 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/12 00:14:21 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/12 00:14:21 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/12 00:14:22 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/12 00:14:22 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:22 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.42s)
plan9_b8_census_test.go:371: record-write census rejected 7 site(s); update the explicit inventory only after reviewing the source:
changed site internal/host/claimlaunch_production.go:claimLaunchFor: source differs from the reviewed digest
changed site internal/hostops/operations.go:Controller.Attempt: source differs from the reviewed digest
changed site internal/provision/reconcile.go:Reconcile: source differs from the reviewed digest
changed site internal/provision/reconcile.go:mintHeld: source differs from the reviewed digest
unlisted site internal/host/relocationbuild.go:authorityStore.CopyVersion [def:CopyVersion,sel:CopyVersion]: review it, classify it, and update its explicit inventory entry
unlisted site internal/host/relocationbuild.go:authorityStore.PutIfAbsent [def:PutIfAbsent,sel:PutIfAbsent]: review it, classify it, and update its explicit inventory entry
unlisted site internal/host/relocationinstall.go:relocationRecord [key:RecordKey]: review it, classify it, and update its explicit inventory entry
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io channel stable -> edge
2026/10/12 00:14:22 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/12 00:14:22 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/12 00:14:22 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/12 00:14:22 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io hibernate false -> true
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:22 provision: converged acme.go.tito.io status-named false -> true
2026/10/12 00:14:22 provision: converged acme.go.tito.io status-named true -> false
2026/10/12 00:14:22 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/12 00:14:23 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/12 00:14:23 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/12 00:14:23 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/12 00:14:23 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/12 00:14:23 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/12 00:14:23 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/12 00:14:23 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/12 00:14:23 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/12 00:14:23 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/12 00:14:23 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/12 00:14:23 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/12 00:14:23 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/12 00:14:23 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:23 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/12 00:14:23 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/12 00:14:23 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/12 00:14:23 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/12 00:14:23 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/12 00:14:23 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: converged acme.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/12 00:14:23 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/12 00:14:23 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/12 00:14:23 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/12 00:14:23 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/12 00:14:23 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/12 00:14:23 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/12 00:14:23 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/12 00:14:23 provision: dry-run: would converge owned.go.tito.io seed
2026/10/12 00:14:23 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/12 00:14:23 provision: converged owned.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/12 00:14:23 provision: converged owned.go.tito.io hibernate false -> true
2026/10/12 00:14:23 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/12 00:14:23 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: removed retired static route site.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/12 00:14:23 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/12 00:14:23 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/12 00:14:23 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/12 00:14:23 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/12 00:14:23 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/12 00:14:23 provision: converged acme.go.tito.io channel stable -> edge
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/12 00:14:23 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/12 00:14:23 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/12 00:14:23 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/12 00:14:23 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/12 00:14:23 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/12 00:14:23 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/12 00:14:23 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/12 00:14:23 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/12 00:14:23 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/12 00:14:23 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/12 00:14:24 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/12 00:14:24 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/12 00:14:24 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/12 00:14:24 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/12 00:14:24 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/12 00:14:24 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/12 00:14:24 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/12 00:14:24 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/12 00:14:24 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/12 00:14:24 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/12 00:14:24 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/12 00:14:24 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/12 00:14:24 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/12 00:14:24 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/12 00:14:24 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/12 00:14:24 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/12 00:14:24 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/12 00:14:24 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/12 00:14:24 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/12 00:14:24 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
FAIL
FAIL github.com/carlosframework/platform/internal/provision 3.005s
ok github.com/carlosframework/platform/internal/publicrelease 0.006s
ok github.com/carlosframework/platform/internal/registry 3.509s
ok github.com/carlosframework/platform/internal/relay 7.224s
ok github.com/carlosframework/platform/internal/release 18.125s
ok github.com/carlosframework/platform/internal/rollout 1.465s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.003s
ok github.com/carlosframework/platform/internal/schedule 2.172s
ok github.com/carlosframework/platform/internal/sim/hostlife 2.337s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 3.744s
ok github.com/carlosframework/platform/internal/status 0.037s
ok github.com/carlosframework/platform/internal/steering 0.005s
ok github.com/carlosframework/platform/internal/stopevidence 0.133s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.002s
ok github.com/carlosframework/platform/internal/updatewords 0.002s
ok github.com/carlosframework/platform/internal/usage 0.004s
ok github.com/carlosframework/platform/internal/version 0.001s
ok github.com/carlosframework/platform/internal/vet 1.539s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/host/claimowners.go b/internal/host/claimowners.go| index c93b8d0c7..834216981 100644 |
| --- a/internal/host/claimowners.go |
| +++ b/internal/host/claimowners.go |
| @@ -151,6 +151,12 @@ type ClaimServingSources struct { |
| Launch func(context.Context, registry.Route) (ClaimServingLaunch, error) |
| Markers ClaimMarkerStore |
| Files ClaimFilesManifest |
| + // Identity verifies this box's own identity (the roster entry and its |
| + // evidence, hostbox.IdentityProvider.Verify) before every data-bucket |
| + // request region HA's relocation inputs make, as main's serving owners |
| + // do through their RequestStore's Check (claimconstruction.go). Only |
| + // the relocation source requires it; main binds its own. |
| + Identity func(context.Context) error |
| } |
| // claimMovesC1 decides whether the moves C1 integration (Group D) is bound. |
diff --git a/internal/host/movesprobeowners.go b/internal/host/movesprobeowners.go| index 3ea0cf25e..70d3cbca0 100644 |
| --- a/internal/host/movesprobeowners.go |
| +++ b/internal/host/movesprobeowners.go |
| @@ -402,8 +402,16 @@ func markerKey(h *hostclaim.Holder) string { |
| // currentMarker is the location's current generation.json version, or nil. |
| func (m *movesProbeOwners) currentMarker(ctx context.Context, h *hostclaim.Holder) (*claimMarkerVersion, error) { |
| + return currentMarker(ctx, m.Data, m.Bucket, h) |
| +} |
| + |
| +// currentMarker is the location's current generation.json version, or |
| +// nil when the key has none; a delete marker, or two latest versions, is |
| +// ErrChanged. The moves probe owners and region HA's relocation inputs |
| +// share it (8d fix round 1, M3). |
| +func currentMarker(ctx context.Context, data genselect.Reader, bucket string, h *hostclaim.Holder) (*claimMarkerVersion, error) { |
| key := markerKey(h) |
| - versions, err := m.Data.ListVersions(ctx, m.Bucket, key) |
| + versions, err := data.ListVersions(ctx, bucket, key) |
| if err != nil { |
| return nil, err |
| } |
diff --git a/internal/host/relocationassembly.go b/internal/host/relocationassembly.go| index 9db05306f..6c8c4594d 100644 |
| --- a/internal/host/relocationassembly.go |
| +++ b/internal/host/relocationassembly.go |
| @@ -150,6 +150,13 @@ func (a ClaimAssembly) relocationPreflight(cfg RelocationAssemblyConfig) func(co |
| if hostclaim.PreparedOpened(o.Claim) && !cfg.LaterProbeAttempts { |
| return hostrelocate.ErrLaterAttemptUnlaunchable |
| } |
| + // A defect in the claim the attempt would be made over (the |
| + // release, the prepared authority, the pin, probing_since) is |
| + // deterministic: it is found here, before the attempt is counted, |
| + // never after three counted refusals (review M1). |
| + if _, err := relocationProbeClaim(o.Claim); err != nil { |
| + return err |
| + } |
| if err := a.Wake.Roles.Preflight(ctx, ClaimLaunchInputs{}); err != nil { |
| return err |
| } |
diff --git a/internal/host/relocationassembly_test.go b/internal/host/relocationassembly_test.go| index c0fd96c5b..07da054e9 100644 |
| --- a/internal/host/relocationassembly_test.go |
| +++ b/internal/host/relocationassembly_test.go |
| @@ -121,7 +121,7 @@ func TestRelocationPreflightAsksRootForTheRelocatingKind(t *testing.T) { |
| ctx := context.Background() |
| refused := errors.New("root: no attempt reader for relocating") |
| probing := func(opened bool) hostclaim.Observed { |
| - j := map[string]string{"prepared": "P1"} |
| + j := map[string]string{"prepared": "P1", "prepared_authority": `{"generation":"P1"}`, "pin": `{"empty":true}`, "probing_since": "2026-10-07T02:00:00Z"} |
| if opened { |
| j["prepared_opened"] = "P1" |
| } |
diff --git a/internal/host/relocationattempts.go b/internal/host/relocationattempts.go| index 9f142057c..77fe61b05 100644 |
| --- a/internal/host/relocationattempts.go |
| +++ b/internal/host/relocationattempts.go |
| @@ -5,6 +5,7 @@ import ( |
| "encoding/json" |
| "errors" |
| "fmt" |
| + "io" |
| "os" |
| "os/exec" |
| "path/filepath" |
| @@ -15,6 +16,7 @@ import ( |
| "github.com/carlosframework/platform/internal/hostbox" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/instancefs" |
| "github.com/carlosframework/platform/internal/moves" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -54,6 +56,8 @@ func relocationAttemptsPath(dataDir, op string) (string, error) { |
| return filepath.Join(dir, "attempts.json"), nil |
| } |
| +// readRelocationAttempts is the agent's read of its own journal: a |
| +// missing file is the journal's first state (OpenProbe creates it). |
| func readRelocationAttempts(dataDir, host, op string) (relocationAttemptJournal, error) { |
| path, err := relocationAttemptsPath(dataDir, op) |
| if err != nil { |
| @@ -66,6 +70,10 @@ func readRelocationAttempts(dataDir, host, op string) (relocationAttemptJournal, |
| if err != nil { |
| return relocationAttemptJournal{}, err |
| } |
| + return decodeRelocationAttempts(raw, host, op) |
| +} |
| + |
| +func decodeRelocationAttempts(raw []byte, host, op string) (relocationAttemptJournal, error) { |
| var j relocationAttemptJournal |
| if err := json.Unmarshal(raw, &j); err != nil || j.V != 1 || j.Host != host || j.Op != op || j.Opened == nil || j.Sessions == nil || j.Starts == nil { |
| return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s", hostbox.ErrChanged, op) |
| @@ -73,6 +81,52 @@ func readRelocationAttempts(dataDir, host, op string) (relocationAttemptJournal, |
| return j, nil |
| } |
| +// relocationAttemptsLimit bounds root's read of the journal: three record |
| +// attempts, each with one start and one session binding, is well under a |
| +// kibibyte. |
| +const relocationAttemptsLimit = 64 << 10 |
| + |
| +// readRelocationAttemptsBeneath is root's read of the agent-written |
| +// journal (fix round 1, I1). The file and its directories belong to the |
| +// agent's uid, which may replace them at will, so root opens it the way |
| +// it opens every agent-writable path on main: beneath the data root with |
| +// openat2 (RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS, instancefs), O_NONBLOCK |
| +// so a FIFO cannot hold root under the host guard, a regular file only, |
| +// the size capped before and during the read, and the context honoured. |
| +// A missing journal is a refusal, never a fresh one: root takes nothing |
| +// the journal does not hold. |
| +func readRelocationAttemptsBeneath(ctx context.Context, dataDir, host, op string) (relocationAttemptJournal, error) { |
| + if err := ctx.Err(); err != nil { |
| + return relocationAttemptJournal{}, err |
| + } |
| + if dataDir == "" || !validOpID(op) { |
| + return relocationAttemptJournal{}, ErrClaimStopUnavailable |
| + } |
| + f, err := instancefs.OpenBeneath(dataDir, filepath.Join("relocate", "ops", op, "attempts.json"), os.O_RDONLY, 0) |
| + if err != nil { |
| + return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s: %w", hostbox.ErrChanged, op, err) |
| + } |
| + defer f.Close() |
| + st, err := f.Stat() |
| + if err != nil { |
| + return relocationAttemptJournal{}, err |
| + } |
| + if !st.Mode().IsRegular() || st.Size() > relocationAttemptsLimit { |
| + return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s is not a regular file within %d bytes", hostbox.ErrChanged, op, relocationAttemptsLimit) |
| + } |
| + raw, err := io.ReadAll(io.LimitReader(f, relocationAttemptsLimit+1)) |
| + if err != nil { |
| + return relocationAttemptJournal{}, err |
| + } |
| + if len(raw) > relocationAttemptsLimit { |
| + return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s grew past %d bytes", hostbox.ErrChanged, op, relocationAttemptsLimit) |
| + } |
| + if err := ctx.Err(); err != nil { |
| + return relocationAttemptJournal{}, err |
| + } |
| + return decodeRelocationAttempts(raw, host, op) |
| +} |
| + |
| func (j relocationAttemptJournal) write(dataDir string) error { |
| path, err := relocationAttemptsPath(dataDir, j.Op) |
| if err != nil { |
| @@ -327,9 +381,17 @@ func (p *relocationProbeAttempts) Healthy(ctx context.Context, s OperationStart, |
| // op at probing, its executor's attempt k, the holder's session, |
| // generation and stamp (the journaled install stamp). Root's authorize |
| // then binds the reservation, the launch-input digest and the peer. |
| +// |
| +// The journal is the agent's file (the same shape as moves' probe journal |
| +// on main; claim ask C14 confirms it as root authority). Every field root |
| +// uses that the claim also has is cross-checked against root's own claim |
| +// read: the op (the journal's header), the host, the record attempt k |
| +// (probe_attempts.<exec>), the holder's session, generation and stamp, and |
| +// the executor as the attempt's box. The read itself is readRelocation- |
| +// AttemptsBeneath's (I1). |
| type relocationAttemptReader struct{ DataDir string } |
| -func (r relocationAttemptReader) ReadAttempt(_ context.Context, o hostclaim.Observed) (hostclaim.WakeRetryAttempt, error) { |
| +func (r relocationAttemptReader) ReadAttempt(ctx context.Context, o hostclaim.Observed) (hostclaim.WakeRetryAttempt, error) { |
| c := o.Claim |
| if r.DataDir == "" || c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseRelocatingProbing || c.Op.Journal["stuck"] != "" { |
| return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged |
| @@ -347,7 +409,7 @@ func (r relocationAttemptReader) ReadAttempt(_ context.Context, o hostclaim.Obse |
| if err != nil || k < 1 || k > hostclaim.MaxProbeAttempts { |
| return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged |
| } |
| - j, err := readRelocationAttempts(r.DataDir, o.Host, c.Op.ID) |
| + j, err := readRelocationAttemptsBeneath(ctx, r.DataDir, o.Host, c.Op.ID) |
| if err != nil { |
| return hostclaim.WakeRetryAttempt{}, err |
| } |
diff --git a/internal/host/relocationattempts_linux_test.go b/internal/host/relocationattempts_linux_test.go| new file mode 100644 |
| index 000000000..cc61728e9 |
| --- /dev/null |
| +++ b/internal/host/relocationattempts_linux_test.go |
| @@ -0,0 +1,258 @@ |
| +//go:build linux |
| + |
| +package host |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execlease "github.com/carlosframework/platform/internal/hostexec" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/instancectl" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "golang.org/x/sys/unix" |
| +) |
| + |
| +// Fix round 1, I1: root reads the agent-written journal the way it opens |
| +// every agent-writable path: a FIFO at the path answers at once rather |
| +// than holding root under the host guard, a symlink is refused, a file |
| +// past the cap is refused before it is decoded, and a cancelled context |
| +// reads nothing. |
| +func TestRootReaderOpensTheJournalBeneathTheDataRoot(t *testing.T) { |
| + ctx := context.Background() |
| + claim := &hostclaim.Claim{Host: "shop.example", Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 9, Session: 7, Generation: "P1", WriterBox: "dst"}, |
| + Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "op1", Phase: hostclaim.PhaseRelocatingProbing, Exec: "dst", Journal: map[string]string{"install_stamp": "9", "prepared": "P1", "probe_attempts.dst": "1"}}} |
| + o := hostclaim.Observed{Host: "shop.example", Claim: claim} |
| + attempt := hostclaim.WakeRetryAttempt{ID: "a", InvocationID: "b", Session: 7, Box: "dst"} |
| + good := func(t *testing.T, dd string) relocationAttemptJournal { |
| + t.Helper() |
| + j, _ := readRelocationAttempts(dd, "shop.example", "op1") |
| + j.Sessions["7"] = 1 |
| + j.Starts["1"] = relocationStart{Session: 7, Generation: "P1", Stamp: 9, State: probeReserved, Attempt: attempt} |
| + return j |
| + } |
| + t.Run("a FIFO", func(t *testing.T) { |
| + dd := t.TempDir() |
| + path, _ := relocationAttemptsPath(dd, "op1") |
| + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := unix.Mkfifo(path, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + done := make(chan error, 1) |
| + go func() { |
| + _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o) |
| + done <- err |
| + }() |
| + select { |
| + case err := <-done: |
| + if !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a FIFO: %v", err) |
| + } |
| + case <-time.After(5 * time.Second): |
| + t.Fatal("root blocked on a FIFO at the journal's path") |
| + } |
| + }) |
| + t.Run("a symlink", func(t *testing.T) { |
| + dd := t.TempDir() |
| + elsewhere := t.TempDir() |
| + if err := good(t, elsewhere).write(elsewhere); err != nil { |
| + t.Fatal(err) |
| + } |
| + real, _ := relocationAttemptsPath(elsewhere, "op1") |
| + path, _ := relocationAttemptsPath(dd, "op1") |
| + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Symlink(real, path); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a symlink to a sound journal was followed: %v", err) |
| + } |
| + // The same journal in place reads. |
| + if err := os.Remove(path); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := good(t, dd).write(dd); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { |
| + t.Fatalf("the journal in place: %+v %v", got, err) |
| + } |
| + }) |
| + t.Run("a symlinked directory", func(t *testing.T) { |
| + dd := t.TempDir() |
| + elsewhere := t.TempDir() |
| + if err := good(t, elsewhere).write(elsewhere); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.MkdirAll(filepath.Join(dd, "relocate"), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Symlink(filepath.Join(elsewhere, "relocate", "ops"), filepath.Join(dd, "relocate", "ops")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a symlinked ops directory was followed: %v", err) |
| + } |
| + }) |
| + t.Run("past the cap", func(t *testing.T) { |
| + dd := t.TempDir() |
| + j := good(t, dd) |
| + raw, _ := json.Marshal(j) |
| + raw = append(raw, bytes.Repeat([]byte(" "), relocationAttemptsLimit+1-len(raw))...) |
| + path, _ := relocationAttemptsPath(dd, "op1") |
| + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(path, raw, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a journal past the cap: %v", err) |
| + } |
| + // Trailing whitespace within the cap decodes. |
| + if err := os.WriteFile(path, raw[:relocationAttemptsLimit], 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { |
| + t.Fatalf("a journal at the cap: %+v %v", got, err) |
| + } |
| + }) |
| + t.Run("a cancelled context", func(t *testing.T) { |
| + dd := t.TempDir() |
| + if err := good(t, dd).write(dd); err != nil { |
| + t.Fatal(err) |
| + } |
| + gone, cancel := context.WithCancel(ctx) |
| + cancel() |
| + if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(gone, o); !errors.Is(err, context.Canceled) { |
| + t.Fatalf("a cancelled read: %v", err) |
| + } |
| + }) |
| +} |
| + |
| +// Fix round 1, M4: root's authorize, with the relocation reader installed, |
| +// over a journal relocationProbeAttempts.Reserve actually wrote for P's |
| +// opening. The reservation the agent presents binds end to end: the |
| +// attempt id as the role reservation, the invocation id, the launch-input |
| +// digest, the peer's boot and controller. Another invocation, a journal |
| +// rewritten under the agent, and a journal replaced by a FIFO are refused. |
| +func TestRootAuthorizesAReservationTheRelocationJournalHolds(t *testing.T) { |
| + ctx := t.Context() |
| + dataDir := t.TempDir() |
| + f := newROFixture(t, roOptions{}) |
| + f.inputs.Launch = func(context.Context, registry.Route) (ClaimServingLaunch, error) { |
| + return ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/opt/carlos/versions/v1/app", ExecutableSHA256: strings.Repeat("b", 64), Version: "v1", |
| + Args: []string{"--socket", filepath.Join(provision.InstanceRunDir(roHost), "app.sock"), "--db", provision.NestedDBPath(dataDir, roHost)}, Env: []string{"CARLOS_ADMIN_TOKEN=test"}}, nil |
| + } |
| + in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + admitted := f.admit(f.read(), in) |
| + h := admitted.Claim.Holder |
| + enrollment := strings.Repeat("a", 32) |
| + peer := StopPeer{UID: 1001, PID: 42, ProcessStart: "start-42", BootID: "boot-1", ControllerID: "controller-1"} |
| + target := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Op: roOp, Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: enrollment, BootID: peer.BootID, ControllerID: peer.ControllerID, ClaimETag: admitted.ETag, DirectoryRevision: "7"} |
| + |
| + // The agent's side: the journal Reserve writes for the admitted session. |
| + attempts := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: f.inputs.Now} |
| + s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: target} |
| + if err := attempts.Reserve(ctx, s, 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + a := attempts.CurrentAttempt(s) |
| + if a.ID == "" { |
| + t.Fatal("no reservation") |
| + } |
| + |
| + // Root's side: its own providers, the reader over the same data root. |
| + exe := hostbox.LaunchFile{Device: 5, Inode: 6} |
| + kernel := &launchKernel{path: in.Executable, want: exe, digest: in.ExecutableSHA256} |
| + root := &ClaimLaunchRoot{ValidateExecutable: func(string, string, string) error { return nil }, Local: &rev2Local{}, Clock: &rev2Clock{wall: time.Now(), boot: 100}, DataRoot: dataDir, ExecLockDir: t.TempDir(), |
| + Attempts: map[string]claimLaunchAttemptReader{hostclaim.KindRelocating: relocationAttemptReader{DataDir: dataDir}}, |
| + Guard: hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.claims}}, |
| + Actor: func(context.Context) (hostbox.Actor, error) { |
| + return hostbox.Actor{Box: "dst", Enrollment: enrollment, Revision: 7, Region: "ie", Eligible: true}, nil |
| + }, |
| + Kernel: kernel} |
| + state := execlease.State{V: 1, Binding: execlease.Binding{Host: roHost, Incarnation: h.Incarnation, Op: roOp, Box: "dst", Enrollment: enrollment, BootID: peer.BootID, ControllerID: "lease-controller"}, |
| + Process: execlease.ControllerProcess{PID: peer.PID, Start: peer.ProcessStart, KernelBootID: peer.BootID}, State: "active", LeaseETag: "lease-etag", Expires: time.Now().Add(time.Hour), SafeUntil: 1000} |
| + if err := os.MkdirAll(filepath.Join(dataDir, ".hostexec"), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _ := json.Marshal(state) |
| + if err := os.WriteFile(filepath.Join(dataDir, ".hostexec", roHost+".json"), raw, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, suffix := range []string{".lock", ".live"} { |
| + file, err := os.OpenFile(filepath.Join(root.ExecLockDir, roHost+suffix), os.O_CREATE|os.O_RDWR, 0o660) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { file.Close() }) |
| + } |
| + spec := &hostbox.LaunchSpec{Identity: in.Identity, Executable: in.Executable, Digest: in.ExecutableSHA256, Args: in.Args, Env: in.Env} |
| + reservation := hostbox.LaunchReservation{ID: a.ID + ":main", Role: "main", InvocationID: a.InvocationID, Inputs: a.LaunchInputs, Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(roHost)}, Executable: exe} |
| + q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Spec: spec, Reservation: &reservation, Executable: in.Executable, MemoryMax: cgroup.TierDefault.MemoryMax} |
| + if err := root.authorize(ctx, peer, q, admitted.Claim); err != nil { |
| + t.Fatalf("the reservation the journal holds: %v", err) |
| + } |
| + other := reservation |
| + other.InvocationID = "another" |
| + q.Reservation = &other |
| + if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("another invocation: %v", err) |
| + } |
| + q.Reservation = &reservation |
| + // The agent rewrites its journal: root's reservation no longer matches. |
| + j, err := readRelocationAttempts(dataDir, roHost, roOp) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + st := j.Starts["1"] |
| + st.Attempt.InvocationID = "rewritten" |
| + j.Starts["1"] = st |
| + if err := j.write(dataDir); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a rewritten journal: %v", err) |
| + } |
| + // A FIFO in the journal's place refuses at once. |
| + path, _ := relocationAttemptsPath(dataDir, roOp) |
| + if err := os.Remove(path); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := unix.Mkfifo(path, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + done := make(chan error, 1) |
| + go func() { done <- root.authorize(ctx, peer, q, admitted.Claim) }() |
| + select { |
| + case err := <-done: |
| + if !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a FIFO: %v", err) |
| + } |
| + case <-time.After(5 * time.Second): |
| + t.Fatal("root blocked on a FIFO in the journal's place") |
| + } |
| +} |
diff --git a/internal/host/relocationattempts_test.go b/internal/host/relocationattempts_test.go| index 1329a2051..829ea8d8d 100644 |
| --- a/internal/host/relocationattempts_test.go |
| +++ b/internal/host/relocationattempts_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "net" |
| "os" |
| "path/filepath" |
| + "strconv" |
| "strings" |
| "testing" |
| "time" |
| @@ -225,6 +226,11 @@ func TestRootReaderNeedsABoundReservedStart(t *testing.T) { |
| "session bound to another attempt": func(j *relocationAttemptJournal) { j.Sessions["7"] = 2 }, |
| "a failed start": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.State = probeFailed; j.Starts["1"] = s }, |
| "a healthy start": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.State = probeHealthy; j.Starts["1"] = s }, |
| + "a start under another session": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.Session = 8; j.Starts["1"] = s }, |
| + "a start at another stamp": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.Stamp = 10; j.Starts["1"] = s }, |
| + "an attempt for another session": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.Attempt.Session = 8; j.Starts["1"] = s }, |
| + "an attempt for another box": func(j *relocationAttemptJournal) { s := j.Starts["1"]; s.Attempt.Box = "src"; j.Starts["1"] = s }, |
| + "a journal for another op": func(j *relocationAttemptJournal) { j.Op = "op2" }, |
| } { |
| dd := t.TempDir() |
| j, _ := readRelocationAttempts(dd, "shop.example", "op1") |
| @@ -238,4 +244,16 @@ func TestRootReaderNeedsABoundReservedStart(t *testing.T) { |
| t.Fatalf("%s: read %+v %v", name, got, err) |
| } |
| } |
| + // The record attempt is root's claim read, bounded by MaxProbeAttempts. |
| + dd := t.TempDir() |
| + j, _ := readRelocationAttempts(dd, "shop.example", "op1") |
| + j.Sessions["7"] = hostclaim.MaxProbeAttempts + 1 |
| + j.Starts[strconv.Itoa(hostclaim.MaxProbeAttempts+1)] = relocationStart{Session: 7, Generation: "P1", Stamp: 9, State: probeReserved, Attempt: attempt} |
| + if err := j.write(dd); err != nil { |
| + t.Fatal(err) |
| + } |
| + claim.Op.Journal["probe_attempts.dst"] = strconv.Itoa(hostclaim.MaxProbeAttempts + 1) |
| + if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("an attempt past the bound: %v", err) |
| + } |
| } |
diff --git a/internal/host/relocationowners.go b/internal/host/relocationowners.go| index 5093cf20a..45fd801f6 100644 |
| --- a/internal/host/relocationowners.go |
| +++ b/internal/host/relocationowners.go |
| @@ -68,7 +68,7 @@ func (s *RelocationSource) Open(ctx context.Context, store release.Store, data l |
| log.Printf("region HA: relocation owners not opened: the moves C8 files-manifest provider is not bound") |
| return ClaimRelocationOwners{}, nil, nil |
| } |
| - if src.Launch == nil || src.Markers == nil { |
| + if src.Launch == nil || src.Markers == nil || src.Identity == nil { |
| return ClaimRelocationOwners{}, nil, hostwrite.ErrUnavailable |
| } |
| files, err := s.FilesStore(ctx, actor.Bucket) |
| @@ -77,7 +77,7 @@ func (s *RelocationSource) Open(ctx context.Context, store release.Store, data l |
| } |
| releases := StopReceiptReleases{Receipts: RelocationReleaseReceipts{Store: store, Directory: s.Directory}} |
| inputs := &RelocationWakeInputs{Launch: src.Launch, FilesManifest: src.Files, Markers: src.Markers, Claims: store, FilesStore: files, |
| - Data: hostbox.RequestStore{Store: data, Check: func(context.Context) error { return nil }}, Bucket: actor.Bucket, Region: actor.Region, Box: actor.Box, |
| + Data: hostbox.RequestStore{Store: data, Check: src.Identity}, Bucket: actor.Bucket, Region: actor.Region, Box: actor.Box, |
| NewID: newClaimID, Now: time.Now, Releases: releases} |
| attempts := &relocationProbeAttempts{Claims: store, DataDir: s.DataDir, Box: actor.Box, NewID: newClaimID, Now: time.Now} |
| owners := ClaimRelocationOwners{ |
| @@ -128,11 +128,15 @@ func relocationBeforeAdmission(box string, releases hostclaim.ReleaseAuthority) |
| if hostclaim.PreparedOpened(c) { |
| return nil |
| } |
| - rel, released, err := journaledRelease(c) |
| + certified, err := hostrelocate.CertifiedFor(c) |
| if err != nil { |
| - return err // present and unreadable is never a failover |
| + return err // present and unreadable, or mismatched, is never a failover |
| } |
| - if released { |
| + if certified { |
| + var rel hostclaim.Release |
| + if err := json.Unmarshal([]byte(c.Op.Journal["release"]), &rel); err != nil { |
| + return err |
| + } |
| return releases.VerifyRelease(ctx, *c, rel) |
| } |
| src := c.Op.Intent["source_box"] |
| @@ -174,41 +178,67 @@ func relocationAfterAdmission(dataDir, box string) func(context.Context, hostcla |
| } |
| } |
| +// relocationProbe is what one probe attempt over the claim needs and what |
| +// Preflight reads before the attempt is counted (ruling 48; review M1): |
| +// the journaled release's verdict, the prepared authority, the pin's |
| +// agreement with it, and when probing began. Each is a deterministic |
| +// reading of the claim, so a defect never heals by itself: it is found |
| +// before an attempt is spent on it, and found again at the attempt. |
| +type relocationProbe struct { |
| + certified bool |
| + auth hostclaim.PreparedAuthority |
| + empty bool |
| + since time.Time |
| +} |
| + |
| +func relocationProbeClaim(c *hostclaim.Claim) (relocationProbe, error) { |
| + if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating { |
| + return relocationProbe{}, hostbox.ErrChanged |
| + } |
| + certified, err := hostrelocate.CertifiedFor(c) |
| + if err != nil { |
| + return relocationProbe{}, err |
| + } |
| + auth, err := preparedAuthority(c) |
| + if err != nil { |
| + return relocationProbe{}, err |
| + } |
| + empty, err := emptyPrepared(c, auth) |
| + if err != nil { |
| + return relocationProbe{}, err |
| + } |
| + since, err := time.Parse(time.RFC3339Nano, c.Op.Journal["probing_since"]) |
| + if err != nil { |
| + return relocationProbe{}, fmt.Errorf("%w: probing_since: %v", hostbox.ErrChanged, err) |
| + } |
| + return relocationProbe{certified: certified, auth: auth, empty: empty, since: since}, nil |
| +} |
| + |
| // relocationAdmit is one probe attempt: the activator's ProbeWake with the |
| -// probe deadline and, on P's prepared opening, P's install manifest. |
| +// probe deadline and, on P's prepared opening, P's install manifest. The |
| +// claim it reads was checked by Preflight before the attempt was counted |
| +// (relocationProbeClaim); it is read again here, on the counted |
| +// observation. |
| func (a ClaimAssembly) relocationAdmit(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed, bool) (uint64, bool, error) { |
| return func(ctx context.Context, o hostclaim.Observed, certified bool) (uint64, bool, error) { |
| if cfg.Activator == nil || cfg.Versions == nil || o.Claim == nil || o.Claim.Holder == nil { |
| return 0, false, ErrClaimStopUnavailable |
| } |
| c := o.Claim |
| - if _, _, err := journaledRelease(c); err != nil { |
| + probe, err := relocationProbeClaim(c) |
| + if err != nil { |
| return 0, false, err |
| } |
| - if certified != hostrelocate.CertifiedFor(c) { |
| + if certified != probe.certified { |
| return 0, false, fmt.Errorf("%w: the participant and the owners disagree on certification", hostrelocate.ErrRefused) |
| } |
| - since, err := time.Parse(time.RFC3339Nano, c.Op.Journal["probing_since"]) |
| - if err != nil { |
| - return 0, false, err |
| - } |
| - pc := activator.ProbeConfig{Host: o.Host, Op: c.Op.ID, HoldOp: c.Op.ID, Deadline: since.Add(hostrelocate.ProbeDeadline)} |
| - if !hostclaim.PreparedOpened(c) { |
| - auth, err := preparedAuthority(c) |
| + pc := activator.ProbeConfig{Host: o.Host, Op: c.Op.ID, HoldOp: c.Op.ID, Deadline: probe.since.Add(hostrelocate.ProbeDeadline)} |
| + if !hostclaim.PreparedOpened(c) && !probe.empty { |
| + im, _, err := readInstallManifest(ctx, cfg.Versions, cfg.Bucket, moves.GenerationPrefix(c.Holder.Location, probe.auth.Generation)) |
| if err != nil { |
| return 0, false, err |
| } |
| - empty, err := emptyPrepared(c, auth) |
| - if err != nil { |
| - return 0, false, err |
| - } |
| - if !empty { |
| - im, _, err := readInstallManifest(ctx, cfg.Versions, cfg.Bucket, moves.GenerationPrefix(c.Holder.Location, auth.Generation)) |
| - if err != nil { |
| - return 0, false, err |
| - } |
| - pc.Prepared = &im |
| - } |
| + pc.Prepared = &im |
| } |
| res, err := cfg.Activator.ProbeWake(ctx, pc) |
| switch { |
diff --git a/internal/host/relocationowners_test.go b/internal/host/relocationowners_test.go| index 6cc657b6e..941e9abc9 100644 |
| --- a/internal/host/relocationowners_test.go |
| +++ b/internal/host/relocationowners_test.go |
| @@ -599,11 +599,15 @@ func TestRelocationSourceOpensACompleteOwnerSet(t *testing.T) { |
| opened = append(opened, bucket) |
| return release.NewDirStore(t.TempDir()), nil |
| }} |
| - src.SetServingSources(ClaimServingSources{ |
| - Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil }, |
| - Markers: plan9D5Markers{leasetest.NewVersionedBucket()}, |
| - Files: &roFiles{}, |
| - }) |
| + identityErr := errors.New("the roster entry has moved") |
| + var identity error |
| + sources := ClaimServingSources{ |
| + Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil }, |
| + Markers: plan9D5Markers{leasetest.NewVersionedBucket()}, |
| + Files: &roFiles{}, |
| + Identity: func(context.Context) error { return identity }, |
| + } |
| + src.SetServingSources(sources) |
| actor := hostbox.Actor{Box: "dst", Bucket: "data-ie", Region: "ie"} |
| owners, refusals, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor) |
| if err != nil { |
| @@ -618,9 +622,29 @@ func TestRelocationSourceOpensACompleteOwnerSet(t *testing.T) { |
| if bound, err := claimRelocationC1(plan11Env("1", "1"), true, owners); !bound || err != nil { |
| t.Fatalf("the relocating dispatch did not bind: %v %v", bound, err) |
| } |
| + // M2: every data-bucket request the inputs make runs the box's own |
| + // identity check first, as main's serving owners do. |
| + identity = identityErr |
| + if _, err := owners.Inputs.(*RelocationWakeInputs).Data.ListVersions(ctx, "data-ie", "instances/"); !errors.Is(err, identityErr) { |
| + t.Fatalf("a read without the identity check: %v", err) |
| + } |
| + identity = nil |
| if _, _, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, hostbox.Actor{Box: "dst", Region: "ie"}); !errors.Is(err, hostwrite.ErrUnavailable) { |
| t.Fatalf("no bucket: %v", err) |
| } |
| + for name, without := range map[string]func(*ClaimServingSources){ |
| + "no launch": func(s *ClaimServingSources) { s.Launch = nil }, |
| + "no markers": func(s *ClaimServingSources) { s.Markers = nil }, |
| + "no identity": func(s *ClaimServingSources) { s.Identity = nil }, |
| + } { |
| + partial := sources |
| + without(&partial) |
| + src.SetServingSources(partial) |
| + if _, _, err := src.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor); !errors.Is(err, hostwrite.ErrUnavailable) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| + src.SetServingSources(sources) |
| var none *RelocationSource |
| if _, _, err := none.Open(ctx, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor); !errors.Is(err, hostwrite.ErrUnavailable) { |
| t.Fatalf("a nil source: %v", err) |
| @@ -657,84 +681,122 @@ func (f *roFixture) authority(mutate func(*hostclaim.PreparedAuthority)) hostcla |
| } |
| // Absence is never evidence: a journaled release that is present but |
| -// unreadable is not "no release". Before the fix a damaged release on a |
| +// unreadable, or that names another writer than the predecessor, is not |
| +// "no release" (fix round 1, I3). Before the fix a damaged release on a |
| // claim whose source is lost and fenced passed the admission hook as a |
| -// failover, and Select, AdmissionEvidence and the probe attempt went on |
| -// uncertified (CertifiedFor reads an unparseable release as none). |
| +// failover, and a mismatched one opened P uncertified, declaring the |
| +// source's writes lost with no failover evidence. |
| func TestRelocationAnUnreadableReleaseIsNeverAFailover(t *testing.T) { |
| ctx := t.Context() |
| - for name, raw := range map[string]string{"unparseable": `{"released_by":`, "no releasing box": `{}`} { |
| + for name, damage := range map[string]func(*hostclaim.Claim){ |
| + "unparseable": func(c *hostclaim.Claim) { c.Op.Journal["release"] = `{"released_by":` }, |
| + "no releasing box": func(c *hostclaim.Claim) { c.Op.Journal["release"] = `{}` }, |
| + "another session": func(c *hostclaim.Claim) { |
| + var rel hostclaim.Release |
| + _ = json.Unmarshal([]byte(c.Op.Journal["release"]), &rel) |
| + rel.Session++ |
| + c.Op.Journal["release"] = string(mustJSON(t, rel)) |
| + }, |
| + "another box": func(c *hostclaim.Claim) { |
| + var rel hostclaim.Release |
| + _ = json.Unmarshal([]byte(c.Op.Journal["release"]), &rel) |
| + rel.ReleasedBy = "other" |
| + c.Op.Journal["release"] = string(mustJSON(t, rel)) |
| + }, |
| + } { |
| t.Run(name, func(t *testing.T) { |
| - f := newROFixture(t, roOptions{failover: true}) |
| + // A failover shape (the source lost and fenced) for the two |
| + // unreadable cases, a drain for the two mismatched ones. |
| + f := newROFixture(t, roOptions{failover: strings.HasPrefix(name, "unparseable") || strings.HasPrefix(name, "no ")}) |
| good, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| if err != nil { |
| t.Fatal(err) |
| } |
| - damaged := f.update(func(c *hostclaim.Claim) error { c.Op.Journal["release"] = raw; return nil }) |
| + damaged := f.update(func(c *hostclaim.Claim) error { damage(c); return nil }) |
| releases := &roReleases{} |
| - if err := relocationBeforeAdmission("dst", releases)(ctx, damaged); !errors.Is(err, hostbox.ErrChanged) || releases.calls != 0 { |
| - t.Fatalf("the hook took a damaged release for a failover: %v (%d checks)", err, releases.calls) |
| + if err := relocationBeforeAdmission("dst", releases)(ctx, damaged); !errors.Is(err, hostrelocate.ErrRelease) || releases.calls != 0 { |
| + t.Fatalf("the hook took a damaged release: %v (%d checks)", err, releases.calls) |
| } |
| - if _, err := f.inputs.Select(ctx, f.route, damaged, roProbe); !errors.Is(err, hostbox.ErrChanged) { |
| - t.Fatalf("Select opened P over a damaged release: %v", err) |
| + checked := f.releases.calls |
| + if _, err := f.inputs.Select(ctx, f.route, damaged, roProbe); !errors.Is(err, hostrelocate.ErrRelease) || f.releases.calls != checked { |
| + t.Fatalf("Select opened P over a damaged release: %v (%d checks)", err, f.releases.calls-checked) |
| } |
| - if err := f.inputs.AdmissionEvidence(ctx, damaged, good.Admission, good.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + if err := f.inputs.AdmissionEvidence(ctx, damaged, good.Admission, good.Baseline); !errors.Is(err, hostrelocate.ErrRelease) { |
| t.Fatalf("AdmissionEvidence passed over a damaged release: %v", err) |
| } |
| act := &relocationProbeActivator{} |
| probe := ClaimAssembly{}.relocationAdmit(RelocationAssemblyConfig{Activator: act, Versions: f.bucket, Bucket: "data"}) |
| - if _, _, err := probe(ctx, damaged, false); !errors.Is(err, hostbox.ErrChanged) || len(act.probes) != 0 { |
| + if _, _, err := probe(ctx, damaged, false); !errors.Is(err, hostrelocate.ErrRelease) || len(act.probes) != 0 { |
| t.Fatalf("the probe attempt ran over a damaged release: %v (%d probes)", err, len(act.probes)) |
| } |
| + // M1: the preflight finds it before any attempt is counted, |
| + // before root is asked. |
| + if err := (ClaimAssembly{Wake: &ClaimWakeAssembly{Roles: &ClaimRoleAssembly{}}}).relocationPreflight(RelocationAssemblyConfig{})(ctx, damaged); !errors.Is(err, hostrelocate.ErrRelease) { |
| + t.Fatalf("the preflight passed a damaged release: %v", err) |
| + } |
| }) |
| } |
| } |
| -// Absence is never evidence: a later attempt's empty listing is an empty |
| -// predecessor only over an evidenced empty P. Under a non-empty pin P's |
| -// snapshot is in the location, so a listing that finds nothing is a wrong |
| -// bucket or a listing not yet visible, never an empty cold admission. |
| -func TestRelocationLaterAttemptEmptyListingNeedsAnEmptyPin(t *testing.T) { |
| +// A drain's P opening is certified only by a release naming the exact |
| +// predecessor; the hook's PreparedOpened short-circuit stands on its own: |
| +// once P is open no release or fence is asked for. |
| +func TestRelocationBeforeAdmissionIgnoresTheReleaseOncePIsOpen(t *testing.T) { |
| ctx := t.Context() |
| f := newROFixture(t, roOptions{}) |
| first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| if err != nil { |
| t.Fatal(err) |
| } |
| - opened := f.admit(f.read(), first) |
| - // The same inputs over a bucket that lists nothing for the location. |
| - blank := leasetest.NewVersionedBucket() |
| - blank.AddBucket("data") |
| - elsewhere := &RelocationWakeInputs{Launch: f.inputs.Launch, FilesManifest: f.c8, Markers: plan9D5Markers{blank}, Claims: f.claims, FilesStore: f.files, Data: blank, Bucket: "data", Region: "ie", Box: "dst", |
| - NewID: f.inputs.NewID, Now: f.inputs.Now, Releases: f.releases} |
| - if in, err := elsewhere.Select(ctx, f.route, opened, roProbe); !errors.Is(err, hostbox.ErrChanged) { |
| - t.Fatalf("an empty listing under a pinned snapshot selected empty=%v: %v", in.Empty, err) |
| - } |
| - h := opened.Claim.Holder |
| - fresh := hostclaim.Admission{ObservedOpID: roOp, ObservedPhase: hostclaim.PhaseRelocatingProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "dst", Mode: "cold", NewGeneration: "fresh9", Region: "ie", PredecessorRegion: "ie"} |
| - if err := elsewhere.EmptyAuthority(ctx, opened, ClaimLaunchInputs{Empty: true, Admission: fresh}); !errors.Is(err, hostbox.ErrChanged) { |
| - t.Fatalf("EmptyAuthority proved an empty later attempt under a pinned snapshot: %v", err) |
| - } |
| - if err := elsewhere.AdmissionEvidence(ctx, opened, fresh, nil); !errors.Is(err, hostbox.ErrChanged) { |
| - t.Fatalf("AdmissionEvidence passed an empty later attempt under a pinned snapshot: %v", err) |
| - } |
| - // Over an evidenced empty P the same later attempt is the ordinary |
| - // empty cold admission. |
| - e := newROFixture(t, roOptions{empty: true}) |
| - ein, err := e.inputs.Select(ctx, e.route, e.read(), roProbe) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - eopened := e.admit(e.read(), ein) |
| - again, err := e.inputs.Select(ctx, e.route, eopened, roProbe) |
| - if err != nil || !again.Empty || again.Baseline != nil || again.Admission.KeepStamp { |
| - t.Fatalf("a later attempt over an empty P: %+v %v", again, err) |
| - } |
| - if err := e.inputs.EmptyAuthority(ctx, eopened, again); err != nil { |
| - t.Fatalf("the empty later attempt's authority: %v", err) |
| + f.admit(f.read(), first) |
| + opened := f.update(func(c *hostclaim.Claim) error { delete(c.Op.Journal, "release"); return nil }) |
| + releases := &roReleases{err: errors.New("never asked")} |
| + if err := relocationBeforeAdmission("dst", releases)(ctx, opened); err != nil || releases.calls != 0 { |
| + t.Fatalf("after P's opening: %v (%d checks)", err, releases.calls) |
| } |
| - if err := e.inputs.AdmissionEvidence(ctx, eopened, again.Admission, nil); err != nil { |
| - t.Fatalf("the empty later attempt's evidence: %v", err) |
| +} |
| + |
| +// Absence is never evidence: a later attempt follows P's admitted |
| +// session, which ran, so a listing that finds nothing is nothing |
| +// replicated or not yet visible, never an empty predecessor. Main's |
| +// serving rule admits an empty cold admission only for a holder that |
| +// never admitted a writer, which a later attempt never is (fix round 1, |
| +// I2). Over an empty P as over a pinned snapshot, Select refuses, no |
| +// EmptyAuthority proves it, and no evidence passes a nil baseline. |
| +func TestRelocationLaterAttemptIsNeverEmpty(t *testing.T) { |
| + ctx := t.Context() |
| + for name, o := range map[string]roOptions{"a pinned snapshot": {}, "an empty P": {empty: true}} { |
| + t.Run(name, func(t *testing.T) { |
| + f := newROFixture(t, o) |
| + first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + opened := f.admit(f.read(), first) |
| + // The same inputs over a bucket that lists nothing for the location. |
| + blank := leasetest.NewVersionedBucket() |
| + blank.AddBucket("data") |
| + elsewhere := &RelocationWakeInputs{Launch: f.inputs.Launch, FilesManifest: f.c8, Markers: plan9D5Markers{blank}, Claims: f.claims, FilesStore: f.files, Data: blank, Bucket: "data", Region: "ie", Box: "dst", |
| + NewID: f.inputs.NewID, Now: f.inputs.Now, Releases: f.releases} |
| + for _, inputs := range []*RelocationWakeInputs{f.inputs, elsewhere} { |
| + // Over the pinned snapshot the real bucket lists P's root, a |
| + // baseline; the blank bucket, and an empty P's own bucket, list |
| + // nothing, which is never an empty admission. |
| + if inputs == elsewhere || o.empty { |
| + if in, err := inputs.Select(ctx, f.route, opened, roProbe); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("an empty listing after P's session selected empty=%v: %v", in.Empty, err) |
| + } |
| + } |
| + h := opened.Claim.Holder |
| + fresh := hostclaim.Admission{ObservedOpID: roOp, ObservedPhase: hostclaim.PhaseRelocatingProbing, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "dst", Mode: "cold", NewGeneration: "fresh9", Region: "ie", PredecessorRegion: "ie"} |
| + if err := inputs.EmptyAuthority(ctx, opened, ClaimLaunchInputs{Empty: true, Admission: fresh}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("EmptyAuthority proved an empty later attempt: %v", err) |
| + } |
| + if err := inputs.AdmissionEvidence(ctx, opened, fresh, nil); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("AdmissionEvidence passed an empty later attempt: %v", err) |
| + } |
| + } |
| + }) |
| } |
| } |
| @@ -783,3 +845,344 @@ func TestRelocationEmptyPNeedsThePinAndTheAuthorityToAgree(t *testing.T) { |
| } |
| }) |
| } |
| + |
| +// roWork is a spawn selection whose fence always holds. |
| +type roWork struct{} |
| + |
| +func (roWork) With(_ context.Context, _ hostwrite.Identity, fn func() error) error { return fn() } |
| + |
| +// putManifest replaces P's install.json in the bucket and the authority's |
| +// digest of it, so only the mutated field disagrees. |
| +func (f *roFixture) putManifest(mutate func(*moves.InstallManifest)) hostclaim.Observed { |
| + f.t.Helper() |
| + im := f.manifest |
| + mutate(&im) |
| + raw, _ := json.Marshal(im) |
| + f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix(f.read().Claim.Holder.Location, roP)), raw) |
| + sum := sha256.Sum256(raw) |
| + return f.authority(func(a *hostclaim.PreparedAuthority) { a.SHA256 = hex.EncodeToString(sum[:]) }) |
| +} |
| + |
| +// atP is the observation as the admission that opens P leaves the holder, |
| +// without writing the claim. |
| +func atP(o hostclaim.Observed) hostclaim.Observed { |
| + c := *o.Claim |
| + h := *c.Holder |
| + h.Generation = roP |
| + c.Holder = &h |
| + o.Claim = &c |
| + return o |
| +} |
| + |
| +// Rulings 37 and 39, by behaviour: the two policy refusals refuse a |
| +// relocating selection at another installation than the holder's, and |
| +// any selection by a box the claim lists as lost; they pass everything |
| +// else and never grant. |
| +func TestRelocationPolicyRefusalsRefuseByBehaviour(t *testing.T) { |
| + ctx := t.Context() |
| + f := newROFixture(t, roOptions{}) |
| + c := f.read().Claim |
| + h := c.Holder |
| + at := hostbox.Selection{Host: roHost, Action: "probe", Box: "dst", Region: "ie", Captured: hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}} |
| + if err := refuseStaleStamp(ctx, c, at, hostwrite.Identity{}); err != nil { |
| + t.Fatalf("the holder's installation: %v", err) |
| + } |
| + stale := at |
| + stale.Captured.Stamp++ |
| + if err := refuseStaleStamp(ctx, c, stale, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a retired stamp: %v", err) |
| + } |
| + other := at |
| + other.Captured.Incarnation = strings.Repeat("9", 64) |
| + if err := refuseStaleStamp(ctx, c, other, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("another incarnation: %v", err) |
| + } |
| + serving := *c |
| + serving.Op = hostclaim.Op{Kind: hostclaim.KindServing} |
| + if err := refuseStaleStamp(ctx, &serving, stale, hostwrite.Identity{}); err != nil { |
| + t.Fatalf("the stamp refusal reaches beyond relocating: %v", err) |
| + } |
| + if err := refuseLostBox(ctx, c, at, hostwrite.Identity{}); err != nil { |
| + t.Fatalf("a box not lost: %v", err) |
| + } |
| + lost := *c |
| + lost.Lost = []string{"dst"} |
| + if err := refuseLostBox(ctx, &lost, at, hostwrite.Identity{}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a lost box: %v", err) |
| + } |
| + if err := refuseLostBox(ctx, &lost, hostbox.Selection{Box: "src"}, hostwrite.Identity{}); err != nil { |
| + t.Fatalf("another box while dst is lost: %v", err) |
| + } |
| +} |
| + |
| +// preparedMarker binds install.json to the prepared authority (its host, |
| +// its snapshot position, the holder at P) and takes an existing marker |
| +// for P only when it is this box's, this incarnation's, this host's, with |
| +// the authority's digest; another op's marker is superseded by its ETag. |
| +func TestRelocationPreparedMarkerBindsTheManifestAndTheMarker(t *testing.T) { |
| + ctx := t.Context() |
| + fence := func(context.Context) error { return nil } |
| + t.Run("install.json", func(t *testing.T) { |
| + for name, mutate := range map[string]func(*moves.InstallManifest){ |
| + "another host": func(im *moves.InstallManifest) { im.Host = "other.example" }, |
| + "another position": func(im *moves.InstallManifest) { im.N = 41 }, |
| + } { |
| + f := newROFixture(t, roOptions{}) |
| + o := f.putManifest(mutate) |
| + if _, err := f.inputs.preparedMarker(ctx, atP(o), fence); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| + f := newROFixture(t, roOptions{}) |
| + if _, err := f.inputs.preparedMarker(ctx, f.read(), fence); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("the holder not at P: %v", err) |
| + } |
| + if _, err := f.inputs.preparedMarker(ctx, atP(f.read()), fence); err != nil { |
| + t.Fatalf("the authority's own manifest: %v", err) |
| + } |
| + }) |
| + t.Run("an existing marker", func(t *testing.T) { |
| + f := newROFixture(t, roOptions{}) |
| + o := atP(f.read()) |
| + h := o.Claim.Holder |
| + auth, _ := preparedAuthority(o.Claim) |
| + mine := genselect.Marker{Op: roP, Incarnation: h.Incarnation, Host: roHost, Box: "dst", ManifestSHA256: auth.SHA256} |
| + for name, mutate := range map[string]func(*genselect.Marker){ |
| + "another box": func(m *genselect.Marker) { m.Box = "src" }, |
| + "another incarnation": func(m *genselect.Marker) { m.Incarnation = strings.Repeat("9", 64) }, |
| + "another host": func(m *genselect.Marker) { m.Host = "other.example" }, |
| + "another manifest": func(m *genselect.Marker) { m.ManifestSHA256 = strings.Repeat("0", 64) }, |
| + } { |
| + m := mine |
| + mutate(&m) |
| + f.bucket.Put("data", markerKey(h), mustJSON(t, m)) |
| + if _, err := f.inputs.preparedMarker(ctx, o, fence); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| + written := f.bucket.Put("data", markerKey(h), mustJSON(t, mine)) |
| + if r, err := f.inputs.preparedMarker(ctx, o, fence); err != nil || r.MarkerVersion != written { |
| + t.Fatalf("P's own marker: %+v %v", r, err) |
| + } |
| + earlier := f.bucket.Put("data", markerKey(h), mustJSON(t, genselect.Marker{Op: "g1", Incarnation: h.Incarnation, Host: roHost, Box: "src"})) |
| + r, err := f.inputs.preparedMarker(ctx, o, fence) |
| + if err != nil || r.MarkerVersion == earlier || r.MarkerVersion != f.bucket.LatestID("data", markerKey(h)) { |
| + t.Fatalf("an earlier generation's marker superseded: %+v %v", r, err) |
| + } |
| + }) |
| +} |
| + |
| +// PublicationEvidence for P's prepared opening requires the holder still |
| +// at P, with the marker exactly the recorded version. |
| +func TestRelocationPublicationEvidenceNeedsTheHolderAtP(t *testing.T) { |
| + ctx := t.Context() |
| + f := newROFixture(t, roOptions{}) |
| + in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + admitted := f.admit(f.read(), in) |
| + if err := f.inputs.Materialize(ctx, admitted, in, roWork{}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.inputs.PublicationEvidence(ctx, admitted); err != nil { |
| + t.Fatalf("P's receipt: %v", err) |
| + } |
| + moved := admitted |
| + c := *moved.Claim |
| + h := *c.Holder |
| + h.Generation = "g9" |
| + c.Holder = &h |
| + moved.Claim = &c |
| + if err := f.inputs.PublicationEvidence(ctx, moved); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("the holder moved off P: %v", err) |
| + } |
| + f.bucket.Put("data", markerKey(admitted.Claim.Holder), mustJSON(t, genselect.Marker{Op: "g9", Box: "dst"})) |
| + if err := f.inputs.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("another marker version: %v", err) |
| + } |
| +} |
| + |
| +// A later attempt's evidence re-selects the predecessor's baseline and |
| +// binds the admission to it: its position, the declared remote position, |
| +// and the exact files. |
| +func TestRelocationLaterAttemptEvidenceReselectsTheBaseline(t *testing.T) { |
| + ctx := t.Context() |
| + f := newROFixture(t, roOptions{}) |
| + first, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + opened := f.admit(f.read(), first) |
| + again, err := f.inputs.Select(ctx, f.route, opened, roProbe) |
| + if err != nil || again.Baseline == nil { |
| + t.Fatalf("the later attempt: %+v %v", again, err) |
| + } |
| + if err := f.inputs.AdmissionEvidence(ctx, opened, again.Admission, again.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + for name, mutate := range map[string]func(*hostclaim.Admission, *genselect.Plan){ |
| + "a stale position": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = 41, 41 }, |
| + "another declared loss": func(a *hostclaim.Admission, _ *genselect.Plan) { a.RemoteMaxTXID++ }, |
| + "other files": func(_ *hostclaim.Admission, p *genselect.Plan) { |
| + p.Files = append([]genselect.File{{Key: "x"}}, p.Files...) |
| + }, |
| + "the predecessor's name": func(_ *hostclaim.Admission, p *genselect.Plan) { p.Generation = "g1" }, |
| + } { |
| + a, p := again.Admission, *again.Baseline |
| + mutate(&a, &p) |
| + if err := f.inputs.AdmissionEvidence(ctx, opened, a, &p); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| +} |
| + |
| +// After the admission, P's opening keeps the install stamp: an admitted |
| +// holder at P with another stamp is refused even with the receipt whole. |
| +func TestRelocationAfterAdmissionNeedsTheInstallStampAtP(t *testing.T) { |
| + ctx := t.Context() |
| + f := newROFixture(t, roOptions{}) |
| + in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + dataDir := t.TempDir() |
| + path, _ := relocationInstallPath(dataDir, roOp) |
| + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + journal := []byte(`{"v":1,"host":"shop.example","op":"rel-1"}`) |
| + if err := os.WriteFile(path, journal, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + sum := sha256.Sum256(journal) |
| + f.update(func(c *hostclaim.Claim) error { |
| + var l hostclaim.LocalInstall |
| + _ = json.Unmarshal([]byte(c.Op.Journal["local_install"]), &l) |
| + l.JournalSHA256 = hex.EncodeToString(sum[:]) |
| + c.Op.Journal["local_install"] = string(mustJSON(t, l)) |
| + return nil |
| + }) |
| + prior := f.read() |
| + admitted := f.admit(prior, in) |
| + after := relocationAfterAdmission(dataDir, "dst") |
| + if err := after(ctx, prior, admitted); err != nil { |
| + t.Fatal(err) |
| + } |
| + moved := admitted |
| + c := *moved.Claim |
| + h := *c.Holder |
| + h.Stamp++ |
| + c.Holder = &h |
| + moved.Claim = &c |
| + if err := after(ctx, prior, moved); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("P at another stamp: %v", err) |
| + } |
| +} |
| + |
| +// The journal refuses once the claim no longer names this box the |
| +// executor at probing, unstuck; a reserved start is bound to the |
| +// admission's generation and stamp; a settle binds the exact reservation |
| +// this process made. |
| +func TestRelocationProbeAttemptsRefuseAMovedClaimOrJournal(t *testing.T) { |
| + ctx := t.Context() |
| + now := time.Date(2026, 10, 7, 4, 0, 0, 0, time.UTC) |
| + start := func(t *testing.T, f *roFixture, dataDir string) (*relocationProbeAttempts, OperationStart) { |
| + t.Helper() |
| + in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + admitted := f.admit(f.read(), in) |
| + h := admitted.Claim.Holder |
| + p := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: func() time.Time { return now }} |
| + owner := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "dst", Enrollment: strings.Repeat("a", 32), BootID: "boot", ControllerID: "ctl"} |
| + s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: owner} |
| + if err := p.Reserve(ctx, s, 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + return p, s |
| + } |
| + t.Run("stuck", func(t *testing.T) { |
| + f := newROFixture(t, roOptions{}) |
| + p, s := start(t, f, t.TempDir()) |
| + f.update(func(c *hostclaim.Claim) error { c.Op.Journal["stuck"] = "an operator"; return nil }) |
| + if err := p.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("a stuck operation: %v", err) |
| + } |
| + }) |
| + t.Run("another executor", func(t *testing.T) { |
| + // A journal bound for src, with src's record attempt counted, over |
| + // a claim whose executor is dst: the admitted writer still matches, |
| + // only the executor does not. |
| + f := newROFixture(t, roOptions{}) |
| + _, s := start(t, f, t.TempDir()) |
| + s.Admitted = f.update(func(c *hostclaim.Claim) error { c.Op.Journal["probe_attempts.src"] = "1"; return nil }) |
| + s.Owner.Box = "src" |
| + src := &relocationProbeAttempts{Claims: f.claims, DataDir: t.TempDir(), Box: "src", NewID: f.inputs.NewID, Now: func() time.Time { return now }} |
| + if err := src.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("another executor: %v", err) |
| + } |
| + }) |
| + for name, rewrite := range map[string]func(*relocationStart){ |
| + "another generation": func(st *relocationStart) { st.Generation = "g9" }, |
| + "another stamp": func(st *relocationStart) { st.Stamp++ }, |
| + } { |
| + t.Run("a journaled start under "+name, func(t *testing.T) { |
| + f := newROFixture(t, roOptions{}) |
| + dd := t.TempDir() |
| + p, s := start(t, f, dd) |
| + j, err := readRelocationAttempts(dd, roHost, roOp) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + st := j.Starts["1"] |
| + rewrite(&st) |
| + j.Starts["1"] = st |
| + if err := j.write(dd); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := p.Reserve(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + }) |
| + } |
| + t.Run("a settle of another reservation", func(t *testing.T) { |
| + f := newROFixture(t, roOptions{}) |
| + dd := t.TempDir() |
| + p, s := start(t, f, dd) |
| + j, _ := readRelocationAttempts(dd, roHost, roOp) |
| + st := j.Starts["1"] |
| + st.Attempt.ID = "someone-else" |
| + j.Starts["1"] = st |
| + if err := j.write(dd); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := p.Failed(ctx, s, 0); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("settled a reservation this process did not make: %v", err) |
| + } |
| + }) |
| +} |
| + |
| +// M1: the preflight refuses a claim whose pin, authority or probing start |
| +// is damaged before root is asked and before any attempt is counted; a |
| +// sound claim goes on to root. |
| +func TestRelocationPreflightRefusesADamagedClaimBeforeCounting(t *testing.T) { |
| + ctx := t.Context() |
| + preflight := (ClaimAssembly{Wake: &ClaimWakeAssembly{Roles: &ClaimRoleAssembly{}}}).relocationPreflight(RelocationAssemblyConfig{}) |
| + for name, damage := range map[string]func(*hostclaim.Claim){ |
| + "an unreadable pin": func(c *hostclaim.Claim) { c.Op.Journal["pin"] = `{"empty":` }, |
| + "an unreadable authority": func(c *hostclaim.Claim) { c.Op.Journal["prepared_authority"] = `{` }, |
| + "no probing start": func(c *hostclaim.Claim) { delete(c.Op.Journal, "probing_since") }, |
| + } { |
| + f := newROFixture(t, roOptions{}) |
| + damaged := f.update(func(c *hostclaim.Claim) error { damage(c); return nil }) |
| + if err := preflight(ctx, damaged); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| + f := newROFixture(t, roOptions{}) |
| + if err := preflight(ctx, f.read()); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { |
| + t.Fatalf("a sound claim went to root: %v", err) |
| + } |
| +} |
diff --git a/internal/host/relocationwake.go b/internal/host/relocationwake.go| index 57c2a5fdf..5d3ebdde1 100644 |
| --- a/internal/host/relocationwake.go |
| +++ b/internal/host/relocationwake.go |
| @@ -93,25 +93,6 @@ func preparedAuthority(c *hostclaim.Claim) (hostclaim.PreparedAuthority, error) |
| return auth, nil |
| } |
| -// journaledRelease is the operation's journaled release: absent, or |
| -// present and naming its releasing box. A present entry that does not |
| -// parse, or names no box, is an error, never "no release": an unreadable |
| -// entry is not absence, and absence is never evidence (Global Constraint). |
| -// Without this, a damaged release on a claim whose source is lost and |
| -// fenced would open P as a failover, uncertified, and declare the |
| -// source's writes lost. |
| -func journaledRelease(c *hostclaim.Claim) (hostclaim.Release, bool, error) { |
| - raw, ok := c.Op.Journal["release"] |
| - if !ok { |
| - return hostclaim.Release{}, false, nil |
| - } |
| - var rel hostclaim.Release |
| - if json.Unmarshal([]byte(raw), &rel) != nil || rel.ReleasedBy == "" { |
| - return hostclaim.Release{}, false, fmt.Errorf("%w: the journaled release is unreadable", hostbox.ErrChanged) |
| - } |
| - return rel, true, nil |
| -} |
| - |
| // emptyPrepared reports whether P is the evidenced empty bootstrap: the |
| // journaled pin says so (RecoveryPin.Empty, which the pinner admits only |
| // for a release reporting final_txid 0 or a holder with no writer) and the |
| @@ -165,12 +146,15 @@ func (r *RelocationWakeInputs) Select(ctx context.Context, rt registry.Route, o |
| adm.PredecessorRegion = h.Admissions[n-1].Region |
| } |
| if !hostclaim.PreparedOpened(c) { |
| - rel, released, err := journaledRelease(c) |
| + certified, err := hostrelocate.CertifiedFor(c) |
| if err != nil { |
| return ClaimLaunchInputs{}, err |
| } |
| - certified := released && hostrelocate.CertifiedFor(c) |
| if certified { |
| + var rel hostclaim.Release |
| + if err := json.Unmarshal([]byte(c.Op.Journal["release"]), &rel); err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| if err := r.Releases.VerifyRelease(ctx, *c, rel); err != nil { |
| return ClaimLaunchInputs{}, err |
| } |
| @@ -210,28 +194,19 @@ func (r *RelocationWakeInputs) Select(ctx context.Context, rt registry.Route, o |
| return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| } |
| plan, err := genselect.Select(ctx, r.Data, r.request(o)) |
| - switch { |
| - case errors.Is(err, genselect.ErrEmpty): |
| - // Nothing under the location is an empty predecessor only over an |
| - // empty P: a pinned snapshot is in the location, so an empty |
| - // listing under a non-empty pin is never evidence. |
| - auth, err := preparedAuthority(c) |
| - if err != nil { |
| - return ClaimLaunchInputs{}, err |
| - } |
| - empty, err := emptyPrepared(c, auth) |
| - if err != nil { |
| - return ClaimLaunchInputs{}, err |
| - } |
| - if !empty { |
| - return ClaimLaunchInputs{}, fmt.Errorf("%w: the location lists nothing under a pinned snapshot", hostbox.ErrChanged) |
| - } |
| - in.Empty = true |
| - case err != nil: |
| + if errors.Is(err, genselect.ErrEmpty) { |
| + // A later attempt follows P's admitted session, which ran: a |
| + // listing that finds nothing is absence (nothing replicated, or |
| + // not yet visible), never an empty predecessor. Main's serving |
| + // rule admits an empty cold admission only for a holder that never |
| + // admitted a writer (claimEmptyBootstrap), which a later attempt |
| + // never is (fix round 1, I2). |
| + return ClaimLaunchInputs{}, fmt.Errorf("%w: the location lists nothing after P's admitted session", hostbox.ErrChanged) |
| + } |
| + if err != nil { |
| return ClaimLaunchInputs{}, err |
| - default: |
| - adm.RemoteMaxTXID, in.Baseline = plan.MaxTXID, plan |
| } |
| + adm.RemoteMaxTXID, in.Baseline = plan.MaxTXID, plan |
| in.Admission = adm |
| return in, nil |
| } |
| @@ -304,7 +279,7 @@ func (r *RelocationWakeInputs) preparedMarker(ctx context.Context, o hostclaim.O |
| return ClaimPublicationReceipt{}, fmt.Errorf("%w: install.json is not the prepared authority's", hostbox.ErrChanged) |
| } |
| root := genselect.File{Key: im.SnapshotKey, VersionID: im.SnapshotVersion, Level: 9, MinTXID: 1, MaxTXID: im.N} |
| - current, err := (&movesProbeOwners{Data: r.Data, Bucket: r.Bucket}).currentMarker(bounded, h) // moves' reader, one implementation |
| + current, err := currentMarker(bounded, r.Data, r.Bucket, h) // moves' reader, one implementation |
| if err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| @@ -343,10 +318,12 @@ func (r *RelocationWakeInputs) preparedMarker(ctx context.Context, o hostclaim.O |
| return ClaimPublicationReceipt{MarkerVersion: version, Root: root}, nil |
| } |
| -// EmptyAuthority: P's opening is empty only with an evidenced empty pin; |
| -// a later attempt over an empty predecessor is an ordinary empty cold |
| -// admission (a complete listing of the location holds nothing). |
| -func (r *RelocationWakeInputs) EmptyAuthority(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| +// EmptyAuthority: P's opening is empty only with an evidenced empty pin |
| +// (the pinner's RecoveryPin.Empty, which the prepared authority agrees |
| +// with). A later attempt is never empty: it follows P's admitted session, |
| +// and main's serving rule admits an empty cold admission only for a holder |
| +// that never admitted a writer (fix round 1, I2). |
| +func (r *RelocationWakeInputs) EmptyAuthority(_ context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| if err := r.probe(o, activator.StartPolicy{Probe: true}); err != nil { |
| return err |
| } |
| @@ -361,23 +338,11 @@ func (r *RelocationWakeInputs) EmptyAuthority(ctx context.Context, o hostclaim.O |
| if err != nil { |
| return err |
| } |
| - if in.Admission.NewGeneration == o.Claim.Op.Journal["prepared"] { |
| - if !empty { |
| - return fmt.Errorf("%w: no evidenced empty pin", hostrelocate.ErrRefused) |
| - } |
| - return nil |
| + if in.Admission.NewGeneration != o.Claim.Op.Journal["prepared"] || hostclaim.PreparedOpened(o.Claim) { |
| + return fmt.Errorf("%w: a later attempt follows P's admitted session and is never empty", hostbox.ErrChanged) |
| } |
| if !empty { |
| - // A later attempt over a non-empty P always has a baseline: P's |
| - // snapshot is in the location. |
| - return fmt.Errorf("%w: an empty later attempt under a pinned snapshot", hostbox.ErrChanged) |
| - } |
| - versions, err := r.Data.ListVersions(ctx, r.Bucket, strings.TrimSuffix(o.Claim.Holder.Location, "/")+"/") |
| - if err != nil { |
| - return err |
| - } |
| - if len(versions) != 0 { |
| - return hostbox.ErrChanged |
| + return fmt.Errorf("%w: no evidenced empty pin", hostrelocate.ErrRefused) |
| } |
| return nil |
| } |
| @@ -443,10 +408,11 @@ func (r *RelocationWakeInputs) AdmissionEvidence(ctx context.Context, o hostclai |
| return err |
| } |
| if in.NewGeneration == prepared { |
| - if _, _, err := journaledRelease(c); err != nil { |
| + certified, err := hostrelocate.CertifiedFor(c) |
| + if err != nil { |
| return err |
| } |
| - if !in.KeepStamp || hostclaim.PreparedOpened(c) || in.PredecessorCertified != hostrelocate.CertifiedFor(c) { |
| + if !in.KeepStamp || hostclaim.PreparedOpened(c) || in.PredecessorCertified != certified { |
| return hostbox.ErrChanged |
| } |
| raw, _, err := r.FilesStore.Get(ctx, hostfiles.ManifestKey(h.Location+"/files/"+prepared+"/")) |
| @@ -471,12 +437,9 @@ func (r *RelocationWakeInputs) AdmissionEvidence(ctx context.Context, o hostclai |
| return hostbox.ErrChanged |
| } |
| if baseline == nil { |
| - // An empty predecessor, proved by EmptyAuthority, and possible |
| - // only over an empty P. |
| - if !empty || in.RemoteMaxTXID != 0 { |
| - return hostbox.ErrChanged |
| - } |
| - return nil |
| + // A later attempt follows P's admitted session and is never an |
| + // empty cold admission (fix round 1, I2). |
| + return hostbox.ErrChanged |
| } |
| fresh, err := genselect.Select(ctx, r.Data, r.request(o)) |
| if err != nil { |
| @@ -502,7 +465,7 @@ func (r *RelocationWakeInputs) PublicationEvidence(ctx context.Context, o hostcl |
| if !prepared { |
| return r.freshOwners().freshEvidence(ctx, o) |
| } |
| - current, err := (&movesProbeOwners{Data: r.Data, Bucket: r.Bucket}).currentMarker(ctx, h) |
| + current, err := currentMarker(ctx, r.Data, r.Bucket, h) |
| if err != nil { |
| return err |
| } |
diff --git a/internal/hostrelocate/participant.go b/internal/hostrelocate/participant.go| index 431b6d53b..3901cef7f 100644 |
| --- a/internal/hostrelocate/participant.go |
| +++ b/internal/hostrelocate/participant.go |
| @@ -213,6 +213,15 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| } |
| obs, c = fresh, fresh.Claim |
| } |
| + // The release the admission would certify is read before any |
| + // attempt is counted (review I3, M1): a journaled release that is |
| + // unreadable, or names another writer than the predecessor, is a |
| + // claim defect no attempt can mend, so the operation is stuck for |
| + // an operator and no attempt is spent on it. |
| + certified, err := CertifiedFor(c) |
| + if err != nil { |
| + return p.update(ctx, obs, hostclaim.MarkStuck(id, "the journaled release cannot certify P's opening (§5.8, C4): "+err.Error())) |
| + } |
| // Root must be able to authorize a relocating launch before an |
| // attempt is spent (ruling 48): a refusal counts nothing. A later |
| // attempt root cannot launch (ruling 58, until claim ask C11) is |
| @@ -232,7 +241,7 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| return obs, err |
| } |
| obs, c = counted, counted.Claim |
| - session, passed, err := p.Effects.Probe(ctx, obs, CertifiedFor(c)) |
| + session, passed, err := p.Effects.Probe(ctx, obs, certified) |
| if err != nil || !passed { |
| return obs, errors.Join(ErrNotYet, err) |
| } |
| @@ -307,27 +316,48 @@ func (p Participant) Sweep(ctx context.Context) error { |
| return errors.Join(errs...) |
| } |
| +// ErrRelease is a journaled release that is present but unreadable, or |
| +// that names another writer than the holder's predecessor. Either is a |
| +// claim defect, never "no release": an uncertified cold opening declares |
| +// the predecessor's writes lost, which only a failover's evidence (the |
| +// source lost and fenced, with no release at all) may do. |
| +var ErrRelease = errors.New("hostrelocate: the journaled release is unreadable or does not name the predecessor") |
| + |
| // CertifiedFor is whether the probe admission certifies its predecessor |
| // (§5.8, L9): only the admission that opens P, when this operation |
| // journaled a release whose session is the predecessor writer's. Claim |
| // Plan 11 gives a relocation no coverage value of its own, so the release |
| // is the generic PredecessorCertified, which AdmitWriter accepts on P's |
| // opening only with that release and a pin reaching it |
| -// (relocationProbeOpening). Otherwise false: a failover, a later attempt, |
| -// or a host that never had a writer is an uncertified cold admission. |
| -func CertifiedFor(c *hostclaim.Claim) bool { |
| +// (relocationProbeOpening). No release at all (a failover, or a host that |
| +// never had a writer) is an uncertified cold admission; after P's opening |
| +// every later attempt is one too. A release that is present but |
| +// unreadable, or that names another writer than the predecessor, is |
| +// ErrRelease: absence is never evidence, and an unreadable or mismatched |
| +// entry is not absence (fix round 1, I3). |
| +func CertifiedFor(c *hostclaim.Claim) (bool, error) { |
| + raw, ok := c.Op.Journal["release"] |
| + if !ok { |
| + return false, nil |
| + } |
| var r hostclaim.Release |
| - if json.Unmarshal([]byte(c.Op.Journal["release"]), &r) != nil || r.ReleasedBy == "" { |
| - return false |
| + if json.Unmarshal([]byte(raw), &r) != nil || r.ReleasedBy == "" { |
| + return false, fmt.Errorf("%w: unreadable", ErrRelease) |
| } |
| if hostclaim.PreparedOpened(c) { |
| - return false |
| + return false, nil |
| + } |
| + if c.Holder == nil { |
| + return false, fmt.Errorf("%w: no holder", ErrRelease) |
| } |
| pred := hostclaim.AdmissionRecord{Session: c.Holder.Session, Box: c.Holder.WriterBox, Generation: c.Holder.Generation} |
| if n := len(c.Holder.Admissions); n > 0 { |
| pred = c.Holder.Admissions[n-1] |
| } |
| - return pred.Box == r.ReleasedBy && pred.Session == r.Session && pred.Generation == r.Generation |
| + if pred.Box != r.ReleasedBy || pred.Session != r.Session || pred.Generation != r.Generation { |
| + return false, fmt.Errorf("%w: released by %s (session %d, generation %q), the predecessor is %s (session %d, generation %q)", ErrRelease, r.ReleasedBy, r.Session, r.Generation, pred.Box, pred.Session, pred.Generation) |
| + } |
| + return true, nil |
| } |
| func (p Participant) update(ctx context.Context, obs hostclaim.Observed, tr func(*hostclaim.Claim) error) (hostclaim.Observed, error) { |
diff --git a/internal/hostrelocate/participant_gate_test.go b/internal/hostrelocate/participant_gate_test.go| index ddec620a5..0f315899a 100644 |
| --- a/internal/hostrelocate/participant_gate_test.go |
| +++ b/internal/hostrelocate/participant_gate_test.go |
| @@ -2,6 +2,7 @@ package hostrelocate |
| import ( |
| "context" |
| + "encoding/json" |
| "errors" |
| "slices" |
| "testing" |
| @@ -210,3 +211,53 @@ func TestSameInstallationRefusesAMovedInstallation(t *testing.T) { |
| } |
| }) |
| } |
| + |
| +// Fix round 1, I3 and M1: a journaled release that is unreadable, or that |
| +// names another writer than the predecessor, is a claim defect. The |
| +// participant reads it before any attempt is counted: the operation is |
| +// stuck for an operator, no attempt is spent, and no probe is made. |
| +func TestParticipantRefusesADamagedReleaseBeforeCountingAnAttempt(t *testing.T) { |
| + ctx := context.Background() |
| + for name, damage := range map[string]func(*hostclaim.Claim) error{ |
| + "unparseable": func(c *hostclaim.Claim) error { c.Op.Journal["release"] = `{"released_by":`; return nil }, |
| + "another session": func(c *hostclaim.Claim) error { |
| + var rel hostclaim.Release |
| + if err := json.Unmarshal([]byte(c.Op.Journal["release"]), &rel); err != nil { |
| + return err |
| + } |
| + rel.Session++ |
| + raw, err := json.Marshal(rel) |
| + c.Op.Journal["release"] = string(raw) |
| + return err |
| + }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + f := newFixture(t) |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + if _, err := hostclaim.Update(ctx, f.s, f.claim(), damage); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := pd.Step(ctx, host); err != nil { |
| + t.Fatal(err) |
| + } |
| + c := f.claim().Claim |
| + if c.Op.Journal["stuck"] == "" || c.Op.Journal["probe_attempts.dst"] != "" || len(dst.certified) != 0 { |
| + t.Fatalf("stuck %q attempts %q probes %d", c.Op.Journal["stuck"], c.Op.Journal["probe_attempts.dst"], len(dst.certified)) |
| + } |
| + }) |
| + } |
| + // A sound release certifies P's opening, and only it. |
| + f := newFixture(t) |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + if certified, err := CertifiedFor(f.claim().Claim); err != nil || !certified { |
| + t.Fatalf("a drain's release: %v %v", certified, err) |
| + } |
| + if _, err := pd.Step(ctx, host); err != nil && !errors.Is(err, ErrNotYet) { |
| + t.Fatal(err) |
| + } |
| + if certified, err := CertifiedFor(f.claim().Claim); err != nil || certified { |
| + t.Fatalf("after P's opening: %v %v", certified, err) |
| + } |
| +} |