amadan
edge: fair share over backhaul slots so one client cannot fill a box's hop (#358)
While a destination box's slots are at least half taken, a client key already holding 8 of them is answered 429 with Retry-After: 1, the same answer the home edge gives. Saturation still answers 503 with Retry-After: 2. Door callers, fleet-tunnelled requests and unkeyable addresses take no part; observe mode logs "would refuse" and admits.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+361
−13
internal/edge/backhaul_share_test.go+268 −0internal/edge/edge.go+91 −11internal/edge/homemap_test.go+2 −2
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok amadan.net/carlos/platform.git/launchctx 0.012s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (18 s)
=== gofmt (1 s)
gofmt -l .: clean
=== rest (171 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.660s
ok github.com/carlosframework/platform/cmd/moveprobe 11.985s
ok github.com/carlosframework/platform/cmd/zddprobe 0.734s
ok github.com/carlosframework/platform/internal/activator 58.049s
ok github.com/carlosframework/platform/internal/agentsock 0.038s
ok github.com/carlosframework/platform/internal/alerting 0.005s
ok github.com/carlosframework/platform/internal/analytics 0.039s
ok github.com/carlosframework/platform/internal/appconfig 3.075s
ok github.com/carlosframework/platform/internal/apperr 0.129s
ok github.com/carlosframework/platform/internal/applog 0.226s
ok github.com/carlosframework/platform/internal/awsinv 0.018s
ok github.com/carlosframework/platform/internal/boxchoice 0.016s
ok github.com/carlosframework/platform/internal/boxid 0.756s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.026s
ok github.com/carlosframework/platform/internal/caps 0.028s
ok github.com/carlosframework/platform/internal/certwarm 0.023s
ok github.com/carlosframework/platform/internal/cgroup 0.823s
ok github.com/carlosframework/platform/internal/changelog 0.517s
ok github.com/carlosframework/platform/internal/claimactivation 1.266s
ok github.com/carlosframework/platform/internal/clamd 0.059s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 8.223s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.031s
ok github.com/carlosframework/platform/internal/dbquery 0.006s
ok github.com/carlosframework/platform/internal/dbreads 2.160s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.009s
ok github.com/carlosframework/platform/internal/dnsrec 0.009s
ok github.com/carlosframework/platform/internal/docsite 0.031s
ok github.com/carlosframework/platform/internal/domains 2.940s
ok github.com/carlosframework/platform/internal/doorkey 0.022s
ok github.com/carlosframework/platform/internal/economics 0.022s
ok github.com/carlosframework/platform/internal/edge 46.175s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.133s
ok github.com/carlosframework/platform/internal/gate 1.654s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.016s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.152s
ok github.com/carlosframework/platform/internal/host 142.846s
ok github.com/carlosframework/platform/internal/hostactors 5.407s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.006s
ok github.com/carlosframework/platform/internal/hostbox 2.899s
ok github.com/carlosframework/platform/internal/hostclaim 0.722s
ok github.com/carlosframework/platform/internal/hostenv 0.007s
ok github.com/carlosframework/platform/internal/hostevidence 0.006s
ok github.com/carlosframework/platform/internal/hostexec 5.308s
ok github.com/carlosframework/platform/internal/hostfence 0.061s
ok github.com/carlosframework/platform/internal/hostfiles 10.375s
ok github.com/carlosframework/platform/internal/hostfloor 0.019s
ok github.com/carlosframework/platform/internal/hostguard 1.017s
ok github.com/carlosframework/platform/internal/hostops 1.933s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.002s
ok github.com/carlosframework/platform/internal/hostquarantine 0.002s
ok github.com/carlosframework/platform/internal/hostrelocate 0.609s
ok github.com/carlosframework/platform/internal/hostresolve 0.173s
ok github.com/carlosframework/platform/internal/hostwrite 0.002s
ok github.com/carlosframework/platform/internal/infraview 0.006s
ok github.com/carlosframework/platform/internal/instancectl 5.227s
ok github.com/carlosframework/platform/internal/instanceexec 2.806s
ok github.com/carlosframework/platform/internal/instancefs 0.013s
ok github.com/carlosframework/platform/internal/instancekey 0.017s
ok github.com/carlosframework/platform/internal/instanceupdates 0.013s
ok github.com/carlosframework/platform/internal/layout 1.743s
ok github.com/carlosframework/platform/internal/lbhealth 0.007s
ok github.com/carlosframework/platform/internal/lease 10.114s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.503s
ok github.com/carlosframework/platform/internal/ledger 0.016s
ok github.com/carlosframework/platform/internal/mail 0.215s
ok github.com/carlosframework/platform/internal/maintenance 0.098s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.614s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.094s
ok github.com/carlosframework/platform/internal/metrics 0.005s
ok github.com/carlosframework/platform/internal/moves 22.543s
ok github.com/carlosframework/platform/internal/moves/archive 1.015s
ok github.com/carlosframework/platform/internal/objstore 0.271s
ok github.com/carlosframework/platform/internal/ops 0.204s
ok github.com/carlosframework/platform/internal/park 0.287s
ok github.com/carlosframework/platform/internal/plans 0.007s
ok github.com/carlosframework/platform/internal/privatefile 0.003s
2026/10/24 21:47:11 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:11 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:11 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:11 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 21:47:11 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:11 provision: an app's records could not be listed: they are skipped
2026/10/24 21:47:11 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:11 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:47:11 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:47:11 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:47:11 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:47:11 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:47:11 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:47:11 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:47:11 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:47:11 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:47:11 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:11 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:47:11 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:47:12 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:47:12 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:47:12 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:47:12 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:47:12 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:47:12 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:47:12 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:47:12 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:12 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:12 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:12 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:47:12 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:47:12 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 21:47:12 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 21:47:12 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:12 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:12 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.76s)
plan9_b8_census_test.go:372: record-write census rejected 1 site(s); update the explicit inventory only after reviewing the source:
changed site internal/provision/reconcile.go:Reconcile: source differs from the reviewed digest
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:47:14 provision: minted instance route shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:14 provision: minted instance route shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 21:47:14 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 21:47:14 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 21:47:14 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 21:47:14 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:47:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:47:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:47:14 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:47:14 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 21:47:14 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:47:14 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:47:14 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:47:14 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:47:14 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:14 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:47:14 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 21:47:14 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 21:47:14 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:14 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:47:14 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:14 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 21:47:14 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:47:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:14 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:47:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:14 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:47:14 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:14 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:14 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 21:47:14 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:47:14 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 21:47:14 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:14 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:47:14 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 21:47:14 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:47:14 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 21:47:14 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 21:47:14 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 21:47:14 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 21:47:14 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:14 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:47:14 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 21:47:14 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 21:47:14 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 21:47:14 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 21:47:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:14 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 21:47:15 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 21:47:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 21:47:15 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 21:47:15 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 21:47:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:15 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:47:15 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:47:15 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:47:15 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:47:15 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 21:47:15 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:47:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:47:15 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:47:15 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:47:15 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 21:47:15 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 21:47:15 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:47:15 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 21:47:15 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 21:47:15 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:47:15 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 21:47:15 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:47:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:15 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:16 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:16 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:16 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:47:16 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:16 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:47:16 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 21:47:16 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 21:47:16 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:47:16 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:47:16 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:16 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:16 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:16 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:47:16 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:16 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:47:16 provision: a.shop.example.com: swapped for import sd_azo2rmmguobouykwklvnshnloa (the original route is journaled)
2026/10/24 21:47:16 provision: a.shop.example.com: import sd_azo2rmmguobouykwklvnshnloa committed: its gate is cleared and its journal row dropped
2026/10/24 21:47:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_bqckdywwbotb2ypkdrkkm7exni rolled back to its original route
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_bqckdywwbotb2ypkdrkkm7exni: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_bqckdywwbotb2ypkdrkkm7exni is cleaned up and its journal row retired
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_v3lbus6h3tnrwdfpyq4yx5nyru rolled back to its original route
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_v3lbus6h3tnrwdfpyq4yx5nyru: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_v3lbus6h3tnrwdfpyq4yx5nyru is cleaned up and its journal row retired
2026/10/24 21:47:17 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_a7myaukzxd4xjbymw22f4szy5e
2026/10/24 21:47:17 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 21:47:17 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 21:47:17 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 21:47:17 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 21:47:17 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_t2vesxsdt2i53rfvobxhewvvuu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_2rncfvuwvxk2yfsigbrm2izsiu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_nntcuk5w3tpa4yraxjyjmj637i is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_y7nzw273krnvdadsouy4o5uewa is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_lyjq76vzvgv45u54tt2g47wdj4 is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:47:17 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 21:47:17 provision: b.shop.example.com: import sd_xzphs4aarcutv6pul3sozl6xnq left alone this pass: the import journal could not be read
2026/10/24 21:47:17 provision: c.shop.example.com: import sd_rx7d7km5dk4zrrt4rgl6e3cuk4 left alone this pass: the import journal could not be read
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_da5l7a4skeuxwxrfsrx7vxf2mi committed: its gate is cleared and its journal row dropped
2026/10/24 21:47:17 provision: a.shop.example.com: not converging import sd_y3kcuwzwpywd5crnc36hnxobka: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:47:17 provision: a.shop.example.com: not converging import sd_v3bohccjrhxualorzw6a4rqsmm: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:47:17 provision: a.shop.example.com: not converging import sd_yztushdggoxq2gr475iu4dfwam: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:47:17 provision: a.shop.example.com: not converging import sd_noiljettjom5hcq4cyv7j7im3q: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:47:17 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 21:47:17 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_on3on6k2mvszrkilya6jo5e5uu committed: its gate is cleared and its journal row dropped
2026/10/24 21:47:17 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:17 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:17 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 21:47:17 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_fy6ovono7y4zvx4eykzm4rn2zq: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_fy6ovono7y4zvx4eykzm4rn2zq rolled back (no box had swapped it)
2026/10/24 21:47:17 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 21:47:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:17 provision: removed instance route acme.go.tito.io (its seed sd_adaupj2bsu6l5janvy32bvgd3e was aborted); the next pass rolls it back
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_adaupj2bsu6l5janvy32bvgd3e: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_adaupj2bsu6l5janvy32bvgd3e rolled back (no box had swapped it)
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_ze7lbv5bifq4riufcgkbqcbitu has a row this pass may not remove; leaving the seed aborted
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_d4u3lzfdapr66gd3yfyocntgle rolled back (no box had swapped it)
2026/10/24 21:47:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:17 provision: removed instance route acme.go.tito.io (its seed sd_uzlvvhev5cu27nj554lzhstism was aborted); the next pass rolls it back
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_uzlvvhev5cu27nj554lzhstism has no route on this box: its local data is removed
2026/10/24 21:47:17 provision: acme.go.tito.io: aborted seed sd_uzlvvhev5cu27nj554lzhstism rolled back (no box had swapped it)
2026/10/24 21:47:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:17 provision: a.shop.example.com: swapped for import sd_pikcmjjgli6nlh3cqfpo2irqwa (the original route is journaled)
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_pikcmjjgli6nlh3cqfpo2irqwa rolled back to its original route
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_pikcmjjgli6nlh3cqfpo2irqwa is cleaned up and its journal row retired
2026/10/24 21:47:17 provision: a.shop.example.com: swapped for import sd_ddaimkqpydnva34eewotoogv6q (the original route is journaled)
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_ddaimkqpydnva34eewotoogv6q rolled back to its original route
2026/10/24 21:47:17 provision: a.shop.example.com: import sd_ddaimkqpydnva34eewotoogv6q is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_l7t7nnbmglkm75y5il25lrtn6m (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_l7t7nnbmglkm75y5il25lrtn6m rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_l7t7nnbmglkm75y5il25lrtn6m: the host's replica is left alone: its record no longer names this seed
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_l7t7nnbmglkm75y5il25lrtn6m is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_sthxzekkm26644jeueryc6taeu (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sthxzekkm26644jeueryc6taeu rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sthxzekkm26644jeueryc6taeu rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sthxzekkm26644jeueryc6taeu is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_joscp3zynvdxrgve5uui3yk3ka (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_joscp3zynvdxrgve5uui3yk3ka rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_yq5roswlbococvnxenmfj4vvvi (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_yq5roswlbococvnxenmfj4vvvi is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_o2vxso66vas67hzcksvegh4hxq (the original route is journaled)
2026/10/24 21:47:18 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_o2vxso66vas67hzcksvegh4hxq left alone this pass: its record could not be read
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_lcyxsbhjmgrnyhihy5xehwuzja (the original route is journaled)
2026/10/24 21:47:18 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_lcyxsbhjmgrnyhihy5xehwuzja left alone this pass: its record could not be read
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_m66enpcmi43lubs3cffra4alhq (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_m66enpcmi43lubs3cffra4alhq: close the import gate: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_m66enpcmi43lubs3cffra4alhq rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_m66enpcmi43lubs3cffra4alhq is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_dkf7po3yhinx22lrcsc6vzuv2u (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_dkf7po3yhinx22lrcsc6vzuv2u: close the import gate: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_dkf7po3yhinx22lrcsc6vzuv2u rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_dkf7po3yhinx22lrcsc6vzuv2u is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_sggv57ex7ajdpwmprkbv5vx3g4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sggv57ex7ajdpwmprkbv5vx3g4 rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sggv57ex7ajdpwmprkbv5vx3g4 rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_sggv57ex7ajdpwmprkbv5vx3g4 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_lf6q4lagxsb4am23mzqqrdyob4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_lf6q4lagxsb4am23mzqqrdyob4 rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_lf6q4lagxsb4am23mzqqrdyob4 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_t6joup5ax2a7svbxxovtwzgvuy (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_t6joup5ax2a7svbxxovtwzgvuy: retire its instance key: injected failure at retire-key
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_t6joup5ax2a7svbxxovtwzgvuy rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_t6joup5ax2a7svbxxovtwzgvuy is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_455qu7ft45meyioqwx5xfwpr6m (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_455qu7ft45meyioqwx5xfwpr6m rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_455qu7ft45meyioqwx5xfwpr6m is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_c65mu636x2wm3y3zc7dggsnakm (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_c65mu636x2wm3y3zc7dggsnakm: remove its local data: injected failure at remove-local
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_c65mu636x2wm3y3zc7dggsnakm rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_c65mu636x2wm3y3zc7dggsnakm is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_ybuu3e6b4uouatvzbznm55maym (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ybuu3e6b4uouatvzbznm55maym rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ybuu3e6b4uouatvzbznm55maym is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_h6ia5loefsxz2xvlnp3msmcmcq (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_h6ia5loefsxz2xvlnp3msmcmcq: remove its local data: injected failure at remove-local-done
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_h6ia5loefsxz2xvlnp3msmcmcq rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_h6ia5loefsxz2xvlnp3msmcmcq is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_ktradbeaoofn4pb6y7274uh5wy (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ktradbeaoofn4pb6y7274uh5wy rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ktradbeaoofn4pb6y7274uh5wy is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_gtasmeyn27bpocluizuyivq2rm (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_gtasmeyn27bpocluizuyivq2rm: restore the original route: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_gtasmeyn27bpocluizuyivq2rm rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_gtasmeyn27bpocluizuyivq2rm is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_oxfhsmxmgngg7hpeyyf5youc6m (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_oxfhsmxmgngg7hpeyyf5youc6m: restore the original route: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_oxfhsmxmgngg7hpeyyf5youc6m rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_oxfhsmxmgngg7hpeyyf5youc6m is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_dmgymi7yvnqaugdegvyxkg2bzi (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_dmgymi7yvnqaugdegvyxkg2bzi is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_smd7rtotxznk6kg4tuluwys4h4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_smd7rtotxznk6kg4tuluwys4h4 rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_smd7rtotxznk6kg4tuluwys4h4: delete its snapshot: injected failure at delete-snapshot
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_smd7rtotxznk6kg4tuluwys4h4 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_tj5os4hvaowxuf3g5gssdmqwyu (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_tj5os4hvaowxuf3g5gssdmqwyu rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_tj5os4hvaowxuf3g5gssdmqwyu is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_wlb67ckcdxavva4nowz4qpxmne (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_wlb67ckcdxavva4nowz4qpxmne rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_wlb67ckcdxavva4nowz4qpxmne: delete the host's replica: injected failure at delete-replica
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_wlb67ckcdxavva4nowz4qpxmne is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_oksnyug4s4fhtuchery6ow56ia (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_oksnyug4s4fhtuchery6ow56ia rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_oksnyug4s4fhtuchery6ow56ia is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_cvrjg2dpritaua6cwlzkcab5km (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_cvrjg2dpritaua6cwlzkcab5km rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_cvrjg2dpritaua6cwlzkcab5km: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_cvrjg2dpritaua6cwlzkcab5km is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_47cml4r2bixzhegi2esakdlhz4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_47cml4r2bixzhegi2esakdlhz4 rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_47cml4r2bixzhegi2esakdlhz4: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_47cml4r2bixzhegi2esakdlhz4 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_nglynikpqaf5xy5rtqheqxyxzi (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_nglynikpqaf5xy5rtqheqxyxzi rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_nglynikpqaf5xy5rtqheqxyxzi is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: mark import sd_nglynikpqaf5xy5rtqheqxyxzi rolled back: injected failure at cas
2026/10/24 21:47:18 provision: a.shop.example.com: aborted seed sd_nglynikpqaf5xy5rtqheqxyxzi rolled back (no box had swapped it)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_ru5r5urufxe5xf6dyhbkk6syuy (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ru5r5urufxe5xf6dyhbkk6syuy rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_ru5r5urufxe5xf6dyhbkk6syuy is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: aborted seed sd_ru5r5urufxe5xf6dyhbkk6syuy rolled back (no box had swapped it)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_urkh5g4ouq2qc5jkdk5lirhx2a (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_urkh5g4ouq2qc5jkdk5lirhx2a rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_urkh5g4ouq2qc5jkdk5lirhx2a rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_urkh5g4ouq2qc5jkdk5lirhx2a rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_urkh5g4ouq2qc5jkdk5lirhx2a is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_zemuciefgryngzt3ux6fcdhjhu (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_zemuciefgryngzt3ux6fcdhjhu rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_zemuciefgryngzt3ux6fcdhjhu is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_4j3jawsqfjjqkuzyyaaqhmy3ki (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_4j3jawsqfjjqkuzyyaaqhmy3ki rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_4j3jawsqfjjqkuzyyaaqhmy3ki is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_gnz63rrtowsrxihiifqm4zk7pm (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_gnz63rrtowsrxihiifqm4zk7pm rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_gnz63rrtowsrxihiifqm4zk7pm is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_3jcegkti7sim7ny2aqilzn6x2u (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_3jcegkti7sim7ny2aqilzn6x2u rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_3jcegkti7sim7ny2aqilzn6x2u is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_35ab4izavgpkrpasrwnzktot2q (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_35ab4izavgpkrpasrwnzktot2q rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_35ab4izavgpkrpasrwnzktot2q is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_jfxdu7kkokmp3qd5457xvlgp64 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_jfxdu7kkokmp3qd5457xvlgp64 rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_jfxdu7kkokmp3qd5457xvlgp64 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_5wfgthk4ymyp3iqi3uodithxmu (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_5wfgthk4ymyp3iqi3uodithxmu rolled back to its original route
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_5wfgthk4ymyp3iqi3uodithxmu is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_4y5gdaxmpgohffettaht53o7ym (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_4y5gdaxmpgohffettaht53o7ym has no route on this box: its local data is removed and no route is put back
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_4y5gdaxmpgohffettaht53o7ym is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_smfxklht7edzpnivemnismfvu4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_otjnjjoticr2dmlt63tkoyvygq (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_hw7tejiuqymjghr6mqigd55wbi (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_hw7tejiuqymjghr6mqigd55wbi cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_xhpe2ygx2t5lkot764xzenfofy (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_xhpe2ygx2t5lkot764xzenfofy is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 21:47:18 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 21:47:18 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_2rylqdiylpvrapvk52wu534bne (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_gsiviv22yyve5jkql6aucx55zu (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_hmmtigfm5zc2qfrdrtiro6ozjm (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_77togni4pvrcmlp5rgyv6sfa5m (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_jt3ciwapgb2iq72oskh2b2o64e (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_jt3ciwapgb2iq72oskh2b2o64e rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_tpeitqvkbh75m4b2eerl2dhh24 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_tpeitqvkbh75m4b2eerl2dhh24 has no route on this box: its local data is removed and no route is put back
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_tpeitqvkbh75m4b2eerl2dhh24 is cleaned up and its journal row retired
2026/10/24 21:47:18 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_3gjk2szte3kpyfc6p3mznyrmui (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_3gjk2szte3kpyfc6p3mznyrmui: remove its local data: injected failure at remove-local
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_3gjk2szte3kpyfc6p3mznyrmui is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_2nuhrofxeqmqntt3cdnoqml4mq (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_2nuhrofxeqmqntt3cdnoqml4mq: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 21:47:18 provision: a.shop.example.com: swapped for import sd_hynmfxk47qhnbyyy6ydqvhrbm4 (the original route is journaled)
2026/10/24 21:47:18 provision: a.shop.example.com: import sd_hynmfxk47qhnbyyy6ydqvhrbm4 rollback deferred: no fence generation: registry busy
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_sy42nwazepfxqyhf4wug6qwlsi (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_sy42nwazepfxqyhf4wug6qwlsi rollback deferred: no fence generation: the generation source answered 0
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_z5z53ercqcwkui3kdmgxgnvwi4 (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_235rz4fsctaz5shhex56vd6dne (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_dpuoj3p7ww4rfogy4rvh7egtbq (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_om35atjlowcghhv7dkzoifbfce (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_om35atjlowcghhv7dkzoifbfce rolled back to its original route
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_om35atjlowcghhv7dkzoifbfce is cleaned up and its journal row retired
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_4vqzfqjtgntzigctjfsyvbdo7q (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4vqzfqjtgntzigctjfsyvbdo7q rolled back to its original route
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4vqzfqjtgntzigctjfsyvbdo7q is cleaned up and its journal row retired
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_sjsa64mgryr56jqh26pb65w6wi (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_sjsa64mgryr56jqh26pb65w6wi rolled back to its original route
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_sjsa64mgryr56jqh26pb65w6wi is cleaned up and its journal row retired
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_4aiitiuwz5f5vv4mnnchjpk2aq (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4aiitiuwz5f5vv4mnnchjpk2aq: retire its instance key: the env file is a symlink
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4aiitiuwz5f5vv4mnnchjpk2aq rolled back to its original route
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4aiitiuwz5f5vv4mnnchjpk2aq is cleaned up and its journal row retired
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_oaiddf4yejk67vmdomgklfmj2u (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_jj2u5idtwxz7jgjh2ryxn3ugse (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_jaljrhoz5f5n4wmpxkgbrclayu refused: there is no route at a.shop.example.com to replace
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_xnegzylctr5l35vthdmnnqe2dy refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_3es6c25rolbk4zei5nz5dsudeu refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_2yq4ewn5n35pz4xgmrgyxdhybu refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_w6kscghs4qix7q5yyngjp3wpdu refused: the route belongs to app other
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_jsrcavixmpajewpdz3neidmiju refused: the route is filed under another account
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_azhocu7wvrybeinhb47odfxc5y refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_bfguex62ez5muauzha73qyhyu4 refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_avh3um6ifll4dcrlxqdpads6ty refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_s35qab5nqcc4jz2x7qkqe3qfba refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_lwnyb33aefbrweui24t2zvg4ba refused: the route is not an address route this import can take over (socket)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_v5l2cqf3grynal6w7j6bd3o56i refused: the route is not an address route this import can take over (no address)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_fwyweueqnlmx3jjj2wh3vpkb3y refused: the route is not an address route this import can take over (provisioned)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_o6wcntndcgxiaartko37lgogmi refused: the route is not an address route this import can take over (kind static)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_2lrbzetxz3j6ugpmgzsdzzhxwm refused: the route is not an address route this import can take over (backing)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_3xt7ynh7y7vyqpexgjhpx3h6qy refused: the route is not an address route this import can take over (tunnel)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_j32vx3q5rybs4jhtuxzrmrgid4 refused: the route is not an address route this import can take over (caps)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_hwdio4n6vm5lcppj66kywhylcy refused: the route is not an address route this import can take over (platform)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_nnl5kybtjcwxmi5r5bta44ap24 refused: the route is not an address route this import can take over (canary)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_ji4iqfajaryltw5mapust4w4uq refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_ybbhnfdrntuukntwgzwnnbd2uu refused: the route is not an address route this import can take over (region)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_pgg54qnvilvkwq62bgrvpszq34 refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_hxrqa4u2gvkcmdtvy4gmnjlraq refused: the route is not an address route this import can take over (hibernate)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_ayyqhejesp3hb7mipknmksxbza refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_ww3kvitjag6rmmpyqdymh2s3gu refused: this box does not run instances nested
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_6pq2ayynfjimoufcg5yar7ho4q refused: this box does not run instances nested
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_4a2enuqtoql6poqel7sb4i5ggu refused: claim control is on for this box
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_fvctjpi72ij3zeq2qyjmihd2fi (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_7c5qjjg65rohnxj4tlwbgzhqwu (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_wd6bugy44brtrldqlhhvl3aoaq: its verify hash is not a sha256
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_jfdphmjd7nimcjq4g5q5cygi3e: its verify hash is not a sha256
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_i4rpuitcxjrfindfchamoufsyq: its verify hash is not a sha256
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 21:47:19 provision: A.shop.example.com/..: not swapping for import sd_i445l2s4dp5vwnzxfpxlumk7ai: its host or seed id is malformed (seeds: not a host)
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_lvivlzy72xd4furunoivfu7gjy: its record does not describe a nested, hibernating instance
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_ffrdfc5upd5xe4mrmjtlhr2wli: its record does not describe a nested, hibernating instance
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_qkd4rvpfzfkihopcvrobvcztt4: its record does not describe a nested, hibernating instance
2026/10/24 21:47:19 provision: a.shop.example.com: not swapping for import sd_2czaulicrqgaewpd37ccnfs3jq: its record does not describe a nested, hibernating instance
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_54ojsyzdbjlepdyzrn6hgunjla (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_4ayac7gc4x4rfedrjox2fygdjm (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_vjjlaxom6gj3dfiuhy7vuec2zq (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 21:47:19 provision: dry-run: would consider swapping a.shop.example.com for import sd_dhvelo3q2k5d3hjz4x4gpfflbu
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_34kqaiwww5sx3jsghgy6k64ajy rolled back to its original route
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_34kqaiwww5sx3jsghgy6k64ajy: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_34kqaiwww5sx3jsghgy6k64ajy is cleaned up and its journal row retired
2026/10/24 21:47:19 provision: a.shop.example.com: import sd_il24ncqwku75iyycklr3bgvn64 left alone: the import journal holds seed sd_qqpbpdi2ykbujkv2qougbaelfq at swapped
2026/10/24 21:47:19 provision: a.shop.example.com: not minting import sd_pcaoq7qjtj3yg7i34r4tktxqwu: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:47:19 provision: a.shop.example.com: not minting import sd_miluulxmjwtlqxf6xfsgu6wvbm: it is placed but not committed, and only its swap makes its row
2026/10/24 21:47:19 provision: a.shop.example.com: swapped for import sd_z27cpi2blrnrhasaubdwn4mx4e (the original route is journaled)
2026/10/24 21:47:19 provision: a.shop.example.com: not minting import sd_z27cpi2blrnrhasaubdwn4mx4e: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:47:19 provision: a.shop.example.com: not minting import sd_z27cpi2blrnrhasaubdwn4mx4e: it is placed but not committed, and only its swap makes its row
2026/10/24 21:47:20 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:47:20 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 21:47:20 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 21:47:20 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 21:47:20 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 21:47:20 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 21:47:20 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:47:20 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 21:47:20 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 21:47:20 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:47:20 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:47:20 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 21:47:20 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 21:47:20 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 21:47:20 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:47:20 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 21:47:20 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:47:20 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:47:20 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 21:47:20 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:47:20 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:47:20 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:20 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 21:47:20 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:20 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:20 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:47:20 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:47:20 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:47:20 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:47:20 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:47:20 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:47:20 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:47:20 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:20 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:20 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:20 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 21:47:20 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:20 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:47:20 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 21:47:20 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:47:20 provision: record view: change index unusable (get refused): walking
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:47:20 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:20 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:47:20 provision: an app's records could not be listed: they are skipped
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:47:20 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:47:20 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 21:47:20 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:47:20 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:47:20 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:47:20 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:47:20 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:47:20 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:47:20 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:47:20 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:47:20 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:47:20 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:47:20 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 21:47:20 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (14.89s)
plan14_root_reads_test.go:600: root read census: 33 discrepancies
changed registry schema
changed root read internal/edge/edge.go:Edge.Reload; review class, task, sites and digest
changed root read internal/edge/edge.go:Edge.ServeHTTP; review class, task, sites and digest
changed root read internal/host/agent.go:runAgent; review class, task, sites and digest
changed root read internal/provision/reconcile.go:Reconcile; review class, task, sites and digest
changed root read internal/provision/reconcile.go:RouteFor; review class, task, sites and digest
changed root read internal/registry/registry.go:Registry.Fingerprint; review class, task, sites and digest
changed root read internal/registry/registry.go:Registry.Upsert; review class, task, sites and digest
changed root read internal/registry/registry.go:Registry.migrate; review class, task, sites and digest
changed root read internal/registry/registry.go:routeArgs; review class, task, sites and digest
changed root read internal/registry/registry.go:scanRoute; review class, task, sites and digest
changed root read internal/registry/remote_dispatch.go:Registry.dispatchRemote; review class, task, sites and digest
changed root source envelope internal/edge
changed root source envelope internal/host
changed root source envelope internal/plans
changed root source envelope internal/provision
changed root source envelope internal/ratelimit
changed root source envelope internal/registry
changed root source envelope internal/usage
unclassified root read internal/edge/ratelimit.go:Edge.noteRateLimited registry:Host@505,registry:Host@597,whole:Route@166
unclassified root read internal/edge/ratelimit.go:Edge.parseRateSettings registry:Host@271,registry:Host@367,registry:RateLimit@158,registry:RateLimit@77
unclassified root read internal/edge/ratelimit.go:Edge.rateAdmit registry:Host@834,registry:Kind@150,whole:Route@153,whole:Route@561,whole:Route@837,whole:Route@943
unclassified root read internal/edge/usagemeter.go:UsageMeter.RecordRateLimited registry:App@206,registry:Host@146,registry:Host@294,whole:Route@191
unclassified root read internal/host/plans_pass.go:planRateCeilingFor whole:Route@108
unclassified root read internal/registry/registry.go:Registry.SetRateLimit file:Exec@225,registry:remote@68,registry:remote@98,registry:sql@225,whole:remoteClient@100,whole:remoteClient@70
changed legacy declaration internal/edge/edge.go:Edge.Reload
changed legacy declaration internal/edge/edge.go:Edge.ServeHTTP
changed legacy declaration internal/provision/reconcile.go:Reconcile
changed legacy declaration internal/registry/registry.go:Registry.Fingerprint
changed legacy declaration internal/registry/registry.go:Registry.migrate
changed legacy declaration internal/registry/registry.go:scanRoute
changed reclassified declaration internal/registry/registry.go:Registry.Upsert
changed reclassified declaration internal/registry/remote_dispatch.go:Registry.dispatchRemote
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 23.666s
ok github.com/carlosframework/platform/internal/publicrelease 0.008s
ok github.com/carlosframework/platform/internal/ratelimit 0.116s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.706s
ok github.com/carlosframework/platform/internal/relay 7.244s
ok github.com/carlosframework/platform/internal/release 25.612s
ok github.com/carlosframework/platform/internal/rollout 3.771s
ok github.com/carlosframework/platform/internal/rolloutwords 0.004s
ok github.com/carlosframework/platform/internal/rootaws 0.008s
ok github.com/carlosframework/platform/internal/rootparticipation 0.062s
ok github.com/carlosframework/platform/internal/rootstate 0.028s
ok github.com/carlosframework/platform/internal/runs 0.540s
ok github.com/carlosframework/platform/internal/s3compat 0.007s
ok github.com/carlosframework/platform/internal/safefile 0.008s
ok github.com/carlosframework/platform/internal/schedule 2.652s
ok github.com/carlosframework/platform/internal/seeds 0.012s
ok github.com/carlosframework/platform/internal/sim/engines 2.616s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.345s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.405s
ok github.com/carlosframework/platform/internal/status 0.084s
ok github.com/carlosframework/platform/internal/steering 0.009s
ok github.com/carlosframework/platform/internal/stopevidence 0.359s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.005s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.007s
ok github.com/carlosframework/platform/internal/unixdial 0.008s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.660s
FAIL
=== internal_console@0 (79 s)
ok github.com/carlosframework/platform/internal/console 78.218s
=== internal_console@1 (43 s)
ok github.com/carlosframework/platform/internal/console 41.781s
=== internal_console@2 (41 s)
ok github.com/carlosframework/platform/internal/console 39.599s
=== internal_console@3 (42 s)
ok github.com/carlosframework/platform/internal/console 40.315s
=== internal_console@4 (41 s)
ok github.com/carlosframework/platform/internal/console 40.086s
=== internal_console@5 (40 s)
ok github.com/carlosframework/platform/internal/console 39.193s
=== internal_console@6 (36 s)
ok github.com/carlosframework/platform/internal/console 35.192s
=== internal_console@7 (46 s)
ok github.com/carlosframework/platform/internal/console 44.503s
=== root@0 (109 s)
ok github.com/carlosframework/platform 107.648s
=== root@1 (82 s)
ok github.com/carlosframework/platform 80.999s
=== root@2 (92 s)
ok github.com/carlosframework/platform 90.585s
FAILED: rest
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/edge/backhaul_share_test.go b/internal/edge/backhaul_share_test.go| new file mode 100644 |
| index 000000000..59af83526 |
| --- /dev/null |
| +++ b/internal/edge/backhaul_share_test.go |
| @@ -0,0 +1,268 @@ |
| +package edge |
| + |
| +// The fair share over the sending edge's backhaul slots (#358): while a |
| +// destination box's slots are at least half taken, one client cannot take |
| +// more than backhaulClientShare of them. |
| + |
| +import ( |
| + "context" |
| + "io" |
| + "net" |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "sync" |
| + "sync/atomic" |
| + "testing" |
| +) |
| + |
| +const ( |
| + shareClientA = "203.0.113.1" |
| + shareClientB = "203.0.113.2" |
| +) |
| + |
| +// shareScene is a sending edge whose host example.com is homed on box |
| +// "betty", fronted by a server that sets each request's peer address from |
| +// rlPeerHeader. |
| +type shareScene struct { |
| + e *Edge |
| + front *httptest.Server |
| + home *homeEdgeRecord |
| + eip string |
| +} |
| + |
| +func newShareScene(t *testing.T, mode RateMode) *shareScene { |
| + t.Helper() |
| + e, m, _ := hopEdge(t, "annie", "local.test") |
| + e.rateMode = mode |
| + rec, eip := fakeHomeEdge(t, nil) |
| + homedOn(t, m, "betty", eip, "example.com") |
| + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if p := r.Header.Get(rlPeerHeader); p != "" { |
| + r.Header.Del(rlPeerHeader) |
| + r.RemoteAddr = net.JoinHostPort(p, "40000") |
| + } |
| + e.ServeHTTP(w, r) |
| + })) |
| + t.Cleanup(front.Close) |
| + return &shareScene{e: e, front: front, home: rec, eip: eip} |
| +} |
| + |
| +// get sends one request for example.com from peer and reads the answer. |
| +func (s *shareScene) get(t *testing.T, peer string) (*http.Response, string) { |
| + t.Helper() |
| + req, err := http.NewRequest(http.MethodGet, s.front.URL+"/", nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + req.Host = "example.com" |
| + req.Header.Set(rlPeerHeader, peer) |
| + req.Header.Set("Accept-Encoding", "identity") |
| + resp, err := rlClient.Do(req) |
| + if err != nil { |
| + t.Fatalf("GET from %s: %v", peer, err) |
| + } |
| + defer resp.Body.Close() |
| + b, err := io.ReadAll(resp.Body) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return resp, string(b) |
| +} |
| + |
| +// hold takes n slots toward betty for peer, failing the test if any is |
| +// refused, and gives them back when the test ends. |
| +func (s *shareScene) hold(t *testing.T, peer string, n int) { |
| + t.Helper() |
| + key := "" |
| + if peer != "" { |
| + key = peer + "/32" |
| + } |
| + for i := 0; i < n; i++ { |
| + release, ok, shed := s.e.backhaulAcquire("betty", key) |
| + if !ok || shed { |
| + t.Fatalf("holding slot %d for %q: ok=%v shed=%v", i+1, peer, ok, shed) |
| + } |
| + t.Cleanup(release) |
| + } |
| +} |
| + |
| +func (s *shareScene) inflight() int64 { |
| + v, ok := s.e.backhaulInflight.Load("betty") |
| + if !ok { |
| + return 0 |
| + } |
| + return v.(*atomic.Int64).Load() |
| +} |
| + |
| +func TestBackhaulShareShedsHeavyClient(t *testing.T) { |
| + s := newShareScene(t, RateEnforce) |
| + buf := captureLog(t) |
| + s.hold(t, shareClientA, 32) // the 32nd is admitted: 31 were held before it |
| + |
| + resp, body := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusTooManyRequests { |
| + t.Fatalf("A's 33rd: status = %d, want 429", resp.StatusCode) |
| + } |
| + if got := resp.Header.Get("Retry-After"); got != "1" { |
| + t.Fatalf("Retry-After = %q, want 1", got) |
| + } |
| + if got := resp.Header.Get("Cache-Control"); got != "no-store" { |
| + t.Fatalf("Cache-Control = %q, want no-store", got) |
| + } |
| + if body != rateLimitBody { |
| + t.Fatalf("body = %q, want the rate-limit body", body) |
| + } |
| + if n := s.home.get().n; n != 0 { |
| + t.Fatalf("the home edge saw %d request(s), want none: a refused hop must not dial", n) |
| + } |
| + if got := s.inflight(); got != 32 { |
| + t.Fatalf("box count = %d after the refusal, want 32: the refusal must roll its increment back", got) |
| + } |
| + if !strings.Contains(buf.String(), "refused") { |
| + t.Fatalf("no refusal logged: %s", buf.String()) |
| + } |
| + |
| + resp, body = s.get(t, shareClientB) |
| + if resp.StatusCode != http.StatusOK || body != homeEdgeBody { |
| + t.Fatalf("B: status = %d body = %q, want the home edge's 200: another client is not shed", resp.StatusCode, body) |
| + } |
| +} |
| + |
| +func TestBackhaulShareQuietAdmitsHeavyClient(t *testing.T) { |
| + s := newShareScene(t, RateEnforce) |
| + s.hold(t, shareClientA, 31) |
| + |
| + resp, body := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusOK || body != homeEdgeBody { |
| + t.Fatalf("status = %d body = %q, want 200: 31 slots held is under half of 64", resp.StatusCode, body) |
| + } |
| +} |
| + |
| +func TestBackhaulSaturationUnchanged(t *testing.T) { |
| + s := newShareScene(t, RateEnforce) |
| + s.hold(t, "", 64) // unkeyed, so the share plays no part |
| + |
| + resp, _ := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusServiceUnavailable { |
| + t.Fatalf("status = %d, want the existing 503 at 64 slots", resp.StatusCode) |
| + } |
| + if got := resp.Header.Get("Retry-After"); got != "2" { |
| + t.Fatalf("Retry-After = %q, want 2", got) |
| + } |
| + if got := s.inflight(); got != 64 { |
| + t.Fatalf("box count = %d, want 64", got) |
| + } |
| +} |
| + |
| +// A heavy client at saturation is still told the box is full, not that it is |
| +// using too much: the cap is checked before the share. |
| +func TestBackhaulSaturationBeatsShare(t *testing.T) { |
| + s := newShareScene(t, RateEnforce) |
| + s.hold(t, shareClientA, 32) |
| + s.hold(t, "", 32) |
| + |
| + resp, _ := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusServiceUnavailable { |
| + t.Fatalf("status = %d, want 503", resp.StatusCode) |
| + } |
| +} |
| + |
| +func TestBackhaulShareObserveOnlyLogs(t *testing.T) { |
| + s := newShareScene(t, RateObserve) |
| + buf := captureLog(t) |
| + s.hold(t, shareClientA, 32) |
| + |
| + resp, body := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusOK || body != homeEdgeBody { |
| + t.Fatalf("status = %d body = %q, want 200: observe never refuses", resp.StatusCode, body) |
| + } |
| + if !strings.Contains(buf.String(), "would refuse") { |
| + t.Fatalf("no \"would refuse\" line: %s", buf.String()) |
| + } |
| +} |
| + |
| +func TestBackhaulShareOffTakesNoPart(t *testing.T) { |
| + s := newShareScene(t, RateOff) |
| + buf := captureLog(t) |
| + s.hold(t, shareClientA, 33) |
| + |
| + resp, _ := s.get(t, shareClientA) |
| + if resp.StatusCode != http.StatusOK { |
| + t.Fatalf("status = %d, want 200: the limiter is off", resp.StatusCode) |
| + } |
| + if strings.Contains(buf.String(), "refuse") { |
| + t.Fatalf("off logged a refusal: %s", buf.String()) |
| + } |
| +} |
| + |
| +// The edge-local door's calls are the box's own traffic and never shed. The |
| +// door reaches backhaulTo directly (serveDoor), so the test does too. |
| +func TestBackhaulShareDoorExempt(t *testing.T) { |
| + s := newShareScene(t, RateEnforce) |
| + s.hold(t, shareClientA, 32) |
| + |
| + req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) |
| + req.RemoteAddr = net.JoinHostPort(shareClientA, "40000") |
| + req = req.WithContext(context.WithValue(req.Context(), doorCallerKey{}, doorCaller{Account: "shop", App: "web"})) |
| + rec := httptest.NewRecorder() |
| + if !s.e.backhaulTo(rec, req, "example.com", s.eip, "betty") { |
| + t.Fatal("backhaulTo declined the request") |
| + } |
| + if rec.Code != http.StatusOK || rec.Body.String() != homeEdgeBody { |
| + t.Fatalf("door caller: status = %d body = %q, want 200: a door caller is never shed", rec.Code, rec.Body.String()) |
| + } |
| +} |
| + |
| +// A slot's share and its box count both come back, once, whatever the |
| +// release order and however often release is called. |
| +func TestBackhaulShareReleaseGivesBothBack(t *testing.T) { |
| + e, _, _ := hopEdge(t, "annie", "local.test") |
| + e.rateMode = RateEnforce |
| + var rels []func() |
| + for i := 0; i < 8; i++ { |
| + rel, ok, shed := e.backhaulAcquire("betty", "198.51.100.7/32") |
| + if !ok || shed { |
| + t.Fatalf("slot %d: ok=%v shed=%v", i+1, ok, shed) |
| + } |
| + rels = append(rels, rel) |
| + } |
| + for _, rel := range rels { |
| + rel() |
| + rel() |
| + } |
| + if v, _ := e.backhaulInflight.Load("betty"); v.(*atomic.Int64).Load() != 0 { |
| + t.Fatalf("box count = %d after every release, want 0", v.(*atomic.Int64).Load()) |
| + } |
| + sv, _ := e.backhaulShares.Load("betty") |
| + if got := sv.(interface{ Held(string) int }).Held("198.51.100.7/32"); got != 0 { |
| + t.Fatalf("client still holds %d slot(s) after every release", got) |
| + } |
| +} |
| + |
| +// Concurrent clients never push the box over its cap, and everything is |
| +// given back; run under -race. |
| +func TestBackhaulShareConcurrent(t *testing.T) { |
| + e, _, _ := hopEdge(t, "annie", "local.test") |
| + e.rateMode = RateEnforce |
| + var wg sync.WaitGroup |
| + for c := 0; c < 8; c++ { |
| + key := "203.0.113." + string(rune('1'+c)) + "/32" |
| + wg.Add(1) |
| + go func() { |
| + defer wg.Done() |
| + for i := 0; i < 500; i++ { |
| + if rel, ok, _ := e.backhaulAcquire("betty", key); ok { |
| + if v, _ := e.backhaulInflight.Load("betty"); v.(*atomic.Int64).Load() > backhaulMaxInflight.Load() { |
| + t.Error("box count over the cap") |
| + } |
| + rel() |
| + } |
| + } |
| + }() |
| + } |
| + wg.Wait() |
| + if v, _ := e.backhaulInflight.Load("betty"); v.(*atomic.Int64).Load() != 0 { |
| + t.Fatalf("box count = %d at rest, want 0", v.(*atomic.Int64).Load()) |
| + } |
| +} |
diff --git a/internal/edge/edge.go b/internal/edge/edge.go| index e4beee12c..ed155629d 100644 |
| --- a/internal/edge/edge.go |
| +++ b/internal/edge/edge.go |
| @@ -282,6 +282,10 @@ type Edge struct { |
| // subdomain a backhaul target, so a per-host key would let a stranger |
| // grow this map without limit. Bounded by the deployment's box count. |
| backhaulInflight sync.Map // box → *atomic.Int64 |
| + // backhaulShares counts, per destination box, the slots each client |
| + // holds, for the fair share in backhaulAcquire. Keyed by box for the |
| + // reason backhaulInflight is; each Shares bounds its own client keys. |
| + backhaulShares sync.Map // box → *ratelimit.Shares |
| // The edge-local door (door.go, #175 P7). doorPath and doorAddr come from |
| // WithEdgeDoor: the socket to listen on ("" = no door, the default) and |
| // the box's public address, which every door request presents as its |
| @@ -2818,11 +2822,45 @@ func backhaulInflightFromEnv() int { |
| return n |
| } |
| -// backhaulAcquire reserves one in-flight slot toward box, or reports |
| -// saturation. releaseSlot must be called exactly once when the hop's |
| -// ServeHTTP returns — a streaming or upgraded response holds its slot for the |
| -// life of the stream, deliberately: a long-lived stream IS sustained |
| -// cross-region egress, which is the thing being bounded. |
| +// backhaulClientShare is the most slots one client may hold toward a |
| +// destination box while the box is at least half full. A fixed number, not a |
| +// setting: the sending edge has no route row to read one from (#358). |
| +const backhaulClientShare = 8 |
| + |
| +// backhaulSharesMaxKeys bounds the clients tracked per destination box. |
| +const backhaulSharesMaxKeys = 10_000 |
| + |
| +// backhaulClientKey is the fair-share key of the client behind r, or "" when |
| +// r is not shared out: the limiter is off, a door caller (the box's own |
| +// traffic), a request that came over a fleet tunnel (its address is the hub's, |
| +// so every client would be one key), or an address with no usable prefix. |
| +func (e *Edge) backhaulClientKey(r *http.Request) string { |
| + if e.rateMode == RateOff { |
| + return "" |
| + } |
| + if _, door := doorCallerFrom(r.Context()); door || fromTunnel(r.Context()) { |
| + return "" |
| + } |
| + p, ok := ratelimit.ClientKey(clientIP(r)) |
| + if !ok { |
| + return "" |
| + } |
| + return p.String() |
| +} |
| + |
| +// backhaulAcquire reserves one in-flight slot toward box, or reports why it |
| +// cannot: saturation (ok and shed both false, the box's cap is spent) or a |
| +// fair-share refusal (shed true). releaseSlot must be called exactly once |
| +// when the hop's ServeHTTP returns — a streaming or upgraded response holds |
| +// its slot for the life of the stream, deliberately: a long-lived stream IS |
| +// sustained cross-region egress, which is the thing being bounded. |
| +// |
| +// The fair share is the home edge's rule over the sending edge's slots: while |
| +// the box's slots are at least half taken, a client that already holds |
| +// backhaulClientShare of them is refused, so one client cannot fill the hop |
| +// and push everyone else to 503 while the instance is quiet. clientKey "" |
| +// takes no part in it. In observe mode the refusal is logged and the request |
| +// takes its slot and share anyway. |
| // |
| // The counter is per home BOX and the map is never keyed by anything a |
| // stranger controls (backhaulInflight's own doc). The count is incremented |
| @@ -2831,8 +2869,9 @@ func backhaulInflightFromEnv() int { |
| // request arriving inside another's rollback window can be refused while a |
| // slot is in fact free. Both errors are in the safe direction — a refusal is |
| // a 503 the client retries in two seconds, where an over-admission would be |
| -// cross-region bytes the cap exists to prevent. |
| -func (e *Edge) backhaulAcquire(box string) (releaseSlot func(), ok bool) { |
| +// cross-region bytes the cap exists to prevent. A fair-share refusal rolls |
| +// the count back the same way. |
| +func (e *Edge) backhaulAcquire(box, clientKey string) (releaseSlot func(), ok bool, shed bool) { |
| // Load first, like backhaulTransport: LoadOrStore would allocate a |
| // counter per backhauled request just to throw it away on the hit path. |
| v, hit := e.backhaulInflight.Load(box) |
| @@ -2840,15 +2879,50 @@ func (e *Edge) backhaulAcquire(box string) (releaseSlot func(), ok bool) { |
| v, _ = e.backhaulInflight.LoadOrStore(box, new(atomic.Int64)) |
| } |
| n := v.(*atomic.Int64) |
| - if n.Add(1) > backhaulMaxInflight.Load() { |
| + limit := backhaulMaxInflight.Load() |
| + held := n.Add(1) |
| + if held > limit { |
| n.Add(-1) |
| - return nil, false |
| + return nil, false, false |
| } |
| // Once-guarded because a slot released twice would let the cap drift |
| // upwards for the life of the process — the one failure mode of a |
| // hand-released semaphore, and cheaper to make impossible than to audit. |
| var once sync.Once |
| - return func() { once.Do(func() { n.Add(-1) }) }, true |
| + var releaseShare func() |
| + if clientKey != "" && e.rateMode != RateOff { |
| + sv, hit := e.backhaulShares.Load(box) |
| + if !hit { |
| + sv, _ = e.backhaulShares.LoadOrStore(box, ratelimit.NewShares(backhaulSharesMaxKeys)) |
| + } |
| + shares := sv.(*ratelimit.Shares) |
| + // Busy is the slots already taken, not counting this request. |
| + busy := held-1 >= limit/2 |
| + rel, admitted := shares.Acquire(clientKey, busy, backhaulClientShare) |
| + if !admitted { |
| + verb := "refused" |
| + if e.rateMode == RateObserve { |
| + verb = "would refuse" |
| + } |
| + e.logThrottled("backhaul-share "+box, fmt.Sprintf( |
| + "backhaul to %s is at least half full (%d of %d slots) and client %s holds %d of them — %s the request (fair share %d)", |
| + box, held-1, limit, clientKey, shares.Held(clientKey), verb, backhaulClientShare)) |
| + if e.rateMode == RateEnforce { |
| + n.Add(-1) |
| + return nil, false, true |
| + } |
| + rel, _ = shares.Acquire(clientKey, false, backhaulClientShare) |
| + } |
| + releaseShare = rel |
| + } |
| + return func() { |
| + once.Do(func() { |
| + if releaseShare != nil { |
| + releaseShare() |
| + } |
| + n.Add(-1) |
| + }) |
| + }, true, false |
| } |
| // backhaulPort is the port a home edge answers on. Always 443 in production — |
| @@ -2940,7 +3014,13 @@ func (e *Edge) backhaulTo(w http.ResponseWriter, r *http.Request, host, eip, box |
| // edge, which would lie to a legitimate client about where its app lives. |
| // The log key is per home BOX, like "backhaul-error "+box below and for |
| // the same bounded-key reason. |
| - releaseSlot, ok := e.backhaulAcquire(box) |
| + releaseSlot, ok, shed := e.backhaulAcquire(box, e.backhaulClientKey(r)) |
| + if shed { |
| + // The fair share, not saturation: this client holds its share of a |
| + // busy hop. The same 429 the home edge gives (writeRateLimited). |
| + writeRateLimited(w, time.Second) |
| + return true |
| + } |
| if !ok { |
| e.logThrottled("backhaul-saturated "+box, |
| fmt.Sprintf("backhaul to %s (%s) is at its in-flight cap (%d) — answering 503 + Retry-After: 2", box, eip, backhaulMaxInflight.Load())) |
diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go| index 898eaf95f..2e38542e8 100644 |
| --- a/internal/edge/homemap_test.go |
| +++ b/internal/edge/homemap_test.go |
| @@ -1360,7 +1360,7 @@ func TestBackhaulInflightCapIsPerHomeBox(t *testing.T) { |
| // would hang for the whole 5s budget and be visible as a failure.) |
| homedOn(t, m, "betty", "192.0.2.5", "saturated.test") |
| - release, ok := e.backhaulAcquire("betty") |
| + release, ok, _ := e.backhaulAcquire("betty", "") |
| if !ok { |
| t.Fatal("the first slot toward a box must always be available") |
| } |
| @@ -1457,7 +1457,7 @@ func TestBackhaulSaturationLogsThrottledPerBox(t *testing.T) { |
| homedOn(t, m, "betty", "192.0.2.5", "betty.test") |
| homedOn(t, m, "carol", "192.0.2.6", "carol.test") |
| for _, box := range []string{"betty", "carol"} { |
| - release, ok := e.backhaulAcquire(box) |
| + release, ok, _ := e.backhaulAcquire(box, "") |
| if !ok { |
| t.Fatalf("%s: the first slot toward a box must always be available", box) |
| } |