amadan
Sidecar isolation Task 5: the per-host agent socket and the worker's prepare (D3)
agentsock gains a host-pinned listener (ListenPinned, ServePinned): mode 0600, the same verbs and token check as the shared socket, admitting only a request with exactly one Carlos-Sidecar-Host value equal to its host, checked before the token, with the uniform 401. While a host's pinned listener is open the shared socket refuses that host (Review Focus 4). The exec backend prepares a nested worker (prepareWorker): directories, the instance's env layers through instancefs, no admin token, the worker log listener, and the pinned listener bound before return; a failure after the bind closes it. newWorkerUnitProc carries the listener and closeListeners closes it after done. workerEnv's socket comes from the backend (agentSocketFor): the per-host path for a nested worker row, the shared socket otherwise. The agent wires its socket server onto the backend.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
9 files changed,
+998
−55
internal/activator/activator.go+20 −2internal/activator/backend_exec.go+9 −1internal/activator/backend_unit.go+12 −0internal/activator/prepare.go+115 −19internal/activator/prepare_test.go+336 −0internal/activator/unitproc.go+19 −0internal/agentsock/agentsock.go+190 −32internal/agentsock/agentsock_test.go+290 −0internal/host/agent.go+7 −1
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok amadan.net/carlos/platform.git/launchctx 0.016s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (11 s)
=== gofmt (1 s)
gofmt -l .: clean
=== rest (163 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.411s
ok github.com/carlosframework/platform/cmd/moveprobe 11.754s
ok github.com/carlosframework/platform/cmd/zddprobe 0.759s
ok github.com/carlosframework/platform/internal/activator 57.541s
ok github.com/carlosframework/platform/internal/agentsock 0.074s
ok github.com/carlosframework/platform/internal/alerting 0.006s
ok github.com/carlosframework/platform/internal/analytics 0.048s
ok github.com/carlosframework/platform/internal/appconfig 2.770s
ok github.com/carlosframework/platform/internal/apperr 0.180s
ok github.com/carlosframework/platform/internal/applog 0.380s
ok github.com/carlosframework/platform/internal/awsinv 0.006s
ok github.com/carlosframework/platform/internal/boxchoice 0.013s
ok github.com/carlosframework/platform/internal/boxid 0.718s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.026s
ok github.com/carlosframework/platform/internal/caps 0.029s
ok github.com/carlosframework/platform/internal/certwarm 0.011s
ok github.com/carlosframework/platform/internal/cgroup 0.800s
ok github.com/carlosframework/platform/internal/changelog 0.499s
ok github.com/carlosframework/platform/internal/claimactivation 1.123s
ok github.com/carlosframework/platform/internal/clamd 0.068s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 7.655s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.032s
ok github.com/carlosframework/platform/internal/dbquery 0.015s
ok github.com/carlosframework/platform/internal/dbreads 2.165s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.010s
ok github.com/carlosframework/platform/internal/dnsrec 0.007s
ok github.com/carlosframework/platform/internal/docsite 0.041s
ok github.com/carlosframework/platform/internal/domains 2.861s
ok github.com/carlosframework/platform/internal/doorkey 0.020s
ok github.com/carlosframework/platform/internal/economics 0.012s
ok github.com/carlosframework/platform/internal/edge 45.075s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.155s
ok github.com/carlosframework/platform/internal/gate 1.657s
ok github.com/carlosframework/platform/internal/genselect 0.022s
ok github.com/carlosframework/platform/internal/geo 0.022s
ok github.com/carlosframework/platform/internal/geoip 0.054s
ok github.com/carlosframework/platform/internal/historyreadback 0.265s
ok github.com/carlosframework/platform/internal/host 142.010s
ok github.com/carlosframework/platform/internal/hostactors 6.172s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.013s
ok github.com/carlosframework/platform/internal/hostbox 2.951s
ok github.com/carlosframework/platform/internal/hostclaim 0.765s
ok github.com/carlosframework/platform/internal/hostenv 0.013s
ok github.com/carlosframework/platform/internal/hostevidence 0.013s
ok github.com/carlosframework/platform/internal/hostexec 5.179s
ok github.com/carlosframework/platform/internal/hostfence 0.041s
ok github.com/carlosframework/platform/internal/hostfiles 10.291s
ok github.com/carlosframework/platform/internal/hostfloor 0.017s
ok github.com/carlosframework/platform/internal/hostguard 1.037s
ok github.com/carlosframework/platform/internal/hostops 2.015s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.013s
ok github.com/carlosframework/platform/internal/hostquarantine 0.014s
ok github.com/carlosframework/platform/internal/hostrelocate 0.673s
ok github.com/carlosframework/platform/internal/hostresolve 0.158s
ok github.com/carlosframework/platform/internal/hostwrite 0.015s
ok github.com/carlosframework/platform/internal/infraview 0.012s
ok github.com/carlosframework/platform/internal/instancectl 6.918s
ok github.com/carlosframework/platform/internal/instanceexec 2.807s
ok github.com/carlosframework/platform/internal/instancefs 0.022s
ok github.com/carlosframework/platform/internal/instancekey 0.015s
ok github.com/carlosframework/platform/internal/instanceupdates 0.019s
ok github.com/carlosframework/platform/internal/layout 1.667s
ok github.com/carlosframework/platform/internal/lbhealth 0.017s
ok github.com/carlosframework/platform/internal/lease 10.103s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.515s
ok github.com/carlosframework/platform/internal/ledger 0.015s
ok github.com/carlosframework/platform/internal/mail 0.302s
ok github.com/carlosframework/platform/internal/maintenance 0.105s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.550s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.078s
ok github.com/carlosframework/platform/internal/metrics 0.008s
ok github.com/carlosframework/platform/internal/moves 22.073s
ok github.com/carlosframework/platform/internal/moves/archive 1.016s
ok github.com/carlosframework/platform/internal/objstore 0.272s
ok github.com/carlosframework/platform/internal/ops 0.306s
ok github.com/carlosframework/platform/internal/park 0.310s
ok github.com/carlosframework/platform/internal/plans 0.007s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
2026/10/24 21:25:25 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:25 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:25 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:25 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 21:25:25 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:25 provision: an app's records could not be listed: they are skipped
2026/10/24 21:25:25 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:25 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:25:25 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:25:25 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:25:25 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:25:25 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:25:25 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:25:25 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:25:25 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:25:25 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:25:25 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:25:25 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:25:25 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:25:25 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:25:25 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:25 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:25:26 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:25:26 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:25:26 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:25:26 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:25:26 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:25:26 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:26 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:26 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:26 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:26 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:25:26 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:25:26 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 21:25:26 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 21:25:26 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:26 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:26 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.73s)
plan9_b8_census_test.go:372: record-write census rejected 2 site(s); update the explicit inventory only after reviewing the source:
changed site internal/instanceexec/exit.go:RunExit: source differs from the reviewed digest
changed site internal/ops/litestream.go:FleetDBsWithSkipped: source differs from the reviewed digest
2026/10/24 21:25:27 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:25:27 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:27 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 21:25:28 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 21:25:28 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 21:25:28 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 21:25:28 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:25:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:25:28 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:25:28 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:25:28 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 21:25:28 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:25:28 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:25:28 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:25:28 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:25:28 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:28 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:25:28 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 21:25:28 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 21:25:28 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:28 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:25:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:28 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 21:25:28 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:25:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:28 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:25:28 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:28 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:25:28 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:28 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:28 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 21:25:28 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:25:28 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 21:25:28 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:28 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:25:28 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 21:25:28 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:25:28 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 21:25:28 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 21:25:28 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 21:25:28 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 21:25:28 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:28 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:25:28 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 21:25:28 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 21:25:28 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 21:25:28 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 21:25:28 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 21:25:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:28 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 21:25:28 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 21:25:29 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:29 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:25:29 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:25:29 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:25:29 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:25:29 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 21:25:29 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:25:29 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:25:29 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:25:29 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:25:29 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 21:25:29 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 21:25:29 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:25:29 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 21:25:29 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 21:25:29 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:25:29 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 21:25:29 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:25:29 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:29 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:30 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:30 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:25:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:30 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:25:30 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 21:25:30 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 21:25:30 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:25:30 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:30 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:25:30 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:30 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:25:30 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:30 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:30 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:25:30 provision: a.shop.example.com: swapped for import sd_rei6aafzalab6yszhqyrmk6ys4 (the original route is journaled)
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_rei6aafzalab6yszhqyrmk6ys4 committed: its gate is cleared and its journal row dropped
2026/10/24 21:25:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_pwxuhur4gzbnnvsqildnnoxwvm rolled back to its original route
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_pwxuhur4gzbnnvsqildnnoxwvm: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_pwxuhur4gzbnnvsqildnnoxwvm is cleaned up and its journal row retired
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_4bw3tf52bbzienn6rftwp2f7ge rolled back to its original route
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_4bw3tf52bbzienn6rftwp2f7ge: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:25:30 provision: a.shop.example.com: import sd_4bw3tf52bbzienn6rftwp2f7ge is cleaned up and its journal row retired
2026/10/24 21:25:30 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_qwd6h2cwsdmd5ntjlcuka2xc6a
2026/10/24 21:25:30 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 21:25:30 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 21:25:30 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:31 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 21:25:31 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 21:25:31 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_ubhioln5kvfh2iw46ycpf5n55i cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_grm255m5wt27omusk4uma7k7ti cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_in6rhtnuh3rzj4qcdqqxwdzl5a is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_oce7lbi2hz3ln7tioj2zkjxbay is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_uxx36qmsnzhzwnhdvrlicq4baa is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:25:31 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 21:25:31 provision: b.shop.example.com: import sd_b6nxbdkxueithsylwb4qtujl74 left alone this pass: the import journal could not be read
2026/10/24 21:25:31 provision: c.shop.example.com: import sd_yot3vxu7fbuo27l6korvnsrvkm left alone this pass: the import journal could not be read
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_mqcgbina4aqsohtnb6dahi52sa committed: its gate is cleared and its journal row dropped
2026/10/24 21:25:31 provision: a.shop.example.com: not converging import sd_ng5cvfll6ouvhaogq5j2jxhi54: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:25:31 provision: a.shop.example.com: not converging import sd_t2omwmmwxsau2lhho7ay4xl754: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:25:31 provision: a.shop.example.com: not converging import sd_ho7rthvvln2iobkl5isjwmlniy: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:25:31 provision: a.shop.example.com: not converging import sd_65ulb4tc6wsras44iuddldxtte: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:25:31 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:31 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 21:25:31 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_5cqkit263esy6kuykmncidhl44 committed: its gate is cleared and its journal row dropped
2026/10/24 21:25:31 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:31 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:31 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 21:25:31 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_vf7cgammrpbvwwssemtpwrbylu: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_vf7cgammrpbvwwssemtpwrbylu rolled back (no box had swapped it)
2026/10/24 21:25:31 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 21:25:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:31 provision: removed instance route acme.go.tito.io (its seed sd_li346mnitdflbcqskrjtrzidpa was aborted); the next pass rolls it back
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_li346mnitdflbcqskrjtrzidpa: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_li346mnitdflbcqskrjtrzidpa rolled back (no box had swapped it)
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_rhjyo2cdia4ypgu5iizrg6dohu has a row this pass may not remove; leaving the seed aborted
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_2l7efurghn5ukno4smetnjabjq rolled back (no box had swapped it)
2026/10/24 21:25:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:31 provision: removed instance route acme.go.tito.io (its seed sd_nda6c4xxilxvcsj6hmzv5ius2m was aborted); the next pass rolls it back
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_nda6c4xxilxvcsj6hmzv5ius2m has no route on this box: its local data is removed
2026/10/24 21:25:31 provision: acme.go.tito.io: aborted seed sd_nda6c4xxilxvcsj6hmzv5ius2m rolled back (no box had swapped it)
2026/10/24 21:25:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_n4mkvv2gcq4j7xdgjdee4rlrj4 (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_n4mkvv2gcq4j7xdgjdee4rlrj4 rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_n4mkvv2gcq4j7xdgjdee4rlrj4 is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_utdxs74a77cnnjyekqpfm5qsxa (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_utdxs74a77cnnjyekqpfm5qsxa rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_utdxs74a77cnnjyekqpfm5qsxa is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_voazvemfzegamrphxvcme4wxam (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_voazvemfzegamrphxvcme4wxam rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_voazvemfzegamrphxvcme4wxam: the host's replica is left alone: its record no longer names this seed
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_voazvemfzegamrphxvcme4wxam is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_mnk4pxo3oejxivdgthbvhjxdcm (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_mnk4pxo3oejxivdgthbvhjxdcm rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_mnk4pxo3oejxivdgthbvhjxdcm rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_mnk4pxo3oejxivdgthbvhjxdcm is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_lusiyycmhe4zasoju4bej364de (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_lusiyycmhe4zasoju4bej364de rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_wrvobaeq22om4scmkk3xb2nrle (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_wrvobaeq22om4scmkk3xb2nrle is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_lm6xae6qe7gaqu4wgirojrqlxu (the original route is journaled)
2026/10/24 21:25:31 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_lm6xae6qe7gaqu4wgirojrqlxu left alone this pass: its record could not be read
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_zeanjpr5ebul7g6dmn3y4g5cca (the original route is journaled)
2026/10/24 21:25:31 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_zeanjpr5ebul7g6dmn3y4g5cca left alone this pass: its record could not be read
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_ngvdq24bu4353s4hkpn6cdnwgm (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_ngvdq24bu4353s4hkpn6cdnwgm: close the import gate: constraint failed: injected (1811)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_ngvdq24bu4353s4hkpn6cdnwgm rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_ngvdq24bu4353s4hkpn6cdnwgm is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_cyeu2zsjqg33dygbko44d5zz34 (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_cyeu2zsjqg33dygbko44d5zz34: close the import gate: constraint failed: injected (1811)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_cyeu2zsjqg33dygbko44d5zz34 rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_cyeu2zsjqg33dygbko44d5zz34 is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_lct4iow2qtvopvqae25yb2f5ge (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_lct4iow2qtvopvqae25yb2f5ge rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_lct4iow2qtvopvqae25yb2f5ge rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_lct4iow2qtvopvqae25yb2f5ge is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_oir4ae6jostuvitscm2mg7ithe (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_oir4ae6jostuvitscm2mg7ithe rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_oir4ae6jostuvitscm2mg7ithe is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_yxj4pml6v4qcfwgej3fpeilmiq (the original route is journaled)
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_yxj4pml6v4qcfwgej3fpeilmiq: retire its instance key: injected failure at retire-key
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_yxj4pml6v4qcfwgej3fpeilmiq rolled back to its original route
2026/10/24 21:25:31 provision: a.shop.example.com: import sd_yxj4pml6v4qcfwgej3fpeilmiq is cleaned up and its journal row retired
2026/10/24 21:25:31 provision: a.shop.example.com: swapped for import sd_l32xei5igoelj7jkbbzndyo73y (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_l32xei5igoelj7jkbbzndyo73y rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_l32xei5igoelj7jkbbzndyo73y is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_qfvcvwnlvrdw4ww3xggg5wmuuq (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_qfvcvwnlvrdw4ww3xggg5wmuuq: remove its local data: injected failure at remove-local
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_qfvcvwnlvrdw4ww3xggg5wmuuq rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_qfvcvwnlvrdw4ww3xggg5wmuuq is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_2jfjm4jo3d5xlvv5u7khvkt4pi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_2jfjm4jo3d5xlvv5u7khvkt4pi rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_2jfjm4jo3d5xlvv5u7khvkt4pi is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_icwuydroql2lcaydcufuymzffi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_icwuydroql2lcaydcufuymzffi: remove its local data: injected failure at remove-local-done
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_icwuydroql2lcaydcufuymzffi rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_icwuydroql2lcaydcufuymzffi is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_g4bwnaggycccsin2luwsodky3q (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_g4bwnaggycccsin2luwsodky3q rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_g4bwnaggycccsin2luwsodky3q is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_nczqeg435hwp7hhnccw7obewee (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_nczqeg435hwp7hhnccw7obewee: restore the original route: constraint failed: injected (1811)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_nczqeg435hwp7hhnccw7obewee rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_nczqeg435hwp7hhnccw7obewee is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_zcn66kz6djmpv3qssi6ta56p5i (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zcn66kz6djmpv3qssi6ta56p5i: restore the original route: constraint failed: injected (1811)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zcn66kz6djmpv3qssi6ta56p5i rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zcn66kz6djmpv3qssi6ta56p5i is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_pou7lihzmiojzmedc3ve35agey (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_pou7lihzmiojzmedc3ve35agey is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_26wfcdfgljdtlca6r4ozxs5cky (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_26wfcdfgljdtlca6r4ozxs5cky rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_26wfcdfgljdtlca6r4ozxs5cky: delete its snapshot: injected failure at delete-snapshot
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_26wfcdfgljdtlca6r4ozxs5cky is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_cfeyerosrareuolhjuxf5k262u (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_cfeyerosrareuolhjuxf5k262u rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_cfeyerosrareuolhjuxf5k262u is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_rmnve7e3qilyre2azjxp22hxsu (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_rmnve7e3qilyre2azjxp22hxsu rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_rmnve7e3qilyre2azjxp22hxsu: delete the host's replica: injected failure at delete-replica
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_rmnve7e3qilyre2azjxp22hxsu is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_775ixmzyqjabyfrwgokx26idha (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_775ixmzyqjabyfrwgokx26idha rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_775ixmzyqjabyfrwgokx26idha is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_lyjper5welkg5dq5e7xwwzsyxi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_lyjper5welkg5dq5e7xwwzsyxi rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_lyjper5welkg5dq5e7xwwzsyxi: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_lyjper5welkg5dq5e7xwwzsyxi is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_hqrl3lfzobj35yajcdx2pczmxu (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hqrl3lfzobj35yajcdx2pczmxu rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hqrl3lfzobj35yajcdx2pczmxu: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hqrl3lfzobj35yajcdx2pczmxu is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_hx3tzamt3y4xvnwjo75y7bghly (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hx3tzamt3y4xvnwjo75y7bghly rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hx3tzamt3y4xvnwjo75y7bghly is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: mark import sd_hx3tzamt3y4xvnwjo75y7bghly rolled back: injected failure at cas
2026/10/24 21:25:32 provision: a.shop.example.com: aborted seed sd_hx3tzamt3y4xvnwjo75y7bghly rolled back (no box had swapped it)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_4uvi2cuiszf6or4jmyfftf4r7e (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_4uvi2cuiszf6or4jmyfftf4r7e rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_4uvi2cuiszf6or4jmyfftf4r7e is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: aborted seed sd_4uvi2cuiszf6or4jmyfftf4r7e rolled back (no box had swapped it)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_hflizc76uuof2ll2kdxvjsk6e4 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hflizc76uuof2ll2kdxvjsk6e4 rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hflizc76uuof2ll2kdxvjsk6e4 rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hflizc76uuof2ll2kdxvjsk6e4 rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_hflizc76uuof2ll2kdxvjsk6e4 is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_ysxjrextgij3ga4zppts5usihi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_ysxjrextgij3ga4zppts5usihi rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_ysxjrextgij3ga4zppts5usihi is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_q4qwytapc2yps2sfjqkyxuwzla (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_q4qwytapc2yps2sfjqkyxuwzla rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_q4qwytapc2yps2sfjqkyxuwzla is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_wewtnomwy24h6pbir5i44jvm2m (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_wewtnomwy24h6pbir5i44jvm2m rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_wewtnomwy24h6pbir5i44jvm2m is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_eks5orad4qh7u2kgvow22epwnq (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_eks5orad4qh7u2kgvow22epwnq rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_eks5orad4qh7u2kgvow22epwnq is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_ropni3cywjrjxhj2dmxrpwxc7i (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_ropni3cywjrjxhj2dmxrpwxc7i rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_ropni3cywjrjxhj2dmxrpwxc7i is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_zo572emk7apaeqk5cfjzmzoyrq (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zo572emk7apaeqk5cfjzmzoyrq rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zo572emk7apaeqk5cfjzmzoyrq is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_tjldnufzwyvhj3kody6wsjhnem (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_tjldnufzwyvhj3kody6wsjhnem rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_tjldnufzwyvhj3kody6wsjhnem is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_uw4kmwtv367ib7avxilsbafgde (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_uw4kmwtv367ib7avxilsbafgde has no route on this box: its local data is removed and no route is put back
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_uw4kmwtv367ib7avxilsbafgde is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_3ngwu7zclraylmbtw42uznaone (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_yufleqdyfwlgpcre6673sjty44 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_spz2oo2rgbgoyyreduviaelgli (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_spz2oo2rgbgoyyreduviaelgli cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_rsms6r3jz736j5w4oti2kyvpfe (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_rsms6r3jz736j5w4oti2kyvpfe is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 21:25:32 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 21:25:32 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_2jmn3jtr4yfif5h5zpjpwh6ms4 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_bhvylj6qabwfbfxm74lo7vhla4 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_vhuaintl3belbrlu7jkbe47nkq (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_ryxnfpkvx3ii5ns77n4jixs4rm (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_7yjqpxc2fnzn7xx37t7wm6bsi4 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_7yjqpxc2fnzn7xx37t7wm6bsi4 rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_lrkwturve7ktjno4gxayupkdmq (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_lrkwturve7ktjno4gxayupkdmq has no route on this box: its local data is removed and no route is put back
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_lrkwturve7ktjno4gxayupkdmq is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_okgx36zvu63paxdgxw33ph5454 (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_okgx36zvu63paxdgxw33ph5454: remove its local data: injected failure at remove-local
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_okgx36zvu63paxdgxw33ph5454 is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_5glxx2to6fxiukdo5s3u7m7maa (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_5glxx2to6fxiukdo5s3u7m7maa: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_zm4bc32mdzfrycjrxvelwkb53i (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zm4bc32mdzfrycjrxvelwkb53i rollback deferred: no fence generation: registry busy
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_opowdqlzna3tdhmjgbapl3hm5u (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_opowdqlzna3tdhmjgbapl3hm5u rollback deferred: no fence generation: the generation source answered 0
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_lh6zfe235gvlgzlnjed6s7nurm (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_edq2bjqkf35625oxs4fcyjfpoi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_yce57f577iz47jw6sn4s6vnney (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 21:25:32 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_6xsvswdasdkxqk6yof2cy4y3ou (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_6xsvswdasdkxqk6yof2cy4y3ou rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_6xsvswdasdkxqk6yof2cy4y3ou is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_eb5mrrisybicngn3djb4keqhiu (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_eb5mrrisybicngn3djb4keqhiu rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_eb5mrrisybicngn3djb4keqhiu is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_srqoqgcvhcqrmvgok6w6oive4a (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_srqoqgcvhcqrmvgok6w6oive4a rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_srqoqgcvhcqrmvgok6w6oive4a is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_zfhnc7iaj5y5wss5ndf35birbi (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zfhnc7iaj5y5wss5ndf35birbi: retire its instance key: the env file is a symlink
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zfhnc7iaj5y5wss5ndf35birbi rolled back to its original route
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_zfhnc7iaj5y5wss5ndf35birbi is cleaned up and its journal row retired
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_l3u4xis6pnpx3mbeh3zd7gci6i (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:25:32 provision: a.shop.example.com: swapped for import sd_j2ajr3dp2laripdpimgpfmidou (the original route is journaled)
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_6omczky2rt7zlklkfhmfvqezmy refused: there is no route at a.shop.example.com to replace
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_wm5spva76ngcxg6or4spqimkl4 refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:25:32 provision: a.shop.example.com: import sd_jk7xudvgdv3j2uhx4zekg6hyry refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_dexbdqal2cz6o5tqz2f4w2w46e refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_5xnyl2yekrwfto7se4unmk5jqy refused: the route belongs to app other
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_7i4p63poexkmkmscy2bfvm3mt4 refused: the route is filed under another account
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_hlp5xbcbseshd4swircocsllce refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_d7glidswjmna3cewq3jcdpfrym refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_azs5bbtuqtqm3ctliwvwux46am refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_pglrr7tbwqim7t7ph435ywy5oi refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_b2apqqeukhwffztzjhysew6ofu refused: the route is not an address route this import can take over (socket)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_cvmqlwofsixsupglguhrf6cl6m refused: the route is not an address route this import can take over (no address)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_4mv67jrdpwo75pzujdgrnsdo4y refused: the route is not an address route this import can take over (provisioned)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_ljt2ie3m6v47d4xtlhws7q2ueu refused: the route is not an address route this import can take over (kind static)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_5mafnint5yh6ibgvb6sz3e6j2e refused: the route is not an address route this import can take over (backing)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_o57vlrwha2yfo2jk2y3kxrwq5y refused: the route is not an address route this import can take over (tunnel)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_qryqdfbjnldvxm2uoft32nlpna refused: the route is not an address route this import can take over (caps)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_qwt55nm2u2vdx5jes3bur4ucbe refused: the route is not an address route this import can take over (platform)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_7k4qmas2xikjw6p3jqpiycsptu refused: the route is not an address route this import can take over (canary)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_4ms5prgtd62p2zjil4vv476b24 refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_pduxgang7g773li5nibkihnlp4 refused: the route is not an address route this import can take over (region)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_73j24kf4nunfcs5ug3jdojvfki refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_mwonws5xcpvsael3xnjijvukwy refused: the route is not an address route this import can take over (hibernate)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_5zcj2svzadfsbhwmv6dloafraq refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_vhenm6myefk3l4ym47p3y3xr6u refused: this box does not run instances nested
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_cvyfqqi2uqx7rum5kbnvepuijy refused: this box does not run instances nested
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_fnkx6yhikx5glz52qiek34j66i refused: claim control is on for this box
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_s33j6bpllxwcg6pr2ckiey47au (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_ngjxdokaik2stc7xl7mhwnmxia (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 21:25:33 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_duhftv6kgsbz2tmlu2tfzeav4a: its verify hash is not a sha256
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_rg3y2sshl6xbqulyngzvaxz3na: its verify hash is not a sha256
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_txzkxhjrx2id5woniqo3vin7u4: its verify hash is not a sha256
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 21:25:33 provision: A.shop.example.com/..: not swapping for import sd_qigedxvfmkcqbny7ym753ejuay: its host or seed id is malformed (seeds: not a host)
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_dye7rwts57rm3gp4q3wob4oomy: its record does not describe a nested, hibernating instance
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_kml3fsjnqbn5tbncih32mufllm: its record does not describe a nested, hibernating instance
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_zftxzatgu2qhsg2mmfipq7cpmu: its record does not describe a nested, hibernating instance
2026/10/24 21:25:33 provision: a.shop.example.com: not swapping for import sd_fy2sndejqd5wa7rm2hmee5wf3i: its record does not describe a nested, hibernating instance
2026/10/24 21:25:33 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_href6th24rio2lu7vb4r2uvg5i (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_6ltrbyifzpz4onxpwedgwavfbu (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_l4ca4lcxuni26e7njdudkstms4 (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 21:25:33 provision: dry-run: would consider swapping a.shop.example.com for import sd_e54nmbanc7fla6ulkp6piw4qqe
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_s4ljetm3ch2brwplwlyl5ya67i rolled back to its original route
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_s4ljetm3ch2brwplwlyl5ya67i: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_s4ljetm3ch2brwplwlyl5ya67i is cleaned up and its journal row retired
2026/10/24 21:25:33 provision: a.shop.example.com: import sd_6icifffijigqw5vepckwq346mu left alone: the import journal holds seed sd_cv6adlnhpxncsvuckkzltd4u3e at swapped
2026/10/24 21:25:33 provision: a.shop.example.com: not minting import sd_czfjihj3luzhbyfw6rbziw7uky: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:25:33 provision: a.shop.example.com: not minting import sd_lna3t3zbbj6i676gnej6477gzy: it is placed but not committed, and only its swap makes its row
2026/10/24 21:25:33 provision: a.shop.example.com: swapped for import sd_ndvv5o7yqckzbgmn3ag5ndhlm4 (the original route is journaled)
2026/10/24 21:25:33 provision: a.shop.example.com: not minting import sd_ndvv5o7yqckzbgmn3ag5ndhlm4: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:25:33 provision: a.shop.example.com: not minting import sd_ndvv5o7yqckzbgmn3ag5ndhlm4: it is placed but not committed, and only its swap makes its row
2026/10/24 21:25:33 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:25:33 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 21:25:33 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 21:25:33 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 21:25:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 21:25:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 21:25:33 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:25:33 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 21:25:33 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 21:25:33 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:25:33 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:25:33 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 21:25:33 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 21:25:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 21:25:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:25:33 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:33 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:25:33 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 21:25:33 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:25:34 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:25:34 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 21:25:34 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:25:34 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:25:34 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:34 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 21:25:34 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:34 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:34 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:25:34 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:25:34 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:25:34 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:25:34 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:25:34 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:25:34 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:25:34 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 21:25:34 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:34 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:34 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:25:34 provision: record view: change index unusable (get refused): walking
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:25:34 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:34 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:25:34 provision: an app's records could not be listed: they are skipped
2026/10/24 21:25:34 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:25:34 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:25:34 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:25:34 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:25:34 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 21:25:34 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:25:34 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:25:34 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:25:34 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:25:34 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:25:34 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:25:34 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:25:34 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:25:34 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:25:34 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:25:34 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 21:25:34 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (11.68s)
plan14_root_reads_test.go:600: root read census: 40 discrepancies
changed root read internal/activator/activator.go:Service.doWake; review class, task, sites and digest
changed root read internal/activator/activator.go:Service.respawnWorkerLocked; review class, task, sites and digest
changed root read internal/activator/backend_exec.go:ExecBackend.spawnUnitFrom; review class, task, sites and digest
changed root read internal/activator/layout.go:Classify; review class, task, sites and digest
changed root read internal/activator/prepare.go:ExecBackend.prepareAt; review class, task, sites and digest
changed root read internal/activator/prepare.go:ExecBackend.prepareKeyedAt; review class, task, sites and digest
changed root read internal/host/agent.go:runAgent; review class, task, sites and digest
changed root read internal/instancectl/server.go:parseRequest; review class, task, sites and digest
changed root read internal/instancectl/server.go:type Server; review class, task, sites and digest
changed root read internal/ops/hibernate_verify.go:HibernatingHosts; review class, task, sites and digest
changed root read internal/ops/litestream.go:FleetDBsWithSkipped; review class, task, sites and digest
changed root read internal/ops/ops.go:runLitestream; review class, task, sites and digest
changed root source envelope internal/activator
changed root source envelope internal/agentsock
changed root source envelope internal/host
changed root source envelope internal/host/units
changed root source envelope internal/instancectl
changed root source envelope internal/instanceexec
changed root source envelope internal/ops
unclassified root read internal/activator/activator.go:Service.agentSocketFor whole:Route@148
unclassified root read internal/activator/backend_unit.go:SplitBackend.perHostAgentSocket file:Exec@154,registry:Backing@82,whole:Route@217
unclassified root read internal/activator/layout.go:WorkerManaged registry:Backing@71,registry:Tunnel@91
unclassified root read internal/activator/prepare.go:ExecBackend.agentSocketFor whole:Route@102
unclassified root read internal/activator/prepare.go:ExecBackend.perHostAgentSocket whole:Route@141,whole:Route@96
unclassified root read internal/activator/prepare.go:ExecBackend.prepareWorker whole:Route@210
unclassified root read internal/activator/unitproc.go:ExecBackend.newWorkerUnitProc registry:Host@244,registry:Host@286,whole:Route@167,whole:Route@171
unclassified root read internal/host/units/carlos-instance-worker@.service:external file:external@0
unclassified root read internal/ops/litestream.go:ActivatorReplicated registry:Backing@59,registry:DBPath@97,registry:Hibernate@117
unclassified root read internal/ops/litestream.go:ActivatorReplicatedPaths registry:DBPath@257,whole:Route@178,whole:Route@212
changed legacy declaration internal/activator/activator.go:Service.doWake
changed legacy declaration internal/activator/backend_exec.go:ExecBackend.spawnUnitFrom
changed legacy declaration internal/activator/layout.go:Classify
changed legacy declaration internal/activator/prepare.go:ExecBackend.prepareAt
changed legacy declaration internal/activator/prepare.go:ExecBackend.prepareKeyedAt
changed legacy declaration internal/instancectl/server.go:parseRequest
changed legacy declaration internal/instancectl/server.go:type Server
changed legacy declaration internal/ops/hibernate_verify.go:HibernatingHosts
changed legacy declaration internal/ops/litestream.go:FleetDBsWithSkipped
changed legacy declaration internal/ops/ops.go:runLitestream
changed reclassified declaration internal/activator/activator.go:Service.respawnWorkerLocked
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 20.238s
ok github.com/carlosframework/platform/internal/publicrelease 0.007s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.547s
ok github.com/carlosframework/platform/internal/relay 7.237s
ok github.com/carlosframework/platform/internal/release 25.599s
ok github.com/carlosframework/platform/internal/rollout 3.780s
ok github.com/carlosframework/platform/internal/rolloutwords 0.006s
ok github.com/carlosframework/platform/internal/rootaws 0.011s
ok github.com/carlosframework/platform/internal/rootparticipation 0.059s
ok github.com/carlosframework/platform/internal/rootstate 0.030s
ok github.com/carlosframework/platform/internal/runs 0.605s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.719s
ok github.com/carlosframework/platform/internal/seeds 0.017s
ok github.com/carlosframework/platform/internal/sim/engines 2.482s
ok github.com/carlosframework/platform/internal/sim/hostlife 9.920s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 3.953s
ok github.com/carlosframework/platform/internal/status 0.078s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.308s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.009s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.006s
ok github.com/carlosframework/platform/internal/unixdial 0.002s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.006s
ok github.com/carlosframework/platform/internal/version 0.003s
ok github.com/carlosframework/platform/internal/vet 1.665s
FAIL
=== internal_console@0 (80 s)
ok github.com/carlosframework/platform/internal/console 78.463s
=== internal_console@1 (43 s)
ok github.com/carlosframework/platform/internal/console 41.466s
=== internal_console@2 (41 s)
ok github.com/carlosframework/platform/internal/console 39.264s
=== internal_console@3 (41 s)
ok github.com/carlosframework/platform/internal/console 39.448s
=== internal_console@4 (42 s)
ok github.com/carlosframework/platform/internal/console 41.029s
=== internal_console@5 (41 s)
ok github.com/carlosframework/platform/internal/console 39.133s
=== internal_console@6 (36 s)
ok github.com/carlosframework/platform/internal/console 34.923s
=== internal_console@7 (46 s)
ok github.com/carlosframework/platform/internal/console 44.394s
=== root@0 (109 s)
ok github.com/carlosframework/platform 107.226s
=== root@1 (84 s)
ok github.com/carlosframework/platform 82.450s
=== root@2 (91 s)
using http://127.0.0.1:39523 as a@keymail.dev
using http://127.0.0.1:34877 as vicky@keymail.dev
titogo (staging):
KEY VALUE
PORT 8080
--- FAIL: TestRunBuiltCopyIsApplied (0.14s)
approved_run_copy_test.go:139: RUN-C13: internal/instanceexec/exec.go does not contain the approved text "instance-exec: role <role> is not run, seed, sidecar or cmd"
approved_run_copy_test.go:139: RUN-C14: internal/instanceexec/exit.go does not contain the approved text "instance-exit: role <role> is not run, seed, sidecar or cmd"
2026/10/24 21:25:38 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:38 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A37309": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:38 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:38 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A37309": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:38 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:38 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:38 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A43805": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:38 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:40 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:40 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A33765": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:40 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:40 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:40 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A38431": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:40 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:42 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:42 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A36409": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:42 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:44 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:44 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A46571": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:44 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
using http://127.0.0.1:40681 as paul@keymail.dev
account: ops (bab)
2026/10/24 21:25:46 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A40681": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:46 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
set 1 env var(s) for titogo (default)
using http://127.0.0.1:40681 as paul@keymail.dev
account: ops (bab)
set 1 secret(s) for titogo (default)
account: ops (bab)
account: ops (bab)
using http://127.0.0.1:40681 as paul@keymail.dev
account: ops (bab)
config sync requested for titogo the files land within seconds; programs carlos owns restart themselves, and anything else keeps its old values until you restart it
2026/10/24 21:25:46 console: delivery to mallory@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=mallory%40keymail.dev&from=console%40127.0.0.1%3A40681": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:46 console: could not notify mallory@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:48 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:25:48 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:25:48 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:25:49 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:25:49 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:25:49 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:25:49 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:49 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A35879": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:49 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:25:49 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:25:49 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:25:49 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:25:50 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:25:50 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:25:50 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:25:51 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:25:51 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A40213": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:25:51 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:32 console: db read dbr_t32gwhej43mjur352n77ohhdwq: relayed 1 chunks (101 bytes) in 41.03s, ended done
2026/10/24 21:26:32 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:26:32 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:26:32 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:26:33 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:26:33 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:26:33 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:26:33 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:26:33 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A35757": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:33 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:34 console: db read dbr_wpncm5hdp5duhqzguci4rvztp4: relayed 1 chunks (1070 bytes) in 1.001s, ended done
2026/10/24 21:26:35 console: db read dbr_iw3jtxwuyiv2wxwa74cii2k2hm: relayed 1 chunks (805 bytes) in 1.001s, ended done
2026/10/24 21:26:35 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:26:35 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:26:35 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:26:36 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:26:36 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:26:36 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:26:36 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:26:36 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A42117": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:36 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:37 console: db read dbr_ax6g5tcxywbfz5w4vm2hbkw4ru: relayed 2 chunks (8386 bytes) in 1.043s, ended client gone
using http://127.0.0.1:36657 as paul@keymail.dev
account: ops (bab)
attached tickets.example.com -> default.go.tito.io
DNS: pointing
using http://127.0.0.1:36657 as paul@keymail.dev
account: ops (bab)
attached carloku.example.com -> default.go.tito.io
2026/10/24 21:26:38 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:26:38 console: delivery to rival@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=rival%40keymail.dev&from=console%40127.0.0.1%3A41297": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:38 console: could not notify rival@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:38 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A41297": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:38 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:46 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A33105": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:46 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:47 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:26:47 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A32985": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:26:47 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:26:48 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 21:26:48 domains: minted platform domain blog.mdbbndjqx.oncarlos.test (upstream blog.kass.test)
2026/10/24 21:26:48 domains: minted platform domain blog.gdpkwhwrw.oncarlos.test (upstream blog.woodstar.test)
2026/10/24 21:26:48 domains: minted platform domain oldsite.bab.oncarlos.test (upstream old.test)
2026/10/24 21:26:48 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 21:26:48 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 21:26:48 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
FAIL
FAIL github.com/carlosframework/platform 89.250s
FAIL
FAILED: rest root@2
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 8aa16e0f5..63f39416b 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -2466,7 +2466,7 @@ func (a *Service) doWake(ctx context.Context, e *actEntry) (*wokenInstance, erro |
| ident release.StagedIdentity |
| pend pendingGeneration |
| ) |
| - env := workerEnv(a.agentSocket(), host, token) |
| + env := workerEnv(a.agentSocketFor(rt), host, token) |
| if plan.planned() { |
| // A planned worker (zero-downtime spec §6.5): the named build, |
| // behind planGate at each exec, with startWorker's publication |
| @@ -3311,6 +3311,24 @@ func (a *Service) agentSocket() string { |
| return a.cfg.AgentSocket |
| } |
| +// agentSocketRouter is a backend that knows a row's own agent socket: a |
| +// nested worker's listener in its run directory (sidecar isolation D3). |
| +type agentSocketRouter interface { |
| + perHostAgentSocket(rt registry.Route) (string, bool) |
| +} |
| + |
| +// agentSocketFor is the agent socket the worker for rt dials, workerEnv's |
| +// socket: the backend's per-host path for a nested worker row, the shared |
| +// socket (agentSocket) for every other row, exactly as before D3. |
| +func (a *Service) agentSocketFor(rt registry.Route) string { |
| + if r, ok := a.backend.(agentSocketRouter); ok { |
| + if path, ok := r.perHostAgentSocket(rt); ok { |
| + return path |
| + } |
| + } |
| + return a.agentSocket() |
| +} |
| + |
| // workerEnv is what a spawn hands the child: where to call back, who it is, |
| // and this incarnation's credential. The backend applies it OVER the tenant's |
| // own env file so a tenant cannot forge its own token (StartWorker's |
| @@ -3620,7 +3638,7 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string, ont |
| if refused == nil { |
| // No registry write (spec §7.9): see doWake. |
| if tok, err = newWorkerToken(); err == nil { |
| - proc, ident, pend, err = a.startWorker(ctx, rt, workerEnv(a.agentSocket(), host, tok), onto, gate) |
| + proc, ident, pend, err = a.startWorker(ctx, rt, workerEnv(a.agentSocketFor(rt), host, tok), onto, gate) |
| } |
| } |
| cancel() |
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index 6656e8ee9..4ed763f7b 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -22,6 +22,7 @@ import ( |
| "syscall" |
| "time" |
| + "github.com/carlosframework/platform/internal/agentsock" |
| "github.com/carlosframework/platform/internal/applog" |
| "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/edge" |
| @@ -188,6 +189,13 @@ type ExecBackend struct { |
| // channel not configured" rather than dialing nothing. A flat or |
| // foreign host never needs it, and a nil Control does not touch them. |
| Control Control |
| + // AgentSock is the agent socket's server, the one that serves the |
| + // shared socket (Cfg.AgentSocket). A nested worker's own listener, in |
| + // its run directory, is bound on it (sidecar isolation D3), so the two |
| + // share one verb set and one token check, and the shared socket knows |
| + // to refuse a host its pinned listener serves. Nil refuses every nested |
| + // worker's prepare: a worker with no channel back is not started. |
| + AgentSock *agentsock.Server |
| // UnitState answers `systemctl show` for the observer (design §6.5). |
| // Nil: systemctlUnitState, the real systemctl binary. |
| UnitState UnitState |
| @@ -2421,7 +2429,7 @@ func (b *ExecBackend) spawnUnitFrom(ctx context.Context, rt registry.Route, sock |
| return nil, "", err |
| } |
| } |
| - p, ctl, logs, fileEnv, keyLayer, err := b.prepareKeyedAt(rt, instancectl.RoleRun, sock, gen) |
| + p, ctl, logs, fileEnv, keyLayer, _, err := b.prepareKeyedAt(rt, instancectl.RoleRun, sock, gen) |
| if err != nil { |
| return nil, "", err |
| } |
diff --git a/internal/activator/backend_unit.go b/internal/activator/backend_unit.go| index e6b64abb3..29e89d1e6 100644 |
| --- a/internal/activator/backend_unit.go |
| +++ b/internal/activator/backend_unit.go |
| @@ -401,6 +401,18 @@ func (b *SplitBackend) Layout(rt registry.Route) (Layout, string) { |
| return LayoutFlat, "" |
| } |
| +// perHostAgentSocket forwards to the exec side: a unit-backed row is never |
| +// a nested worker, and dials the shared socket. |
| +func (b *SplitBackend) perHostAgentSocket(rt registry.Route) (string, bool) { |
| + if rt.Backing == registry.BackingUnit { |
| + return "", false |
| + } |
| + if s, ok := b.Exec.(agentSocketRouter); ok { |
| + return s.perHostAgentSocket(rt) |
| + } |
| + return "", false |
| +} |
| + |
| func (b *SplitBackend) StartSidecar(ctx context.Context, rt registry.Route) (Proc, error) { |
| return b.pick(rt).StartSidecar(ctx, rt) |
| } |
diff --git a/internal/activator/prepare.go b/internal/activator/prepare.go| index 39eadb475..3eef175ea 100644 |
| --- a/internal/activator/prepare.go |
| +++ b/internal/activator/prepare.go |
| @@ -17,6 +17,7 @@ import ( |
| "strings" |
| "time" |
| + "github.com/carlosframework/platform/internal/agentsock" |
| "github.com/carlosframework/platform/internal/edge" |
| "github.com/carlosframework/platform/internal/instancectl" |
| "github.com/carlosframework/platform/internal/instancefs" |
| @@ -46,25 +47,91 @@ func (b *ExecBackend) prepare(rt registry.Route, role instancectl.Role) (instanc |
| // gen is the generation unit being prepared (0 for the original templates): |
| // it keys the log socket and the exit records cleared (zero-downtime spec |
| // §7.6). |
| +// |
| +// The worker role is prepareWorker's: its agent listener must reach the |
| +// proc that closes it, and prepareAt has no way to hand it over. |
| func (b *ExecBackend) prepareAt(rt registry.Route, role instancectl.Role, sock string, gen uint64) (instancePaths, *edge.ControlListener, *logListener, map[string]string, error) { |
| - p, ctl, logs, env, _, err := b.prepareKeyedAt(rt, role, sock, gen) |
| + if role == instancectl.RoleWorker { |
| + return b.pathsFor(rt), nil, nil, nil, fmt.Errorf("prepare %s: the worker role is prepared by prepareWorker", rt.Host) |
| + } |
| + p, ctl, logs, env, _, _, err := b.prepareKeyedAt(rt, role, sock, gen) |
| return p, ctl, logs, env, err |
| } |
| +// prepareWorker is prepare for a nested worker (sidecar isolation D5): the |
| +// directories, the env layers read through instancefs, no admin token, the |
| +// log listener, and the worker's own agent listener (D3), bound and |
| +// answering before it returns. The caller hands logs and agent to the |
| +// worker's unitProc (newWorkerUnitProc), whose closeListeners closes both |
| +// after the proc's done, or closes them itself on a failed start. |
| +func (b *ExecBackend) prepareWorker(rt registry.Route) (instancePaths, *logListener, *agentsock.Pinned, map[string]string, instancekey.Layer, error) { |
| + p, _, logs, env, keyLayer, agent, err := b.prepareKeyedAt(rt, instancectl.RoleWorker, "", 0) |
| + return p, logs, agent, env, keyLayer, err |
| +} |
| + |
| +// agentSocketName is a nested worker's agent socket, in its run directory. |
| +const agentSocketName = "agent.sock" |
| + |
| +// agentSocketFor is the agent socket a worker for rt dials (sidecar |
| +// isolation D3): its own listener at <InstanceRunDir(host)>/agent.sock for |
| +// a nested worker row, the shared socket otherwise. |
| +func (b *ExecBackend) agentSocketFor(rt registry.Route) string { |
| + if path, ok := b.perHostAgentSocket(rt); ok { |
| + return path |
| + } |
| + if b.Cfg.AgentSocket != "" { |
| + return b.Cfg.AgentSocket |
| + } |
| + return DefaultAgentSocket |
| +} |
| + |
| +// perHostAgentSocket is agentSocketFor's per-host half: the pinned |
| +// listener's path and true for a nested worker row, false for every other |
| +// row, which dials the shared socket. The run directory honours |
| +// RunDirOverride as pathsFor does; in production it is |
| +// provision.InstanceRunDir(rt.Host). |
| +func (b *ExecBackend) perHostAgentSocket(rt registry.Route) (string, bool) { |
| + if !WorkerManaged(rt) { |
| + return "", false |
| + } |
| + p := b.pathsFor(rt) |
| + if p.layout != LayoutNested { |
| + return "", false |
| + } |
| + return filepath.Join(p.runDir, agentSocketName), true |
| +} |
| + |
| // prepareKeyedAt is prepareAt returning, with the env, the layer its |
| // CARLOS_INSTANCE_KEY came from, decided by the same read (instance-key |
| // ruling 29). The nested run spawn keeps both on the generation's handle, |
| // for a run's command against that generation (StartCmd). |
| -func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, sock string, gen uint64) (instancePaths, *edge.ControlListener, *logListener, map[string]string, instancekey.Layer, error) { |
| +// |
| +// For the worker role it also returns the worker's own agent listener |
| +// (sidecar isolation D3), nil for every other role. Every listener it bound |
| +// is closed before an error return. |
| +func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, sock string, gen uint64) (instancePaths, *edge.ControlListener, *logListener, map[string]string, instancekey.Layer, *agentsock.Pinned, error) { |
| p := b.pathsFor(rt) |
| + var agentPath string |
| + if role == instancectl.RoleWorker { |
| + // Checked before anything is made: a worker's prepare is the |
| + // nested arm's, and the listener needs a server to be bound on. |
| + path, ok := b.perHostAgentSocket(rt) |
| + if !ok { |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("prepare worker %s: not a nested worker row (layout %s)", rt.Host, p.layout) |
| + } |
| + if b.AgentSock == nil { |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("prepare worker %s: agent socket server not configured", rt.Host) |
| + } |
| + agentPath = path |
| + } |
| if err := os.MkdirAll(p.dataDir, 0o700); err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| if err := os.MkdirAll(p.runDir, 0o750|os.ModeSetgid); err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| if err := os.MkdirAll(filepath.Dir(b.logSocketFor(rt.Host, role, gen)), 0o700); err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| var env map[string]string |
| var keyLayer instancekey.Layer // the layer env's key came from, read with it (ruling 29) |
| @@ -74,15 +141,26 @@ func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, s |
| // (design §3.2), so the token file is minted and read through |
| // instancefs rather than a plain path-based open. |
| if err := EnsureAdminTokenBeneath(p.dataDir, rt.Host+".env"); err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("admin token for %s: %w", rt.Host, err) |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("admin token for %s: %w", rt.Host, err) |
| } |
| if _, err := b.ensureInstanceKeyFor(context.Background(), rt, p); err != nil { // prepareAt has no ctx to bound the mint's bucket read; configReadTimeout does |
| - return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("instance key for %s: %w", rt.Host, err) |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("instance key for %s: %w", rt.Host, err) |
| + } |
| + var err error |
| + env, keyLayer, err = b.instanceEnvLayer(rt) |
| + if err != nil { |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| + case instancectl.RoleWorker: |
| + // A worker's env is the instance's two layers (as StartWorkerStaged |
| + // reads them for a flat worker), read beneath the instance |
| + // directory for this nested row (instanceEnvParts). No admin token: |
| + // a worker has no control socket for one to open. No mint either: |
| + // a worker never mints its key (ruling 12). |
| var err error |
| env, keyLayer, err = b.instanceEnvLayer(rt) |
| if err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| case instancectl.RoleCmd: |
| // A run's environment is the one its generation launched with |
| @@ -93,7 +171,7 @@ func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, s |
| var err error |
| env, err = readEnvFileBeneath(p.dataDir, filepath.Base(p.sidecarEnv)) |
| if err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| keyLayer = sidecarKeyLayer(env) |
| } |
| @@ -132,12 +210,34 @@ func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, s |
| // is) can bind its socket there. |
| if dir := filepath.Dir(sock); dir != p.runDir { |
| if err := os.MkdirAll(dir, 0o750|os.ModeSetgid); err != nil { |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| } |
| _ = os.Remove(sock) |
| ctl = b.listenControl(rt, sock+".control", p.envFile, gen) |
| } |
| + // A nested worker's own agent listener (sidecar isolation D3), where |
| + // the run role binds its control listener: before the start request, |
| + // so the worker's first /v1/ready finds it, and closed with the proc's |
| + // other listeners after its done. A worker with no channel back is not |
| + // started, so a bind failure fails prepare (unlike the control |
| + // listener's, which an instance runs without). |
| + var agent *agentsock.Pinned |
| + if agentPath != "" { |
| + var err error |
| + agent, err = b.AgentSock.ListenPinned(agentPath, rt.Host) |
| + if err != nil { |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("agent listener for %s: %w", rt.Host, err) |
| + } |
| + } |
| + closeBound := func() { |
| + if ctl != nil { |
| + ctl.Close() |
| + } |
| + if agent != nil { |
| + agent.Close() |
| + } |
| + } |
| // A replica's roles are gated like a flat launch (ruling 26): the seed |
| // and the sidecar have no control socket, the run role has the one just |
| // bound or none. The sidecar's key, if its own file holds one, is |
| @@ -146,10 +246,8 @@ func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, s |
| // read none for it. |
| if role != instancectl.RoleCmd { |
| if err := b.replicaKeyGate(rt, env, keyLayer, ctl != nil); err != nil { |
| - if ctl != nil { |
| - ctl.Close() |
| - } |
| - return p, nil, nil, nil, instancekey.LayerNone, err |
| + closeBound() |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, err |
| } |
| } |
| // The cmd role's output goes only to its own chunks, never to the |
| @@ -159,13 +257,11 @@ func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, s |
| var err error |
| logs, err = listenLogs(b.logSocketFor(rt.Host, role, gen), b.logEmitter(rt)) |
| if err != nil { |
| - if ctl != nil { |
| - ctl.Close() |
| - } |
| - return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("log listener for %s %s: %w", rt.Host, role, err) |
| + closeBound() |
| + return p, nil, nil, nil, instancekey.LayerNone, nil, fmt.Errorf("log listener for %s %s: %w", rt.Host, role, err) |
| } |
| } |
| - return p, ctl, logs, env, keyLayer, nil |
| + return p, ctl, logs, env, keyLayer, agent, nil |
| } |
| // exitPrefix is the exit records prepare clears before a start: this |
diff --git a/internal/activator/prepare_test.go b/internal/activator/prepare_test.go| new file mode 100644 |
| index 000000000..3b21cb8c0 |
| --- /dev/null |
| +++ b/internal/activator/prepare_test.go |
| @@ -0,0 +1,336 @@ |
| +//go:build linux |
| + |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "net" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/agentsock" |
| + "github.com/carlosframework/platform/internal/instancectl" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// pinWorkers is agentsock.Workers for the prepare tests: one live token per |
| +// host, and the hosts whose heartbeats got through. |
| +type pinWorkers struct { |
| + mu sync.Mutex |
| + tokens map[string]string |
| + contacts []string |
| +} |
| + |
| +func (w *pinWorkers) WorkerToken(host string) string { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + return w.tokens[host] |
| +} |
| +func (w *pinWorkers) NoteWorkerContact(host string) { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + w.contacts = append(w.contacts, host) |
| +} |
| +func (w *pinWorkers) SetWorkerNextDue(string, int64) error { return nil } |
| +func (w *pinWorkers) SleepWorker(string) {} |
| +func (w *pinWorkers) NoteWorkerReady(string, string) {} |
| + |
| +func (w *pinWorkers) contactCount() int { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + return len(w.contacts) |
| +} |
| + |
| +const pinWorkerHost = "bot.shop.abc123.sidecar.internal" |
| + |
| +// nestedWorkerBackend is nestedBackend for a worker row in the nested |
| +// layout (D2: kind sidecar, no socket, the nested database path), with the |
| +// agent socket server wired as the agent wires it. |
| +func nestedWorkerBackend(t *testing.T, ctl *fakeControl, unit *fakeUnit) (*ExecBackend, registry.Route, *pinWorkers) { |
| + t.Helper() |
| + b, _ := nestedBackend(t, ctl, unit) |
| + rt := registry.Route{Host: pinWorkerHost, Kind: "sidecar", Args: `["work"]`, |
| + DBPath: provision.NestedDBPath(b.DataDir, pinWorkerHost)} |
| + if l, why := b.Layout(rt); l != LayoutNested { |
| + t.Fatalf("fixture worker row classifies as %s (%s), want nested", l, why) |
| + } |
| + w := &pinWorkers{tokens: map[string]string{pinWorkerHost: "tok"}} |
| + b.AgentSock = agentsock.New(w) |
| + return b, rt, w |
| +} |
| + |
| +// heartbeatOver posts one heartbeat for host over the unix socket at path. |
| +// A dial failure is returned as status 0. |
| +func heartbeatOver(path, host, tok string) int { |
| + cl := &http.Client{ |
| + Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&net.Dialer{}).DialContext(ctx, "unix", path) |
| + }}, |
| + Timeout: 2 * time.Second, |
| + } |
| + req, _ := http.NewRequest("POST", "http://agent/v1/heartbeat", nil) |
| + req.Header.Set("Carlos-Sidecar-Host", host) |
| + req.Header.Set("Authorization", "Bearer "+tok) |
| + resp, err := cl.Do(req) |
| + if err != nil { |
| + return 0 |
| + } |
| + resp.Body.Close() |
| + return resp.StatusCode |
| +} |
| + |
| +// sharedHeartbeat is a heartbeat for host on the shared socket's handler. |
| +func sharedHeartbeat(s *agentsock.Server, host, tok string) int { |
| + req := httptest.NewRequest("POST", "/v1/heartbeat", nil) |
| + req.Header.Set("Carlos-Sidecar-Host", host) |
| + req.Header.Set("Authorization", "Bearer "+tok) |
| + rec := httptest.NewRecorder() |
| + s.Handler().ServeHTTP(rec, req) |
| + return rec.Code |
| +} |
| + |
| +// TestPrepareWorkerBindsAgentSocketBeforeReturn: the worker's own agent |
| +// listener (D3) is at <run dir>/agent.sock, the path workerEnv hands the |
| +// worker, mode 0600, and answering the moment prepareWorker returns, with |
| +// no wait: the worker's first /v1/ready may arrive as soon as root starts |
| +// the unit. The log listener is bound too. |
| +func TestPrepareWorkerBindsAgentSocketBeforeReturn(t *testing.T) { |
| + b, rt, w := nestedWorkerBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{}) |
| + p, logs, agent, _, _, err := b.prepareWorker(rt) |
| + if err != nil { |
| + t.Fatalf("prepareWorker: %v", err) |
| + } |
| + defer logs.Close() |
| + defer agent.Close() |
| + |
| + want := filepath.Join(p.runDir, "agent.sock") |
| + if agent == nil || agent.Path() != want || agent.Host() != rt.Host { |
| + t.Fatalf("agent listener = %+v, want %s for %s", agent, want, rt.Host) |
| + } |
| + if got := b.agentSocketFor(rt); got != want { |
| + t.Fatalf("agentSocketFor = %s, want the listener's path %s", got, want) |
| + } |
| + // No waitFor: bound before return is the property. |
| + if code := heartbeatOver(want, rt.Host, "tok"); code != http.StatusNoContent { |
| + t.Fatalf("heartbeat on the worker's listener straight after prepare: %d, want 204", code) |
| + } |
| + if w.contactCount() != 1 { |
| + t.Fatalf("contacts = %d, want 1", w.contactCount()) |
| + } |
| + fi, err := os.Lstat(want) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fi.Mode()&os.ModeSocket == 0 || fi.Mode().Perm() != 0o600 { |
| + t.Fatalf("agent socket mode = %v, want a 0600 socket", fi.Mode()) |
| + } |
| + c, err := net.Dial("unix", b.logSocket(rt.Host, instancectl.RoleWorker)) |
| + if err != nil { |
| + t.Fatalf("worker log listener not bound: %v", err) |
| + } |
| + c.Close() |
| + if !strings.HasSuffix(b.logSocket(rt.Host, instancectl.RoleWorker), ".worker.sock") { |
| + t.Fatalf("log socket %s is not the worker template's", b.logSocket(rt.Host, instancectl.RoleWorker)) |
| + } |
| +} |
| + |
| +// TestPrepareWorkerHasNoAdminToken: a worker has no control socket, so its |
| +// prepare mints no admin token (the run role's does), and its env is the |
| +// instance's two layers read beneath the instance directory, own over |
| +// config, as a flat worker's are. A symlink planted at the env file is not |
| +// followed. |
| +func TestPrepareWorkerHasNoAdminToken(t *testing.T) { |
| + b, rt, _ := nestedWorkerBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{}) |
| + inst := provision.InstanceDir(b.DataDir, rt.Host) |
| + if err := os.MkdirAll(inst, 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + own := filepath.Join(inst, rt.Host+".env") |
| + if err := os.WriteFile(filepath.Join(inst, rt.Host+".config.env"), []byte("SHARED=cfg\nONLY_CFG=1\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(own, []byte("SHARED=own\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + _, logs, agent, env, _, err := b.prepareWorker(rt) |
| + if err != nil { |
| + t.Fatalf("prepareWorker: %v", err) |
| + } |
| + logs.Close() |
| + agent.Close() |
| + if _, ok := env["CARLOS_ADMIN_TOKEN"]; ok { |
| + t.Fatal("a worker's env carries CARLOS_ADMIN_TOKEN") |
| + } |
| + if env["SHARED"] != "own" || env["ONLY_CFG"] != "1" { |
| + t.Fatalf("env = %v, want the two layers with own over config", env) |
| + } |
| + raw, err := os.ReadFile(own) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if string(raw) != "SHARED=own\n" { |
| + t.Fatalf("prepareWorker wrote the env file (an admin token?): %q", raw) |
| + } |
| + |
| + // Through instancefs: a link at the env file is refused, not read. |
| + outside := filepath.Join(t.TempDir(), "elsewhere.env") |
| + if err := os.WriteFile(outside, []byte("STOLEN=1\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Remove(own); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Symlink(outside, own); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, logs, agent, env, _, err := b.prepareWorker(rt); err == nil { |
| + logs.Close() |
| + agent.Close() |
| + t.Fatalf("prepareWorker followed a symlinked env file: env %v", env) |
| + } |
| + if _, err := os.Lstat(filepath.Join(b.pathsFor(rt).runDir, "agent.sock")); !os.IsNotExist(err) { |
| + t.Fatalf("a failed prepare left the agent socket bound: %v", err) |
| + } |
| +} |
| + |
| +// TestPrepareWorkerAgentListenerClosesAfterDone: the worker's unitProc |
| +// owns the agent listener. An unproven stop leaves it answering (a worker |
| +// that may still run keeps its channel), and once the unit is proven gone |
| +// closeListeners closes it with no second Stop: the socket file is gone and |
| +// the host is the shared socket's again. |
| +func TestPrepareWorkerAgentListenerClosesAfterDone(t *testing.T) { |
| + ctl := &fakeControl{inv: "inv1"} |
| + unit := &fakeUnit{} |
| + b, rt, _ := nestedWorkerBackend(t, ctl, unit) |
| + p, logs, agent, _, _, err := b.prepareWorker(rt) |
| + if err != nil { |
| + t.Fatalf("prepareWorker: %v", err) |
| + } |
| + if code := sharedHeartbeat(b.AgentSock, rt.Host, "tok"); code != http.StatusUnauthorized { |
| + t.Fatalf("the moved worker's token on the shared socket: %d, want 401", code) |
| + } |
| + unit.set("active", "inv1", true, true) |
| + up := b.newWorkerUnitProc(rt, p, "inv1", logs, agent) |
| + if up.role != instancectl.RoleWorker || up.unit != instancectl.UnitName(instancectl.RoleWorker, rt.Host) { |
| + t.Fatalf("proc = role %s unit %s", up.role, up.unit) |
| + } |
| + |
| + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) |
| + defer cancel() |
| + if err := up.Stop(ctx); !errors.Is(err, ErrStillRunning) { |
| + t.Fatalf("Stop = %v, want ErrStillRunning", err) |
| + } |
| + if code := heartbeatOver(agent.Path(), rt.Host, "tok"); code != http.StatusNoContent { |
| + t.Fatalf("agent listener after an unproven stop: %d, want 204", code) |
| + } |
| + |
| + unit.set("inactive", "inv1", false, false) |
| + waitFor(t, "the worker unit to exit", func() bool { return up.Exited() }) |
| + waitFor(t, "the agent listener to close", func() bool { |
| + _, err := os.Lstat(agent.Path()) |
| + return os.IsNotExist(err) |
| + }) |
| + if code := heartbeatOver(agent.Path(), rt.Host, "tok"); code != 0 { |
| + t.Fatalf("agent listener still answers after done: %d", code) |
| + } |
| + if code := sharedHeartbeat(b.AgentSock, rt.Host, "tok"); code != http.StatusNoContent { |
| + t.Fatalf("the host on the shared socket once its listener closed: %d, want 204", code) |
| + } |
| +} |
| + |
| +// TestPrepareWorkerFailedGateClosesTheAgentListener: a refusal after the |
| +// listener is bound (here the replica key gate: a replica worker with no |
| +// verified home key) closes it before prepare returns. |
| +func TestPrepareWorkerFailedGateClosesTheAgentListener(t *testing.T) { |
| + b, rt, _ := nestedWorkerBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{}) |
| + rt.Replica = true |
| + if _, logs, agent, _, _, err := b.prepareWorker(rt); err == nil { |
| + logs.Close() |
| + agent.Close() |
| + t.Fatal("a keyless replica worker was prepared") |
| + } |
| + if _, err := os.Lstat(filepath.Join(b.pathsFor(rt).runDir, "agent.sock")); !os.IsNotExist(err) { |
| + t.Fatalf("the refused prepare left the agent socket: %v", err) |
| + } |
| + if code := sharedHeartbeat(b.AgentSock, rt.Host, "tok"); code != http.StatusNoContent { |
| + t.Fatalf("the host stayed pinned after a refused prepare: %d", code) |
| + } |
| +} |
| + |
| +// TestPrepareWorkerRefusesWhatItCannotServe: a flat worker row, a backend |
| +// with no agent socket server, and the generic prepare entry point are all |
| +// refused before anything is bound. |
| +func TestPrepareWorkerRefusesWhatItCannotServe(t *testing.T) { |
| + b, rt, _ := nestedWorkerBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{}) |
| + |
| + flat := rt |
| + flat.DBPath = provision.DefaultDBPath(b.DataDir, rt.Host) |
| + if _, _, _, _, _, err := b.prepareWorker(flat); err == nil { |
| + t.Fatal("prepareWorker bound a pinned listener for a flat worker row") |
| + } |
| + |
| + if _, _, _, _, err := b.prepare(rt, instancectl.RoleWorker); err == nil { |
| + t.Fatal("prepare(RoleWorker) succeeded; its agent listener would have no owner") |
| + } |
| + |
| + b.AgentSock = nil |
| + if _, _, _, _, _, err := b.prepareWorker(rt); err == nil { |
| + t.Fatal("prepareWorker with no agent socket server succeeded") |
| + } |
| + if _, err := os.Lstat(filepath.Join(b.pathsFor(rt).runDir, "agent.sock")); !os.IsNotExist(err) { |
| + t.Fatalf("a refused prepare left an agent socket: %v", err) |
| + } |
| +} |
| + |
| +// TestAgentSocketForNestedWorkerIsPerHost is workerEnv's socket argument |
| +// (D3, D5): the per-host path for a nested worker row, and for every other |
| +// row the shared socket exactly as before, through the composed backend |
| +// the agent runs. |
| +func TestAgentSocketForNestedWorkerIsPerHost(t *testing.T) { |
| + b, rt, _ := nestedWorkerBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{}) |
| + perHost := filepath.Join(b.RunDirOverride, "instances", provision.InstanceID(rt.Host), "agent.sock") |
| + |
| + flatWorker := rt |
| + flatWorker.DBPath = provision.DefaultDBPath(b.DataDir, rt.Host) |
| + _, instance := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| + |
| + if got := b.agentSocketFor(rt); got != perHost { |
| + t.Errorf("nested worker: %s, want %s", got, perHost) |
| + } |
| + if got := b.agentSocketFor(flatWorker); got != DefaultAgentSocket { |
| + t.Errorf("flat worker, default config: %s, want %s", got, DefaultAgentSocket) |
| + } |
| + b.Cfg.AgentSocket = "/s/agent.sock" |
| + if got := b.agentSocketFor(flatWorker); got != "/s/agent.sock" { |
| + t.Errorf("flat worker: %s, want the configured shared socket", got) |
| + } |
| + |
| + a := New(Config{AgentSocket: "/shared/agent.sock"}, &SplitBackend{Exec: b, Unit: &UnitBackend{}}, nil) |
| + for _, tc := range []struct { |
| + name string |
| + rt registry.Route |
| + want string |
| + }{ |
| + {"nested worker", rt, perHost}, |
| + {"flat worker", flatWorker, "/shared/agent.sock"}, |
| + {"instance row", instance, "/shared/agent.sock"}, |
| + {"unit-backed worker", registry.Route{Host: rt.Host, Kind: "sidecar", Backing: registry.BackingUnit, DBPath: rt.DBPath}, "/shared/agent.sock"}, |
| + } { |
| + if got := a.agentSocketFor(tc.rt); got != tc.want { |
| + t.Errorf("Service %s: %s, want %s", tc.name, got, tc.want) |
| + } |
| + } |
| + // A backend that knows nothing of per-host sockets keeps the shared one. |
| + if got := New(Config{AgentSocket: "/shared/agent.sock"}, &fakeBackend{}, nil).agentSocketFor(rt); got != "/shared/agent.sock" { |
| + t.Errorf("plain backend: %s, want the shared socket", got) |
| + } |
| +} |
diff --git a/internal/activator/unitproc.go b/internal/activator/unitproc.go| index b69a2eab4..8e877b142 100644 |
| --- a/internal/activator/unitproc.go |
| +++ b/internal/activator/unitproc.go |
| @@ -11,6 +11,7 @@ import ( |
| "sync/atomic" |
| "time" |
| + "github.com/carlosframework/platform/internal/agentsock" |
| "github.com/carlosframework/platform/internal/edge" |
| "github.com/carlosframework/platform/internal/instancectl" |
| "github.com/carlosframework/platform/internal/instancefs" |
| @@ -73,6 +74,10 @@ type unitProc struct { |
| // from, decided by the read that composed launchEnv (instance-key |
| // ruling 29). Written with launchEnv, never again. |
| launchKeyLayer instancekey.Layer |
| + // agent is a nested worker's own agent listener (sidecar isolation |
| + // D3), nil for every other role. closeListeners closes it with ctl and |
| + // logs, after done. |
| + agent *agentsock.Pinned |
| } |
| // launchEnvProc is a generation handle that kept the file environment its |
| @@ -106,6 +111,17 @@ func (b *ExecBackend) newUnitProcFor(rt registry.Route, role instancectl.Role, g |
| return up |
| } |
| +// newWorkerUnitProc is a nested worker's proc (sidecar isolation D5): the |
| +// worker role's unit, with the log listener and the worker's own agent |
| +// listener prepareWorker bound, both closed by closeListeners after done. |
| +// The listener is set before observe starts, so no close can miss it. |
| +func (b *ExecBackend) newWorkerUnitProc(rt registry.Route, p instancePaths, inv string, logs *logListener, agent *agentsock.Pinned) *unitProc { |
| + up := &unitProc{b: b, rt: rt, role: instancectl.RoleWorker, unit: unitName(instancectl.RoleWorker, rt.Host, 0), slice: instancectl.SliceName(rt.Host), |
| + invocation: inv, runDir: p.runDir, dataDir: p.dataDir, logs: logs, agent: agent, done: make(chan struct{})} |
| + go b.observe(up) |
| + return up |
| +} |
| + |
| // unitName is the unit a role of host runs as: the original template's, or |
| // generation gen's (zero-downtime spec §7.6). |
| func unitName(role instancectl.Role, host string, gen uint64) string { |
| @@ -210,6 +226,9 @@ func (p *unitProc) closeListeners() { |
| if p.logs != nil { |
| p.logs.Close() |
| } |
| + if p.agent != nil { |
| + p.agent.Close() |
| + } |
| } |
| // observe is the goroutine of design §6.5. It proves termination from two |
diff --git a/internal/agentsock/agentsock.go b/internal/agentsock/agentsock.go| index 4158ee387..d2d0d3c5c 100644 |
| --- a/internal/agentsock/agentsock.go |
| +++ b/internal/agentsock/agentsock.go |
| @@ -39,6 +39,7 @@ import ( |
| "net/http" |
| "os" |
| "strings" |
| + "sync" |
| "time" |
| ) |
| @@ -74,6 +75,11 @@ const ( |
| // tell one child from another. The token does that. |
| socketMode = 0o660 |
| + // pinnedMode is a pinned listener's mode (sidecar isolation D3): the |
| + // agent's alone. The run directory it sits in is group-traversable by |
| + // enrolled cross-user consumers; the socket is not theirs to dial. |
| + pinnedMode = 0o600 |
| + |
| // Local calls, all of them. Nothing here should ever be slow, and a |
| // child that stalls mid-request must not pin a connection. |
| readHeaderTimeout = 2 * time.Second |
| @@ -88,32 +94,53 @@ const ( |
| ) |
| // Server serves the agent socket. Zero per-request state: one instance |
| -// answers every child on the box concurrently. |
| +// answers every child on the box concurrently. Its one piece of state is |
| +// which hosts are currently served by a pinned listener of their own |
| +// (ListenPinned), which the shared socket then refuses. |
| type Server struct { |
| w Workers |
| + |
| + mu sync.Mutex |
| + pinned map[string]int // host -> live pinned listeners for it |
| } |
| // New builds the socket server over w. |
| func New(w Workers) *Server { return &Server{w: w} } |
| -// Handler is the routing table, exported so tests can exercise it without |
| -// binding anything. Every verb is POST-only: the method pattern means a GET |
| -// is turned away by the router, before authorize and before any effect. |
| -func (s *Server) Handler() http.Handler { |
| +// Handler is the shared socket's routing table, exported so tests can |
| +// exercise it without binding anything. Every verb is POST-only: the method |
| +// pattern means a GET is turned away by the router, before authorize and |
| +// before any effect. |
| +func (s *Server) Handler() http.Handler { return s.handler("") } |
| + |
| +// handler is the routing table for one listener: the shared socket's when |
| +// pin is "", or the pinned listener's for host pin. The verbs and their |
| +// bodies are identical; only guard's admission differs. |
| +func (s *Server) handler(pin string) http.Handler { |
| mux := http.NewServeMux() |
| - mux.Handle("POST /v1/ready", s.guard(s.ready)) |
| - mux.Handle("POST /v1/next-due", s.guard(s.nextDue)) |
| - mux.Handle("POST /v1/idle", s.guard(s.idle)) |
| - mux.Handle("POST /v1/heartbeat", s.guard(s.heartbeat)) |
| + mux.Handle("POST /v1/ready", s.guard(pin, s.ready)) |
| + mux.Handle("POST /v1/next-due", s.guard(pin, s.nextDue)) |
| + mux.Handle("POST /v1/idle", s.guard(pin, s.idle)) |
| + mux.Handle("POST /v1/heartbeat", s.guard(pin, s.heartbeat)) |
| return mux |
| } |
| // guard is the single gate. Nothing reaches a verb without passing it, and a |
| // verb never sees the request's claimed host — only the authorized one, so |
| // there is no way for a handler to act on a host it was not entitled to. |
| -func (s *Server) guard(fn func(http.ResponseWriter, *http.Request, string)) http.Handler { |
| +// |
| +// On a pinned listener (pin != "") the pin is checked first: the request |
| +// must carry exactly one Carlos-Sidecar-Host value, equal to pin. Anything |
| +// else (absent, repeated, another host) gets the same refusal as a wrong |
| +// token, before the token is compared. The pin is not a secret (the caller |
| +// dialled the listener's path), so the early return leaks nothing. |
| +func (s *Server) guard(pin string, fn func(http.ResponseWriter, *http.Request, string)) http.Handler { |
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| - host, ok := s.authorize(r) |
| + if pin != "" && !pinnedTo(r, pin) { |
| + refuse(w) |
| + return |
| + } |
| + host, ok := s.authorize(r, pin != "") |
| if !ok { |
| refuse(w) |
| return |
| @@ -122,6 +149,38 @@ func (s *Server) guard(fn func(http.ResponseWriter, *http.Request, string)) http |
| }) |
| } |
| +// pinnedTo reports whether r names exactly host, once. Header.Get would |
| +// take the first of several values; a request that names two hosts is |
| +// refused instead, so which value "counts" is never a question. |
| +func pinnedTo(r *http.Request, host string) bool { |
| + v := r.Header.Values(hostHeader) |
| + return len(v) == 1 && v[0] == host |
| +} |
| + |
| +// isPinned reports whether host is currently served by a pinned listener. |
| +func (s *Server) isPinned(host string) bool { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return s.pinned[host] > 0 |
| +} |
| + |
| +func (s *Server) pin(host string) { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + if s.pinned == nil { |
| + s.pinned = map[string]int{} |
| + } |
| + s.pinned[host]++ |
| +} |
| + |
| +func (s *Server) unpin(host string) { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + if s.pinned[host]--; s.pinned[host] <= 0 { |
| + delete(s.pinned, host) |
| + } |
| +} |
| + |
| // authorize resolves the caller to the one host it may speak about. |
| // |
| // The dangerous shape here is WorkerToken returning "" — unknown host, asleep, |
| @@ -141,12 +200,18 @@ func (s *Server) guard(fn func(http.ResponseWriter, *http.Request, string)) http |
| // The compare itself is constant-time, and both failure paths — no live token, |
| // wrong live token — run the same code, so the structure carries no timing |
| // signal beyond the length leak inherent to any comparison. |
| -func (s *Server) authorize(r *http.Request) (string, bool) { |
| +// |
| +// onPinned says the request arrived on a host's pinned listener (guard has |
| +// already checked the pin). On the shared socket a host that is served by a |
| +// pinned listener has no token here (sidecar isolation D3, Review Focus 4): |
| +// a nested worker's token, leaked or not, is useless on the shared socket, |
| +// and the refusal runs the same compare against an unmatchable value. |
| +func (s *Server) authorize(r *http.Request, onPinned bool) (string, bool) { |
| host := r.Header.Get(hostHeader) |
| presented := bearer(r.Header.Get("Authorization")) |
| want := "" |
| - if host != "" { |
| + if host != "" && (onPinned || !s.isPinned(host)) { |
| want = s.w.WorkerToken(host) |
| } |
| if want == "" { |
| @@ -277,15 +342,44 @@ func displayable(s string) bool { |
| // reach. So we probe first, and refuse rather than steal — and, for the same |
| // reason, this function unlinks nothing it did not create (see below). |
| func (s *Server) Serve(ctx context.Context, path string) error { |
| + ln, err := bind(path, socketMode) |
| + if err != nil { |
| + return err |
| + } |
| + srv := newHTTPServer(s.Handler()) |
| + |
| + errc := make(chan error, 1) |
| + go func() { errc <- srv.Serve(ln) }() |
| + |
| + select { |
| + case err := <-errc: |
| + if errors.Is(err, http.ErrServerClosed) { |
| + return nil |
| + } |
| + return err |
| + case <-ctx.Done(): |
| + // ctx is already cancelled, so Shutdown gets its own deadline. |
| + sctx, cancel := context.WithTimeout(context.Background(), shutdownGrace) |
| + defer cancel() |
| + err := srv.Shutdown(sctx) |
| + <-errc |
| + return err |
| + } |
| +} |
| + |
| +// bind takes path for a new listener with mode, Serve's stale-file rules: |
| +// a live listener is refused, a stale file is replaced. The mode is set |
| +// before bind returns, so nothing is served on the umask's mode. |
| +func bind(path string, mode os.FileMode) (net.Listener, error) { |
| if listening(path) { |
| - return fmt.Errorf("agentsock: %s is already served by a live listener", path) |
| + return nil, fmt.Errorf("agentsock: %s is already served by a live listener", path) |
| } |
| if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { |
| - return fmt.Errorf("agentsock: remove stale socket %s: %w", path, err) |
| + return nil, fmt.Errorf("agentsock: remove stale socket %s: %w", path, err) |
| } |
| ln, err := net.Listen("unix", path) |
| if err != nil { |
| - return fmt.Errorf("agentsock: listen on %s: %w", path, err) |
| + return nil, fmt.Errorf("agentsock: listen on %s: %w", path, err) |
| } |
| // Cleanup is the LISTENER's job and nothing else's. Go unlinks a unix |
| // socket it created when the listener closes, and that unlink happens |
| @@ -304,37 +398,101 @@ func (s *Server) Serve(ctx context.Context, path string) error { |
| ul.SetUnlinkOnClose(true) // the default; pinned because it is load-bearing |
| } |
| - if err := os.Chmod(path, socketMode); err != nil { |
| + // A path-based chmod follows a symlink. The shared socket's directory |
| + // is the agent's own; a pinned listener's run directory is on the |
| + // nosymfollow mount (sidecar isolation D3), so a link planted between |
| + // the bind and this call is not followed there either. |
| + if err := os.Chmod(path, mode); err != nil { |
| ln.Close() // unlinks the socket we just created, and only that one |
| - return fmt.Errorf("agentsock: chmod %s: %w", path, err) |
| + return nil, fmt.Errorf("agentsock: chmod %s: %w", path, err) |
| } |
| + return ln, nil |
| +} |
| - srv := &http.Server{ |
| - Handler: s.Handler(), |
| +func newHTTPServer(h http.Handler) *http.Server { |
| + return &http.Server{ |
| + Handler: h, |
| ReadHeaderTimeout: readHeaderTimeout, |
| ReadTimeout: readTimeout, |
| WriteTimeout: writeTimeout, |
| IdleTimeout: idleTimeout, |
| MaxHeaderBytes: 1 << 14, |
| } |
| +} |
| - errc := make(chan error, 1) |
| - go func() { errc <- srv.Serve(ln) }() |
| +// Pinned is a listener bound for one host (sidecar isolation D3): a nested |
| +// worker's own agent socket, in its run directory. It is served by the same |
| +// Server as the shared socket, with the same verbs and the same token check, |
| +// and admits only requests that name its host (guard). While it is open the |
| +// shared socket refuses that host. |
| +type Pinned struct { |
| + s *Server |
| + path string |
| + host string |
| + srv *http.Server |
| + errc chan error |
| - select { |
| - case err := <-errc: |
| - if errors.Is(err, http.ErrServerClosed) { |
| - return nil |
| - } |
| - return err |
| - case <-ctx.Done(): |
| - // ctx is already cancelled, so Shutdown gets its own deadline. |
| + once sync.Once |
| + closeErr error |
| +} |
| + |
| +// ListenPinned binds path for host, mode 0600, and starts serving it. The |
| +// socket answers by the time ListenPinned returns; Close stops it and |
| +// removes the socket file. A path a live listener serves is refused, as |
| +// Serve refuses one. |
| +func (s *Server) ListenPinned(path, host string) (*Pinned, error) { |
| + if host == "" { |
| + return nil, errors.New("agentsock: a pinned listener needs a host") |
| + } |
| + ln, err := bind(path, pinnedMode) |
| + if err != nil { |
| + return nil, err |
| + } |
| + // Pinned before the first request can arrive: from here on the shared |
| + // socket refuses host. |
| + s.pin(host) |
| + p := &Pinned{s: s, path: path, host: host, srv: newHTTPServer(s.handler(host)), errc: make(chan error, 1)} |
| + go func() { p.errc <- p.srv.Serve(ln) }() |
| + return p, nil |
| +} |
| + |
| +// Path is the socket's path. |
| +func (p *Pinned) Path() string { return p.path } |
| + |
| +// Host is the host the listener answers for. |
| +func (p *Pinned) Host() string { return p.host } |
| + |
| +// Close stops serving, closes the listener (which unlinks the socket file |
| +// it created, and only that one) and returns host to the shared socket. |
| +// Safe to call more than once and concurrently; every call returns the |
| +// first call's result. |
| +func (p *Pinned) Close() error { |
| + p.once.Do(func() { |
| sctx, cancel := context.WithTimeout(context.Background(), shutdownGrace) |
| defer cancel() |
| - err := srv.Shutdown(sctx) |
| - <-errc |
| + p.closeErr = p.srv.Shutdown(sctx) |
| + if err := <-p.errc; !errors.Is(err, http.ErrServerClosed) && p.closeErr == nil { |
| + p.closeErr = err |
| + } |
| + p.s.unpin(p.host) |
| + }) |
| + return p.closeErr |
| +} |
| + |
| +// ServePinned is ListenPinned for host at path, served until ctx is done |
| +// (or the server fails), then closed: it returns with the socket file gone. |
| +func (s *Server) ServePinned(ctx context.Context, path, host string) error { |
| + p, err := s.ListenPinned(path, host) |
| + if err != nil { |
| return err |
| } |
| + select { |
| + case <-ctx.Done(): |
| + case err := <-p.errc: |
| + // The server stopped on its own; hand the error back to Close. |
| + p.errc <- err |
| + } |
| + return p.Close() |
| } |
| // listening reports whether something is currently accepting on path. A stale |
diff --git a/internal/agentsock/agentsock_test.go b/internal/agentsock/agentsock_test.go| index ea46b0fe0..111a7887d 100644 |
| --- a/internal/agentsock/agentsock_test.go |
| +++ b/internal/agentsock/agentsock_test.go |
| @@ -3,6 +3,7 @@ package agentsock |
| import ( |
| "context" |
| "errors" |
| + "io" |
| "net" |
| "net/http" |
| "net/http/httptest" |
| @@ -585,3 +586,292 @@ func TestAgentSockConcurrentCallers(t *testing.T) { |
| t.Fatalf("contacts = %d, want 25 — only a.x.internal's own token may stamp it", contacts) |
| } |
| } |
| + |
| +// --- the host-pinned listener (sidecar isolation D3) --- |
| + |
| +// shortSockDir keeps a unix socket path under the kernel's length limit: |
| +// t.TempDir() nests the test's name, which these long names overrun. |
| +func shortSockDir(t *testing.T) string { |
| + t.Helper() |
| + dir, err := os.MkdirTemp("", "as") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.RemoveAll(dir) }) |
| + return dir |
| +} |
| + |
| +// pinnedCall is one POST over the unix socket at path, with the host |
| +// headers exactly as given (none, one, or several) and the bearer token. |
| +// It returns the status and the body, so a refusal's wire shape can be |
| +// compared with the shared socket's. |
| +func pinnedCall(t *testing.T, path, verb string, hosts []string, tok, body string) (int, string) { |
| + t.Helper() |
| + cl := &http.Client{ |
| + Transport: &http.Transport{ |
| + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&net.Dialer{}).DialContext(ctx, "unix", path) |
| + }, |
| + }, |
| + Timeout: 5 * time.Second, |
| + } |
| + req, err := http.NewRequest("POST", "http://agent"+verb, strings.NewReader(body)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, h := range hosts { |
| + req.Header.Add("Carlos-Sidecar-Host", h) |
| + } |
| + if tok != "" { |
| + req.Header.Set("Authorization", "Bearer "+tok) |
| + } |
| + resp, err := cl.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer resp.Body.Close() |
| + raw, _ := io.ReadAll(resp.Body) |
| + return resp.StatusCode, string(raw) |
| +} |
| + |
| +// servePinned runs ServePinned for host at a fresh path and waits until it |
| +// answers. The returned stop cancels it and reports ServePinned's error. |
| +func servePinned(t *testing.T, s *Server, host string) (path string, stop func() error) { |
| + t.Helper() |
| + path = filepath.Join(shortSockDir(t), "agent.sock") |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + done := make(chan error, 1) |
| + go func() { done <- s.ServePinned(ctx, path, host) }() |
| + waitFor(t, "pinned socket to answer", func() bool { |
| + c, err := net.Dial("unix", path) |
| + if err != nil { |
| + return false |
| + } |
| + c.Close() |
| + return true |
| + }) |
| + var once sync.Once |
| + var err error |
| + stop = func() error { |
| + once.Do(func() { cancel(); err = <-done }) |
| + return err |
| + } |
| + t.Cleanup(func() { stop() }) |
| + return path, stop |
| +} |
| + |
| +// sharedRefusal is the shared socket's 401 body, the one every pinned |
| +// refusal must match. |
| +func sharedRefusal(t *testing.T) string { |
| + t.Helper() |
| + rec := post(New(&fakeWorkers{}).Handler(), "/v1/heartbeat", "ghost.x.internal", "nope", "") |
| + if rec.Code != http.StatusUnauthorized { |
| + t.Fatalf("shared refusal code = %d", rec.Code) |
| + } |
| + return rec.Body.String() |
| +} |
| + |
| +// TestPinnedSocketRefusesOtherHost is Review Focus 4's first half: a token |
| +// leaked from worker B is useless on worker A's listener, whether the |
| +// caller names B (the token's own host) or A (the listener's), and the |
| +// refusal is the shared socket's uniform 401 with no effect at all. |
| +func TestPinnedSocketRefusesOtherHost(t *testing.T) { |
| + const a, b = "a.x.internal", "b.x.internal" |
| + f := &fakeWorkers{tokens: map[string]string{a: "tok-a", b: "tok-b"}} |
| + path, stop := servePinned(t, New(f), a) |
| + want := sharedRefusal(t) |
| + |
| + for _, tc := range []struct { |
| + name, host, tok string |
| + }{ |
| + {"B names itself with its own live token", b, "tok-b"}, |
| + {"B's token naming A", a, "tok-b"}, |
| + {"an unknown host", "ghost.x.internal", "tok-a"}, |
| + {"A's token naming B", b, "tok-a"}, |
| + } { |
| + for _, verb := range allVerbs { |
| + body := "" |
| + switch verb { |
| + case "/v1/ready": |
| + body = `{"version":"v1"}` |
| + case "/v1/next-due": |
| + body = `{"at":1756}` |
| + } |
| + code, got := pinnedCall(t, path, verb, []string{tc.host}, tc.tok, body) |
| + if code != http.StatusUnauthorized || got != want { |
| + t.Fatalf("%s %s: %d %q, want the uniform 401 %q", tc.name, verb, code, got, want) |
| + } |
| + } |
| + } |
| + f.assertNoEffects(t, "pinned listener, another host") |
| + |
| + // The listener's own host with its own token still gets through: the |
| + // pin narrows the audience, it does not replace the token check. |
| + if code, _ := pinnedCall(t, path, "/v1/heartbeat", []string{a}, "tok-a", ""); code != http.StatusNoContent { |
| + t.Fatalf("A on its own listener: %d, want 204", code) |
| + } |
| + if code, _ := pinnedCall(t, path, "/v1/heartbeat", []string{a}, "wrong", ""); code != http.StatusUnauthorized { |
| + t.Fatalf("A with a wrong token on its own listener: %d, want 401", code) |
| + } |
| + contacts, _, _, _ := f.effects() |
| + if contacts != 1 { |
| + t.Fatalf("contacts = %d, want exactly A's one authorized heartbeat", contacts) |
| + } |
| + |
| + if err := stop(); err != nil { |
| + t.Fatalf("ServePinned returned %v on a clean shutdown", err) |
| + } |
| + if _, err := os.Lstat(path); !os.IsNotExist(err) { |
| + t.Fatalf("the pinned socket outlived its listener: %v", err) |
| + } |
| +} |
| + |
| +// TestPinnedSocketRefusesRepeatedHostHeader: the pin is one exact header |
| +// value. Two values, even two copies of the listener's own host, are not |
| +// "this host", and refusing them keeps Header.Get's first-value choice from |
| +// ever being what decides whom a request speaks for. |
| +func TestPinnedSocketRefusesRepeatedHostHeader(t *testing.T) { |
| + const a = "a.x.internal" |
| + f := &fakeWorkers{tokens: map[string]string{a: "tok-a", "b.x.internal": "tok-b"}} |
| + path, _ := servePinned(t, New(f), a) |
| + want := sharedRefusal(t) |
| + for _, hosts := range [][]string{ |
| + {a, a}, |
| + {a, "b.x.internal"}, |
| + {"b.x.internal", a}, |
| + {a + ", " + a}, |
| + {"A.x.internal"}, |
| + } { |
| + if code, got := pinnedCall(t, path, "/v1/heartbeat", hosts, "tok-a", ""); code != http.StatusUnauthorized || got != want { |
| + t.Fatalf("hosts %q: %d %q, want the uniform 401", hosts, code, got) |
| + } |
| + } |
| + f.assertNoEffects(t, "repeated host header") |
| +} |
| + |
| +// TestPinnedSocketRefusesAbsentHostHeader: a caller that names no host is |
| +// refused on a pinned listener too, though the listener knows which host |
| +// it serves. Filling the host in for it would make the header optional, |
| +// and the shared socket's contract (name yourself, then prove it) would |
| +// differ between the two listeners. |
| +func TestPinnedSocketRefusesAbsentHostHeader(t *testing.T) { |
| + const a = "a.x.internal" |
| + f := &fakeWorkers{tokens: map[string]string{a: "tok-a"}} |
| + path, _ := servePinned(t, New(f), a) |
| + want := sharedRefusal(t) |
| + for _, verb := range allVerbs { |
| + if code, got := pinnedCall(t, path, verb, nil, "tok-a", `{"version":"v1","at":1}`); code != http.StatusUnauthorized || got != want { |
| + t.Fatalf("%s with no host header: %d %q, want the uniform 401", verb, code, got) |
| + } |
| + } |
| + if code, got := pinnedCall(t, path, "/v1/heartbeat", []string{""}, "tok-a", ""); code != http.StatusUnauthorized || got != want { |
| + t.Fatalf("an empty host header: %d %q, want the uniform 401", code, got) |
| + } |
| + f.assertNoEffects(t, "absent host header") |
| +} |
| + |
| +// TestPinnedSocketMode0600: the pinned socket is the agent's alone (the |
| +// run directory is group-traversable by enrolled consumers, D3), and the |
| +// mode is set before ListenPinned returns, so a caller holding the |
| +// listener never sees the umask's mode. |
| +func TestPinnedSocketMode0600(t *testing.T) { |
| + path := filepath.Join(shortSockDir(t), "agent.sock") |
| + pl, err := New(&fakeWorkers{}).ListenPinned(path, "a.x.internal") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer pl.Close() |
| + fi, err := os.Lstat(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fi.Mode()&os.ModeSocket == 0 { |
| + t.Fatalf("%s is %v, not a socket", path, fi.Mode()) |
| + } |
| + if perm := fi.Mode().Perm(); perm != 0o600 { |
| + t.Fatalf("pinned socket mode = %o, want 600", perm) |
| + } |
| + if err := pl.Close(); err != nil { |
| + t.Fatalf("Close: %v", err) |
| + } |
| + if err := pl.Close(); err != nil { |
| + t.Fatalf("a second Close must be harmless: %v", err) |
| + } |
| + if _, err := os.Lstat(path); !os.IsNotExist(err) { |
| + t.Fatalf("Close left the socket file behind: %v", err) |
| + } |
| +} |
| + |
| +// TestPinnedSocketRefusesALiveSocket: a path a listener is already serving |
| +// is not stale, on the pinned listener as on the shared one. |
| +func TestPinnedSocketRefusesALiveSocket(t *testing.T) { |
| + path := filepath.Join(shortSockDir(t), "agent.sock") |
| + s := New(&fakeWorkers{}) |
| + first, err := s.ListenPinned(path, "a.x.internal") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer first.Close() |
| + if second, err := s.ListenPinned(path, "a.x.internal"); err == nil { |
| + second.Close() |
| + t.Fatal("ListenPinned took over a path a live listener serves") |
| + } |
| + if _, err := os.Lstat(path); err != nil { |
| + t.Fatalf("the live pinned socket was unlinked: %v", err) |
| + } |
| +} |
| + |
| +// TestWorkerTokenRefusedOnSharedSocketAfterMove is Review Focus 4's second |
| +// half. Once a host is served by its own pinned listener (it moved to |
| +// nested), its token is refused on the shared socket with the uniform 401: |
| +// the shared socket is not a second door for a nested worker's leaked |
| +// token. When the pinned listener closes the host is the shared socket's |
| +// again, as a host that moved back to flat must be. |
| +func TestWorkerTokenRefusedOnSharedSocketAfterMove(t *testing.T) { |
| + const a, b = "a.x.internal", "b.x.internal" |
| + f := &fakeWorkers{tokens: map[string]string{a: "tok-a", b: "tok-b"}} |
| + s := New(f) |
| + shared := s.Handler() |
| + want := sharedRefusal(t) |
| + |
| + if rec := post(shared, "/v1/heartbeat", a, "tok-a", ""); rec.Code != http.StatusNoContent { |
| + t.Fatalf("flat A on the shared socket: %d, want 204", rec.Code) |
| + } |
| + |
| + pl, err := s.ListenPinned(filepath.Join(shortSockDir(t), "agent.sock"), a) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + before, _, _, _ := f.effects() |
| + for _, verb := range allVerbs { |
| + body := "" |
| + switch verb { |
| + case "/v1/ready": |
| + body = `{"version":"v1"}` |
| + case "/v1/next-due": |
| + body = `{"at":1756}` |
| + } |
| + rec := post(shared, verb, a, "tok-a", body) |
| + if rec.Code != http.StatusUnauthorized || rec.Body.String() != want { |
| + t.Fatalf("moved A's token on the shared socket %s: %d %q, want the uniform 401", verb, rec.Code, rec.Body.String()) |
| + } |
| + } |
| + if after, slept, due, ready := f.effects(); after != before || slept != 0 || len(due) != 0 || len(ready) != 0 { |
| + t.Fatalf("a refused shared-socket call had an effect: contacts %d->%d slept=%d due=%v ready=%v", before, after, slept, due, ready) |
| + } |
| + // A flat neighbour is untouched by A's move. |
| + if rec := post(shared, "/v1/heartbeat", b, "tok-b", ""); rec.Code != http.StatusNoContent { |
| + t.Fatalf("flat B on the shared socket while A is pinned: %d, want 204", rec.Code) |
| + } |
| + // And A is still answered where it lives now. |
| + if code, _ := pinnedCall(t, pl.Path(), "/v1/heartbeat", []string{a}, "tok-a", ""); code != http.StatusNoContent { |
| + t.Fatalf("A on its pinned listener: %d, want 204", code) |
| + } |
| + |
| + if err := pl.Close(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rec := post(shared, "/v1/heartbeat", a, "tok-a", ""); rec.Code != http.StatusNoContent { |
| + t.Fatalf("A on the shared socket after its pinned listener closed: %d, want 204", rec.Code) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index ef5e93ddf..05c00277c 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -492,6 +492,13 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| Exec: eb, |
| Unit: &activator.UnitBackend{OptDir: release.OptDir()}, |
| }, reg) // the registry satisfies activator.DueStore directly |
| + // The agent socket's server, made here (it holds no state until a |
| + // listener is bound) so the backend has it before anything wakes: |
| + // a nested worker's own listener (sidecar isolation D3) is bound on |
| + // this same server, which is what lets the shared socket, served |
| + // below, refuse a host its pinned listener serves. |
| + agentSrv := agentsock.New(svc) |
| + eb.AgentSock = agentSrv |
| // doWake's own re-read of the committed row before it spawns |
| // (design §6.8). The registry satisfies RouteLookup with its own |
| // Get, exactly as DueStore above. |
| @@ -662,7 +669,6 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // lostcancel check. |
| agentSockCtx, agentSockCancel := context.WithCancel(context.Background()) |
| defer agentSockCancel() |
| - agentSrv := agentsock.New(svc) |
| go serveAgentSocket(agentSockCtx, cfg.AgentSocket, agentSrv.Serve) |
| log.Printf("agent: agent socket armed at %s", cfg.AgentSocket) |