amadan
Sidecar isolation Task 6 (6/6): the nested worker end to end, the crash loop of Review Focus 1, KillHost is no proof for a nested worker
A worker's crash restart retires no child, so nothing closed the dead unit's listeners and the next unit's prepare could not bind the per-host agent socket: the respawn now closes a dead nested unit's listeners once its exit is proven, before anything is prepared. Tests over the real control server and the fake systemd running the trampoline's stub (this test binary as the tenant program): the lifecycle (wake, ready over the per-host socket, next-due, idle, sleep leaving the unit failed without a restart, timer wake, finish park, crash restart); a worker that computes the flat database path crashes, restarts with backoff, keeps its lease, and the app log names the open failure; the old KillHost test is replaced by one showing RoleGone for the worker role is the proof.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 files changed,
+370
−19
internal/activator/activator.go+13 −0internal/activator/activator_test.go+6 −0internal/activator/unitproc.go+8 −0internal/activator/worker_gen_test.go+0 −19internal/activator/worker_nested_test.go+256 −0internal/activator/worker_stub_test.go+87 −0
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (0 s)
ok amadan.net/carlos/platform.git/launchctx 0.013s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (17 s)
=== gofmt (0 s)
gofmt -l .: clean
=== rest (169 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.671s
ok github.com/carlosframework/platform/cmd/moveprobe 11.827s
ok github.com/carlosframework/platform/cmd/zddprobe 0.726s
ok github.com/carlosframework/platform/internal/activator 59.214s
ok github.com/carlosframework/platform/internal/agentsock 0.051s
ok github.com/carlosframework/platform/internal/alerting 0.007s
ok github.com/carlosframework/platform/internal/analytics 0.046s
ok github.com/carlosframework/platform/internal/appconfig 2.886s
ok github.com/carlosframework/platform/internal/apperr 0.165s
ok github.com/carlosframework/platform/internal/applog 0.305s
ok github.com/carlosframework/platform/internal/awsinv 0.007s
ok github.com/carlosframework/platform/internal/boxchoice 0.006s
ok github.com/carlosframework/platform/internal/boxid 1.250s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.011s
ok github.com/carlosframework/platform/internal/caps 0.019s
ok github.com/carlosframework/platform/internal/certwarm 0.003s
ok github.com/carlosframework/platform/internal/cgroup 0.760s
ok github.com/carlosframework/platform/internal/changelog 0.510s
ok github.com/carlosframework/platform/internal/claimactivation 1.173s
ok github.com/carlosframework/platform/internal/clamd 0.060s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 8.126s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.011s
ok github.com/carlosframework/platform/internal/dbquery 0.008s
ok github.com/carlosframework/platform/internal/dbreads 2.274s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.007s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.016s
ok github.com/carlosframework/platform/internal/domains 2.871s
ok github.com/carlosframework/platform/internal/doorkey 0.011s
ok github.com/carlosframework/platform/internal/economics 0.012s
ok github.com/carlosframework/platform/internal/edge 45.656s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.132s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.016s
ok github.com/carlosframework/platform/internal/geo 0.017s
ok github.com/carlosframework/platform/internal/geoip 0.031s
ok github.com/carlosframework/platform/internal/historyreadback 0.157s
ok github.com/carlosframework/platform/internal/host 140.973s
ok github.com/carlosframework/platform/internal/hostactors 5.725s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.005s
ok github.com/carlosframework/platform/internal/hostbox 2.978s
ok github.com/carlosframework/platform/internal/hostclaim 0.765s
ok github.com/carlosframework/platform/internal/hostenv 0.008s
ok github.com/carlosframework/platform/internal/hostevidence 0.006s
ok github.com/carlosframework/platform/internal/hostexec 5.301s
ok github.com/carlosframework/platform/internal/hostfence 0.033s
ok github.com/carlosframework/platform/internal/hostfiles 10.303s
ok github.com/carlosframework/platform/internal/hostfloor 0.019s
ok github.com/carlosframework/platform/internal/hostguard 1.009s
ok github.com/carlosframework/platform/internal/hostops 1.935s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.002s
ok github.com/carlosframework/platform/internal/hostquarantine 0.002s
ok github.com/carlosframework/platform/internal/hostrelocate 0.605s
ok github.com/carlosframework/platform/internal/hostresolve 0.206s
ok github.com/carlosframework/platform/internal/hostwrite 0.002s
ok github.com/carlosframework/platform/internal/infraview 0.002s
ok github.com/carlosframework/platform/internal/instancectl 6.906s
ok github.com/carlosframework/platform/internal/instanceexec 2.782s
ok github.com/carlosframework/platform/internal/instancefs 0.019s
ok github.com/carlosframework/platform/internal/instancekey 0.018s
ok github.com/carlosframework/platform/internal/instanceupdates 0.020s
ok github.com/carlosframework/platform/internal/layout 1.738s
ok github.com/carlosframework/platform/internal/lbhealth 0.019s
ok github.com/carlosframework/platform/internal/lease 10.266s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.550s
ok github.com/carlosframework/platform/internal/ledger 0.024s
ok github.com/carlosframework/platform/internal/mail 0.207s
ok github.com/carlosframework/platform/internal/maintenance 0.104s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.548s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.105s
ok github.com/carlosframework/platform/internal/metrics 0.005s
ok github.com/carlosframework/platform/internal/moves 22.538s
ok github.com/carlosframework/platform/internal/moves/archive 0.923s
ok github.com/carlosframework/platform/internal/objstore 0.315s
ok github.com/carlosframework/platform/internal/ops 0.243s
ok github.com/carlosframework/platform/internal/park 0.269s
ok github.com/carlosframework/platform/internal/plans 0.006s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
2026/10/24 22:11:53 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 22:11:53 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 22:11:53 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 22:11:53 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 22:11:53 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 22:11:53 provision: an app's records could not be listed: they are skipped
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:11:53 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 22:11:53 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 22:11:53 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 22:11:53 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 22:11:53 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 22:11:53 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 22:11:53 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 22:11:53 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 22:11:53 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 22:11:53 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 22:11:53 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 22:11:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:53 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 22:11:53 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 22:11:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 22:11:53 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 22:11:53 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 22:11:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:54 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 22:11:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:11:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:54 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:54 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 22:11:54 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 22:11:54 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 22:11:54 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 22:11:54 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:54 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:54 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.71s)
plan9_b8_census_test.go:372: record-write census rejected 2 site(s); update the explicit inventory only after reviewing the source:
changed site internal/instanceexec/exit.go:RunExit: source differs from the reviewed digest
changed site internal/ops/litestream.go:FleetDBsWithSkipped: source differs from the reviewed digest
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 22:11:55 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 22:11:55 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 22:11:55 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 22:11:55 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 22:11:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:55 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 22:11:56 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 22:11:56 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 22:11:56 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 22:11:56 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 22:11:56 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 22:11:56 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 22:11:56 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 22:11:56 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 22:11:56 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 22:11:56 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:56 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 22:11:56 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 22:11:56 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 22:11:56 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:56 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 22:11:56 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:56 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 22:11:56 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 22:11:56 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:56 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 22:11:56 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:56 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 22:11:56 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 22:11:56 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 22:11:56 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 22:11:56 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 22:11:56 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 22:11:56 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 22:11:56 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 22:11:56 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 22:11:56 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 22:11:56 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 22:11:56 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 22:11:56 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 22:11:56 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 22:11:56 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 22:11:56 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 22:11:56 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 22:11:56 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:56 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 22:11:56 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 22:11:56 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 22:11:56 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 22:11:56 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 22:11:56 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 22:11:57 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 22:11:57 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 22:11:57 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 22:11:57 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 22:11:57 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 22:11:57 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 22:11:57 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 22:11:57 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 22:11:57 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 22:11:57 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 22:11:57 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 22:11:57 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 22:11:57 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:57 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 22:11:58 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:58 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 22:11:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:58 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 22:11:58 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 22:11:58 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 22:11:58 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 22:11:58 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 22:11:58 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:11:58 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:11:58 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 22:11:58 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:11:58 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:11:58 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 22:11:58 provision: a.shop.example.com: swapped for import sd_hy2fmjiuotqtfpkezd7utv4bh4 (the original route is journaled)
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_hy2fmjiuotqtfpkezd7utv4bh4 committed: its gate is cleared and its journal row dropped
2026/10/24 22:11:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_b75sdianfdcmyojart3qqh4o5i rolled back to its original route
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_b75sdianfdcmyojart3qqh4o5i: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_b75sdianfdcmyojart3qqh4o5i is cleaned up and its journal row retired
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_xxbu5grz7oeing4qfyap4izlee rolled back to its original route
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_xxbu5grz7oeing4qfyap4izlee: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_xxbu5grz7oeing4qfyap4izlee is cleaned up and its journal row retired
2026/10/24 22:11:58 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_tvihidsklh2yetl4lmrohlkor4
2026/10/24 22:11:58 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 22:11:58 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 22:11:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 22:11:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 22:11:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_davtprvjgla7jwdykyx3u6scpu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_wx26agtyatas46avvuqzsukbbu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 22:11:58 provision: a.shop.example.com: import sd_2o45jspzl6srysegrbfudsiu7i is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_52ui7zycjbqwdr2hk6a6tppqnu is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_l63enibj73co7oo2fivy7ufl6m is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 22:11:59 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 22:11:59 provision: b.shop.example.com: import sd_3hxgrt6yx5m46lyu46h7pvbo3a left alone this pass: the import journal could not be read
2026/10/24 22:11:59 provision: c.shop.example.com: import sd_ccvspuyzolntahjgde7s7c7nf4 left alone this pass: the import journal could not be read
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_g27kiq27lbiwjukdekxfvgghbe committed: its gate is cleared and its journal row dropped
2026/10/24 22:11:59 provision: a.shop.example.com: not converging import sd_e25xhmakjemjjdosidx5j2jc4i: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 22:11:59 provision: a.shop.example.com: not converging import sd_rto7atoiljftqxqk5kfmqrsrqi: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 22:11:59 provision: a.shop.example.com: not converging import sd_pw5ghe7w3gfbycndxtg2mxaupi: it is placed but not committed, and the row on this box has no import gate
2026/10/24 22:11:59 provision: a.shop.example.com: not converging import sd_wq3aty5lazkns52lj5nphgemre: it is placed but not committed, and the row on this box has no import gate
2026/10/24 22:11:59 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:59 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 22:11:59 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_p3lecubcfepp5rkyzvfpkndwve committed: its gate is cleared and its journal row dropped
2026/10/24 22:11:59 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:59 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 22:11:59 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 22:11:59 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_ri5ocbrfuahpmqdjwosb5j3qyi: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_ri5ocbrfuahpmqdjwosb5j3qyi rolled back (no box had swapped it)
2026/10/24 22:11:59 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 22:11:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:59 provision: removed instance route acme.go.tito.io (its seed sd_ngvagb66ncvwhazqhik2a6gska was aborted); the next pass rolls it back
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_ngvagb66ncvwhazqhik2a6gska: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_ngvagb66ncvwhazqhik2a6gska rolled back (no box had swapped it)
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_ot53cni6xhwqazbumxjajghsau has a row this pass may not remove; leaving the seed aborted
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_j3if2qfgudrmjhwe77hyn2kjce rolled back (no box had swapped it)
2026/10/24 22:11:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:59 provision: removed instance route acme.go.tito.io (its seed sd_gmiz7gvgrcije3oexizzycx3ym was aborted); the next pass rolls it back
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_gmiz7gvgrcije3oexizzycx3ym has no route on this box: its local data is removed
2026/10/24 22:11:59 provision: acme.go.tito.io: aborted seed sd_gmiz7gvgrcije3oexizzycx3ym rolled back (no box had swapped it)
2026/10/24 22:11:59 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_jefwpwts3e4auatkdrrmrtkusi (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_jefwpwts3e4auatkdrrmrtkusi rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_jefwpwts3e4auatkdrrmrtkusi is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_t4fipunqvjd6l5uhbwkcpwwnju (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_t4fipunqvjd6l5uhbwkcpwwnju rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_t4fipunqvjd6l5uhbwkcpwwnju is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_ro22d5vgkvirozya2jnfuqb24u (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ro22d5vgkvirozya2jnfuqb24u rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ro22d5vgkvirozya2jnfuqb24u: the host's replica is left alone: its record no longer names this seed
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ro22d5vgkvirozya2jnfuqb24u is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_6xcvnnxn6edz7hmxrs3cxyy6ni (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_6xcvnnxn6edz7hmxrs3cxyy6ni rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_6xcvnnxn6edz7hmxrs3cxyy6ni rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_6xcvnnxn6edz7hmxrs3cxyy6ni is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_o4jclw4r5h4m3ssqhevnhsjb2m (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_o4jclw4r5h4m3ssqhevnhsjb2m rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_txz3ng2dwjngg7w4mza4lrzkq4 (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_txz3ng2dwjngg7w4mza4lrzkq4 is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_gadn6olbnufnkj537mukoofbq4 (the original route is journaled)
2026/10/24 22:11:59 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_gadn6olbnufnkj537mukoofbq4 left alone this pass: its record could not be read
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_jytirv6kazprffdnllqbzor7jm (the original route is journaled)
2026/10/24 22:11:59 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_jytirv6kazprffdnllqbzor7jm left alone this pass: its record could not be read
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_izmxaqts4zuugcwhl3vhrhxfey (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_izmxaqts4zuugcwhl3vhrhxfey: close the import gate: constraint failed: injected (1811)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_izmxaqts4zuugcwhl3vhrhxfey rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_izmxaqts4zuugcwhl3vhrhxfey is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_ivysywgihuc5zdl6fnnxqrrdhu (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ivysywgihuc5zdl6fnnxqrrdhu: close the import gate: constraint failed: injected (1811)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ivysywgihuc5zdl6fnnxqrrdhu rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ivysywgihuc5zdl6fnnxqrrdhu is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_hkrmrp6d437svjjnrysxeckwpa (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hkrmrp6d437svjjnrysxeckwpa rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hkrmrp6d437svjjnrysxeckwpa rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hkrmrp6d437svjjnrysxeckwpa is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_xfeujc4shm6k6sjsb3guiuznxe (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_xfeujc4shm6k6sjsb3guiuznxe rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_xfeujc4shm6k6sjsb3guiuznxe is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_qwys25f3sg5y6q7stq6k5keozy (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_qwys25f3sg5y6q7stq6k5keozy: retire its instance key: injected failure at retire-key
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_qwys25f3sg5y6q7stq6k5keozy rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_qwys25f3sg5y6q7stq6k5keozy is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_r64t6efdzxixodi4n7l5mtm3wy (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_r64t6efdzxixodi4n7l5mtm3wy rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_r64t6efdzxixodi4n7l5mtm3wy is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_rhgnc65z7nsgxvoh7nukroxyda (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_rhgnc65z7nsgxvoh7nukroxyda: remove its local data: injected failure at remove-local
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_rhgnc65z7nsgxvoh7nukroxyda rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_rhgnc65z7nsgxvoh7nukroxyda is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_ldp5hppico5gyzndpyyelxyvua (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ldp5hppico5gyzndpyyelxyvua rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_ldp5hppico5gyzndpyyelxyvua is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_3ejilt7j5siv5a2p2kffjltynq (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_3ejilt7j5siv5a2p2kffjltynq: remove its local data: injected failure at remove-local-done
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_3ejilt7j5siv5a2p2kffjltynq rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_3ejilt7j5siv5a2p2kffjltynq is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_zjefkny33xmaesdzd4dqhd5rky (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_zjefkny33xmaesdzd4dqhd5rky rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_zjefkny33xmaesdzd4dqhd5rky is cleaned up and its journal row retired
2026/10/24 22:11:59 provision: a.shop.example.com: swapped for import sd_hbikwtyp25vv5ppwsajigcv7n4 (the original route is journaled)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hbikwtyp25vv5ppwsajigcv7n4: restore the original route: constraint failed: injected (1811)
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hbikwtyp25vv5ppwsajigcv7n4 rolled back to its original route
2026/10/24 22:11:59 provision: a.shop.example.com: import sd_hbikwtyp25vv5ppwsajigcv7n4 is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_6gsh7qlu4k62sg2ragyymwt3gy (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_6gsh7qlu4k62sg2ragyymwt3gy: restore the original route: constraint failed: injected (1811)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_6gsh7qlu4k62sg2ragyymwt3gy rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_6gsh7qlu4k62sg2ragyymwt3gy is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_tfh266efcgpe5dx6kazmj7e4b4 (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_tfh266efcgpe5dx6kazmj7e4b4 is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_wwsos7cx6oypvl2chahdaweqtm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_wwsos7cx6oypvl2chahdaweqtm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_wwsos7cx6oypvl2chahdaweqtm: delete its snapshot: injected failure at delete-snapshot
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_wwsos7cx6oypvl2chahdaweqtm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_hjuj46nun3bn2wnf2wbjjvt5ua (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_hjuj46nun3bn2wnf2wbjjvt5ua rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_hjuj46nun3bn2wnf2wbjjvt5ua is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_n65bqoa6cp6eitx52gzhccn7fm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_n65bqoa6cp6eitx52gzhccn7fm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_n65bqoa6cp6eitx52gzhccn7fm: delete the host's replica: injected failure at delete-replica
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_n65bqoa6cp6eitx52gzhccn7fm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_2znnfbxx2klhiuf5dzjm64gaku (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2znnfbxx2klhiuf5dzjm64gaku rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2znnfbxx2klhiuf5dzjm64gaku is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_a36zzcgnowirv5m7gxcee3vr4i (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_a36zzcgnowirv5m7gxcee3vr4i rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_a36zzcgnowirv5m7gxcee3vr4i: retire its journal row: constraint failed: injected (1811)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_a36zzcgnowirv5m7gxcee3vr4i is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_2kfqa7n6jt5qplromatq3nd5hi (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2kfqa7n6jt5qplromatq3nd5hi rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2kfqa7n6jt5qplromatq3nd5hi: retire its journal row: constraint failed: injected (1811)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2kfqa7n6jt5qplromatq3nd5hi is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_eyoq4kg6fzjphzpthfndxk4l54 (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_eyoq4kg6fzjphzpthfndxk4l54 rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_eyoq4kg6fzjphzpthfndxk4l54 is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: mark import sd_eyoq4kg6fzjphzpthfndxk4l54 rolled back: injected failure at cas
2026/10/24 22:12:00 provision: a.shop.example.com: aborted seed sd_eyoq4kg6fzjphzpthfndxk4l54 rolled back (no box had swapped it)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_jojfpndcjgys5eebsx275pn65q (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_jojfpndcjgys5eebsx275pn65q rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_jojfpndcjgys5eebsx275pn65q is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: aborted seed sd_jojfpndcjgys5eebsx275pn65q rolled back (no box had swapped it)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_k7z7anihzerrxfsnl573ycxzbm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7z7anihzerrxfsnl573ycxzbm rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7z7anihzerrxfsnl573ycxzbm rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7z7anihzerrxfsnl573ycxzbm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7z7anihzerrxfsnl573ycxzbm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_k7hhoj5fcuzg6kl5kpfpqlmrsu (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7hhoj5fcuzg6kl5kpfpqlmrsu rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_k7hhoj5fcuzg6kl5kpfpqlmrsu is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_2eqz3hetbxpjk2roi7lx4djyoa (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2eqz3hetbxpjk2roi7lx4djyoa rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_2eqz3hetbxpjk2roi7lx4djyoa is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_3sptnnrojkqtjowhsz7kgw4egm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_3sptnnrojkqtjowhsz7kgw4egm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_3sptnnrojkqtjowhsz7kgw4egm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_ihpebssgerrp3hnnr7sjh4666y (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_ihpebssgerrp3hnnr7sjh4666y rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_ihpebssgerrp3hnnr7sjh4666y is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_krepgrrga5eczv4viafb7ks5me (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_krepgrrga5eczv4viafb7ks5me rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_krepgrrga5eczv4viafb7ks5me is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_j47s7xfdab6dzl5kpyvlx46bka (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_j47s7xfdab6dzl5kpyvlx46bka rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_j47s7xfdab6dzl5kpyvlx46bka is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_dxglmtnwabhuxcpdndnujfge34 (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_dxglmtnwabhuxcpdndnujfge34 rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_dxglmtnwabhuxcpdndnujfge34 is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_6z6wpaweh2yn57i2vgclcb2iwi (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_6z6wpaweh2yn57i2vgclcb2iwi has no route on this box: its local data is removed and no route is put back
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_6z6wpaweh2yn57i2vgclcb2iwi is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_qrealiznnogduh7dip7iq2zo4u (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_vqy3bvwql4oqizcbifns2i7gbm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_op2c5jpj5exwxz7kjssurk54dm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_op2c5jpj5exwxz7kjssurk54dm cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_wut7we4h7adbeyncfa7oqudv2a (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_wut7we4h7adbeyncfa7oqudv2a is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 22:12:00 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 22:12:00 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_ruz7jktz3qrhnhaq3oclg3367q (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_y77l2rhnll2mwcafzr65ezsefm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_fuy2bxhjmqyqu34yxdmxsg6iva (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_gkpghc4kdv4jwki3muzrfleney (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_dqrw7fozzlsin7gtvjrwtyytba (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_dqrw7fozzlsin7gtvjrwtyytba rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_jkmtfvpn5zrdf2pdz5k73ej3ke (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_jkmtfvpn5zrdf2pdz5k73ej3ke has no route on this box: its local data is removed and no route is put back
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_jkmtfvpn5zrdf2pdz5k73ej3ke is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_cqvarv4s4mc5ncbclbaxv3nudy (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_cqvarv4s4mc5ncbclbaxv3nudy: remove its local data: injected failure at remove-local
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_cqvarv4s4mc5ncbclbaxv3nudy is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_i7ic7pmojzwdvxul653f5wv73m (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_i7ic7pmojzwdvxul653f5wv73m: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_r6mhxv5ra6x5vtypnr3bjuwfaq (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_r6mhxv5ra6x5vtypnr3bjuwfaq rollback deferred: no fence generation: registry busy
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_qi6y46fos7na6cv7s4t34wezke (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_qi6y46fos7na6cv7s4t34wezke rollback deferred: no fence generation: the generation source answered 0
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_xb6qexj7r2cqzspyyt424tcbhy (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_bkpmiszadnydve6zawr2s4cyhe (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_ptgvndpvyc6vawnt53elbprli4 (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 22:12:00 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_h4yztrbqqecqmckuqnujfhwdje (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_h4yztrbqqecqmckuqnujfhwdje rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_h4yztrbqqecqmckuqnujfhwdje is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_thjaxm5krkzhopw3rravlogevm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_thjaxm5krkzhopw3rravlogevm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_thjaxm5krkzhopw3rravlogevm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_fcvxl3glq2jxmvexyu4bktb3rm (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_fcvxl3glq2jxmvexyu4bktb3rm rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_fcvxl3glq2jxmvexyu4bktb3rm is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_g4fbfepf65kh6tbpm7j6pghizy (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_g4fbfepf65kh6tbpm7j6pghizy: retire its instance key: the env file is a symlink
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_g4fbfepf65kh6tbpm7j6pghizy rolled back to its original route
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_g4fbfepf65kh6tbpm7j6pghizy is cleaned up and its journal row retired
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_atunvvu4okmkgk6lxokssylchy (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 22:12:00 provision: a.shop.example.com: swapped for import sd_apiboxzk27isehjg5wrvvz5fci (the original route is journaled)
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_ddtumtrlskva3d5ffyrpgbtadu refused: there is no route at a.shop.example.com to replace
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_vnq3cerlvw7qdov5b2qi22xwxq refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_v7vnttp2sorlxhhkjb67ohyija refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_wwykywbl3nklfy2vz6febjtq5i refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_fllj27gekglji5h424py5uha4e refused: the route belongs to app other
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_zjdqg24rzwgpclnht4wvwni7t4 refused: the route is filed under another account
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_fzxgwrelys7ewhwczewcams7ji refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_z465m3sofl7tcaqkig5sq3xxsy refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_dde5axcuaouy6qacljjwjp5l64 refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 22:12:00 provision: a.shop.example.com: import sd_tbrofhssgb4h35s5vfosobauiy refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_y3axi4rk5icplyjp4pmobjyhdu refused: the route is not an address route this import can take over (socket)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_qgpsxjh2w3k4jckincuvtoajaq refused: the route is not an address route this import can take over (no address)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_p6xihbjrlcfwnn23qatqswa5lq refused: the route is not an address route this import can take over (provisioned)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_gzmk7iziczhl2umr2agrb7f7xy refused: the route is not an address route this import can take over (kind static)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_f2lc2gasjxkrcdpdyi3hiq2e4q refused: the route is not an address route this import can take over (backing)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_uue7prtw3g5skubsruaip2wuja refused: the route is not an address route this import can take over (tunnel)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_e4zcd573nayqt3ilqrcbitiyci refused: the route is not an address route this import can take over (caps)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_xxboyqlqmjcoepq6frd3b5n3dq refused: the route is not an address route this import can take over (platform)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_mxnw63ytibu7lteup676jndoz4 refused: the route is not an address route this import can take over (canary)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_k2tezywcffhcgxmu4ddd6mxphi refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_qhptlff6lehoh32nbtpkzpk2gy refused: the route is not an address route this import can take over (region)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_huneba72r6qycptmw63ualophi refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_hjdwqrxxmjp3fp4rgjac63nb2q refused: the route is not an address route this import can take over (hibernate)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_cz7bfwrlbwfkskggbem3s6bjmy refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_mpo3wbdnzq7hecbctichriewku refused: this box does not run instances nested
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_df4vyzegfgbzpswk4ttuhdf6si refused: this box does not run instances nested
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_2w5mecuai5kdexkfggqcn567by refused: claim control is on for this box
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_7dtfbmukvk4x3kpanyrkxqo7je (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_hjkf6nft6tek346odvkqt42aue (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 22:12:01 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_6p5zhohrflsp6dgbizaleyfgqu: its verify hash is not a sha256
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_3mwsv3omjfzmhksdmu4xna735e: its verify hash is not a sha256
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_hephlt4jdvkyoda3td5ummkkzu: its verify hash is not a sha256
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 22:12:01 provision: A.shop.example.com/..: not swapping for import sd_iqisyzy363xcbtlzcolqd4exeu: its host or seed id is malformed (seeds: not a host)
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_fyfibmr7hbpvmjozqn4lkssddy: its record does not describe a nested, hibernating instance
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_ywoixxzparty3eiwqey4m636pa: its record does not describe a nested, hibernating instance
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_nl5bv4cgdblkk2hbd75poq57vi: its record does not describe a nested, hibernating instance
2026/10/24 22:12:01 provision: a.shop.example.com: not swapping for import sd_pb7h23hyq2aezvzpwhimgavcey: its record does not describe a nested, hibernating instance
2026/10/24 22:12:01 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_hnsqfazzeu737fevledxourfdi (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_wfizsa2pc6zpqs3groahyum3ue (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_gm6qaag77bzfobq7niacuh5tyq (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 22:12:01 provision: dry-run: would consider swapping a.shop.example.com for import sd_vt7qhyvzkiiqdmb3q4zu5ayaxu
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_rubnab2jypzfsqzzgvtlvvnswq rolled back to its original route
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_rubnab2jypzfsqzzgvtlvvnswq: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_rubnab2jypzfsqzzgvtlvvnswq is cleaned up and its journal row retired
2026/10/24 22:12:01 provision: a.shop.example.com: import sd_52d66gopcypcl37v2zlk77yt7q left alone: the import journal holds seed sd_zfqb4c2su4xzcxowuckra65kba at swapped
2026/10/24 22:12:01 provision: a.shop.example.com: not minting import sd_dx6vexfutrs6ry2l4tgrmldru4: it is swapped but not committed, and only its swap makes its row
2026/10/24 22:12:01 provision: a.shop.example.com: not minting import sd_gqersbb7cznzbgzjp6jwswvxdi: it is placed but not committed, and only its swap makes its row
2026/10/24 22:12:01 provision: a.shop.example.com: swapped for import sd_jrfy7ki2h76zcep7a5vfixswba (the original route is journaled)
2026/10/24 22:12:01 provision: a.shop.example.com: not minting import sd_jrfy7ki2h76zcep7a5vfixswba: it is swapped but not committed, and only its swap makes its row
2026/10/24 22:12:01 provision: a.shop.example.com: not minting import sd_jrfy7ki2h76zcep7a5vfixswba: it is placed but not committed, and only its swap makes its row
2026/10/24 22:12:01 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 22:12:01 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 22:12:01 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 22:12:01 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 22:12:01 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 22:12:01 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 22:12:01 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 22:12:01 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 22:12:01 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 22:12:01 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 22:12:01 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 22:12:01 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 22:12:01 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 22:12:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 22:12:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 22:12:01 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 22:12:01 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 22:12:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 22:12:01 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 22:12:01 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 22:12:01 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 22:12:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:12:01 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 22:12:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:12:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:12:01 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 22:12:01 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 22:12:01 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 22:12:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 22:12:02 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 22:12:02 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 22:12:02 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 22:12:02 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 22:12:02 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 22:12:02 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 22:12:02 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 22:12:02 provision: record view: change index unusable (get refused): walking
2026/10/24 22:12:02 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 22:12:02 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 22:12:02 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 22:12:02 provision: an app's records could not be listed: they are skipped
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 22:12:02 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 22:12:02 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 22:12:02 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 22:12:02 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 22:12:02 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 22:12:02 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 22:12:02 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 22:12:02 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 22:12:02 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 22:12:02 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 22:12:02 provision: removed retired instance route one.example.com (record gone)
2026/10/24 22:12:02 provision: removed retired instance route one.example.com (record gone)
2026/10/24 22:12:02 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 22:12:02 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 22:12:02 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (14.58s)
plan14_root_reads_test.go:600: root read census: 66 discrepancies
changed root read internal/activator/activator.go:Service.doWake; review class, task, sites and digest
changed root read internal/activator/activator.go:Service.respawnWorkerLocked; review class, task, sites and digest
changed root read internal/activator/activator.go:Service.stop; review class, task, sites and digest
changed root read internal/activator/backend_exec.go:ExecBackend.LegacyRunGone; review class, task, sites and digest
changed root read internal/activator/backend_exec.go:ExecBackend.StartWorkerStaged; review class, task, sites and digest
changed root read internal/activator/backend_exec.go:ExecBackend.spawnUnitFrom; review class, task, sites and digest
changed root read internal/activator/backend_unit.go:SplitBackend.LegacyRunGone; review class, task, sites and digest
changed root read internal/activator/containment.go:Service.legacyRunGone; review class, task, sites and digest
changed root read internal/activator/layout.go:Classify; review class, task, sites and digest
changed root read internal/activator/maintenance.go:Service.cleanupFailedWake; review class, task, sites and digest
changed root read internal/activator/opset.go:Service.reapUnprovenWake; review class, task, sites and digest
changed root read internal/activator/prepare.go:ExecBackend.prepareAt; review class, task, sites and digest
changed root read internal/activator/prepare.go:ExecBackend.prepareKeyedAt; review class, task, sites and digest
changed root read internal/activator/unitproc.go:ExecBackend.observe; review class, task, sites and digest
changed root read internal/activator/workerswap.go:Service.ReplaceWorker; review class, task, sites and digest
changed root read internal/host/agent.go:runAgent; review class, task, sites and digest
changed root read internal/instancectl/server.go:parseRequest; review class, task, sites and digest
changed root read internal/instancectl/server.go:type Server; review class, task, sites and digest
changed root read internal/ops/hibernate_verify.go:HibernatingHosts; review class, task, sites and digest
changed root read internal/ops/litestream.go:FleetDBsWithSkipped; review class, task, sites and digest
changed root read internal/ops/ops.go:runLitestream; review class, task, sites and digest
changed root source envelope internal/activator
changed root source envelope internal/agentsock
changed root source envelope internal/host
changed root source envelope internal/host/units
changed root source envelope internal/instancectl
changed root source envelope internal/instanceexec
changed root source envelope internal/ops
stale root read internal/activator/control.go:systemctlUnitState
unclassified root read internal/activator/activator.go:Service.agentSocketFor whole:Route@148
unclassified root read internal/activator/backend_exec.go:ExecBackend.RoleGone registry:Host@192,registry:Host@228,registry:Host@271,registry:Host@562
unclassified root read internal/activator/backend_exec.go:ExecBackend.startWorkerUnit registry:Host@1137,registry:Host@1299,registry:Host@1647,registry:Host@1775,registry:Host@2094,registry:Host@2193,registry:Host@626,registry:Host@698,whole:Route@1341,whole:Route@2291,whole:Route@328,whole:Route@419
unclassified root read internal/activator/backend_unit.go:SplitBackend.RoleGone file:Exec@263,registry:Backing@107,registry:Host@217,registry:Host@323,whole:Route@377
unclassified root read internal/activator/backend_unit.go:SplitBackend.perHostAgentSocket file:Exec@154,registry:Backing@82,whole:Route@217
unclassified root read internal/activator/containment.go:Service.roleGone whole:Route@581
unclassified root read internal/activator/control.go:systemctlUnitFacts file:CommandContext@105
unclassified root read internal/activator/layout.go:WorkerManaged registry:Backing@71,registry:Tunnel@91
unclassified root read internal/activator/opset.go:Service.stopUnitProven registry:Host@309,whole:Route@101,whole:Route@177
unclassified root read internal/activator/prepare.go:ExecBackend.agentSocketFor whole:Route@102
unclassified root read internal/activator/prepare.go:ExecBackend.perHostAgentSocket whole:Route@141,whole:Route@96
unclassified root read internal/activator/prepare.go:ExecBackend.prepareWorker whole:Route@210
unclassified root read internal/activator/unitproc.go:ExecBackend.newWorkerUnitProc registry:Host@244,registry:Host@286,whole:Route@167,whole:Route@171
unclassified root read internal/host/units/carlos-instance-worker@.service:external file:external@0
unclassified root read internal/ops/litestream.go:ActivatorReplicated registry:Backing@59,registry:DBPath@97,registry:Hibernate@117
unclassified root read internal/ops/litestream.go:ActivatorReplicatedPaths registry:DBPath@257,whole:Route@178,whole:Route@212
changed legacy declaration internal/activator/activator.go:Service.doWake
changed legacy declaration internal/activator/activator.go:Service.stop
changed legacy declaration internal/activator/backend_exec.go:ExecBackend.LegacyRunGone
changed legacy declaration internal/activator/backend_exec.go:ExecBackend.StartWorkerStaged
changed legacy declaration internal/activator/backend_exec.go:ExecBackend.spawnUnitFrom
changed legacy declaration internal/activator/backend_unit.go:SplitBackend.LegacyRunGone
changed legacy declaration internal/activator/containment.go:Service.legacyRunGone
changed legacy declaration internal/activator/layout.go:Classify
changed legacy declaration internal/activator/maintenance.go:Service.cleanupFailedWake
changed legacy declaration internal/activator/opset.go:Service.reapUnprovenWake
changed legacy declaration internal/activator/prepare.go:ExecBackend.prepareAt
changed legacy declaration internal/activator/prepare.go:ExecBackend.prepareKeyedAt
changed legacy declaration internal/activator/unitproc.go:ExecBackend.observe
changed legacy declaration internal/instancectl/server.go:parseRequest
changed legacy declaration internal/instancectl/server.go:type Server
changed legacy declaration internal/ops/hibernate_verify.go:HibernatingHosts
changed legacy declaration internal/ops/litestream.go:FleetDBsWithSkipped
changed legacy declaration internal/ops/ops.go:runLitestream
changed reclassified declaration internal/activator/activator.go:Service.respawnWorkerLocked
changed reclassified declaration internal/activator/workerswap.go:Service.ReplaceWorker
legacy declaration needs explicit reclassification or removal internal/activator/control.go:systemctlUnitState
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 23.151s
ok github.com/carlosframework/platform/internal/publicrelease 0.009s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.608s
ok github.com/carlosframework/platform/internal/relay 7.237s
ok github.com/carlosframework/platform/internal/release 25.717s
ok github.com/carlosframework/platform/internal/rollout 3.779s
ok github.com/carlosframework/platform/internal/rolloutwords 0.006s
ok github.com/carlosframework/platform/internal/rootaws 0.008s
ok github.com/carlosframework/platform/internal/rootparticipation 0.060s
ok github.com/carlosframework/platform/internal/rootstate 0.026s
ok github.com/carlosframework/platform/internal/runs 0.618s
ok github.com/carlosframework/platform/internal/s3compat 0.008s
ok github.com/carlosframework/platform/internal/safefile 0.005s
ok github.com/carlosframework/platform/internal/schedule 2.667s
ok github.com/carlosframework/platform/internal/seeds 0.011s
ok github.com/carlosframework/platform/internal/sim/engines 2.613s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.355s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.130s
ok github.com/carlosframework/platform/internal/status 0.083s
ok github.com/carlosframework/platform/internal/steering 0.007s
ok github.com/carlosframework/platform/internal/stopevidence 0.315s
ok github.com/carlosframework/platform/internal/svcrole 0.004s
ok github.com/carlosframework/platform/internal/telemetry 0.006s
ok github.com/carlosframework/platform/internal/termfence 0.005s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.003s
ok github.com/carlosframework/platform/internal/vet 1.676s
FAIL
=== internal_console@0 (86 s)
ok github.com/carlosframework/platform/internal/console 84.278s
=== internal_console@1 (43 s)
ok github.com/carlosframework/platform/internal/console 41.755s
=== internal_console@2 (41 s)
ok github.com/carlosframework/platform/internal/console 39.374s
=== internal_console@3 (42 s)
ok github.com/carlosframework/platform/internal/console 40.391s
=== internal_console@4 (42 s)
ok github.com/carlosframework/platform/internal/console 40.586s
=== internal_console@5 (42 s)
ok github.com/carlosframework/platform/internal/console 40.184s
=== internal_console@6 (37 s)
ok github.com/carlosframework/platform/internal/console 35.672s
=== internal_console@7 (46 s)
ok github.com/carlosframework/platform/internal/console 44.120s
=== root@0 (109 s)
ok github.com/carlosframework/platform 107.117s
=== root@1 (84 s)
ok github.com/carlosframework/platform 82.357s
=== root@2 (93 s)
using http://127.0.0.1:41707 as a@keymail.dev
using http://127.0.0.1:36519 as vicky@keymail.dev
titogo (staging):
KEY VALUE
PORT 8080
--- FAIL: TestRunBuiltCopyIsApplied (0.12s)
approved_run_copy_test.go:139: RUN-C13: internal/instanceexec/exec.go does not contain the approved text "instance-exec: role <role> is not run, seed, sidecar or cmd"
approved_run_copy_test.go:139: RUN-C14: internal/instanceexec/exit.go does not contain the approved text "instance-exit: role <role> is not run, seed, sidecar or cmd"
2026/10/24 22:12:06 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:06 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A42857": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:06 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:06 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A42857": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:06 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:06 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:06 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A34563": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:06 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:08 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:08 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A44511": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:08 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:08 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:08 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A43575": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:08 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:10 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:10 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A34919": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:10 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:12 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A36531": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:12 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
using http://127.0.0.1:34727 as paul@keymail.dev
account: ops (bab)
2026/10/24 22:12:15 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A34727": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:15 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
set 1 env var(s) for titogo (default)
using http://127.0.0.1:34727 as paul@keymail.dev
account: ops (bab)
set 1 secret(s) for titogo (default)
account: ops (bab)
account: ops (bab)
using http://127.0.0.1:34727 as paul@keymail.dev
account: ops (bab)
config sync requested for titogo the files land within seconds; programs carlos owns restart themselves, and anything else keeps its old values until you restart it
2026/10/24 22:12:15 console: delivery to mallory@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=mallory%40keymail.dev&from=console%40127.0.0.1%3A34727": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:15 console: could not notify mallory@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:12:17 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 22:12:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:12:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:12:18 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 22:12:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:12:18 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:18 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A37967": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:18 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 22:12:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:12:18 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 22:12:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:12:19 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:12:19 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 22:12:19 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:12:20 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:12:20 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A38223": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:12:20 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:01 console: db read dbr_cgkxsta3boixc3srv5bfhnvu4e: relayed 1 chunks (101 bytes) in 41.032s, ended done
2026/10/24 22:13:01 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:13:01 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 22:13:01 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:13:02 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:13:02 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 22:13:02 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:13:02 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:13:02 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A44429": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:02 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:03 console: db read dbr_srercz52nwy3yjjpc6u5pnb2fq: relayed 1 chunks (1070 bytes) in 1.001s, ended done
2026/10/24 22:13:04 console: db read dbr_drx2hfwggpdu3eqzby62hw7tiq: relayed 1 chunks (805 bytes) in 1.001s, ended done
2026/10/24 22:13:04 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:13:04 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 22:13:04 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:13:05 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:13:05 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 22:13:05 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:13:05 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:13:05 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A37103": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:05 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:06 console: db read dbr_tpg6zce3oxambsgpxohm3ealfu: relayed 2 chunks (8386 bytes) in 1.045s, ended client gone
using http://127.0.0.1:38147 as paul@keymail.dev
account: ops (bab)
attached tickets.example.com -> default.go.tito.io
DNS: pointing
using http://127.0.0.1:38147 as paul@keymail.dev
account: ops (bab)
attached carloku.example.com -> default.go.tito.io
2026/10/24 22:13:07 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:13:07 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A40009": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:07 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:07 console: delivery to rival@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=rival%40keymail.dev&from=console%40127.0.0.1%3A40009": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:07 console: could not notify rival@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:15 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A38003": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:15 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 22:13:16 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A36029": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 22:13:16 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 22:13:17 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 22:13:17 domains: minted platform domain blog.mdbbndjqx.oncarlos.test (upstream blog.kass.test)
2026/10/24 22:13:17 domains: minted platform domain blog.gdpkwhwrw.oncarlos.test (upstream blog.woodstar.test)
2026/10/24 22:13:17 domains: minted platform domain oldsite.bab.oncarlos.test (upstream old.test)
2026/10/24 22:13:17 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 22:13:17 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 22:13:17 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
FAIL
FAIL github.com/carlosframework/platform 91.124s
FAIL
FAILED: rest root@2
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 48a6e81ff..a1589d086 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -3557,6 +3557,15 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string, ont |
| m = &genMember{gen: Gen{N: nextGen(), Worker: true}, role: roleSpawning, cancel: cancelSpawn, done: make(chan struct{})} |
| a.addMemberLocked(e, m) |
| } |
| + // A crash restart retires nothing (stopOld nil: the child is dead), so |
| + // nothing calls the dead child's Stop, which is what closes a nested |
| + // unit's listeners (sidecar isolation D3): its per-host agent listener |
| + // would still serve the path the new unit's prepare must bind. Closed |
| + // below, once its exit is proven, before anything is prepared. |
| + var dead Proc |
| + if stopOld == nil { |
| + dead = e.proc |
| + } |
| go func() { |
| // respawnDone closes LAST, once this goroutine is finished with every |
| // child it created — including one it had to SHED. That ordering is |
| @@ -3602,6 +3611,10 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string, ont |
| retireTook = time.Since(retireStart) |
| } |
| + if c, ok := dead.(listenerCloser); ok && dead.Exited() { |
| + c.closeListeners() |
| + } |
| + |
| // The retired child must be proven gone, its host cgroup included, |
| // before anything new execs beside its descendants (zero-downtime |
| // spec §7.4, §7.9). Unproven: nothing spawns; the entry keeps its |
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go| index 54f468b04..5b9e3c0a3 100644 |
| --- a/internal/activator/activator_test.go |
| +++ b/internal/activator/activator_test.go |
| @@ -6113,6 +6113,12 @@ func TestMain(m *testing.M) { |
| fakeAppMain() |
| return |
| } |
| + // The nested worker tests' tenant program (worker_stub_test.go), exec'd |
| + // by the fake systemd from a staged shell script that sets the marker. |
| + if os.Getenv("ACTIVATOR_FAKE_WORKER") == "1" { |
| + fakeWorkerMain() |
| + return |
| + } |
| // A worker's wake waits out its settle window unless the child reports |
| // ready, and no fake child does. Shrink it once for the whole package — |
| // the tests that are ABOUT the window set their own value and restore it |
diff --git a/internal/activator/unitproc.go b/internal/activator/unitproc.go| index 581520afc..edb675595 100644 |
| --- a/internal/activator/unitproc.go |
| +++ b/internal/activator/unitproc.go |
| @@ -238,6 +238,14 @@ func (p *unitProc) finaliseGen() { |
| } |
| } |
| +// listenerCloser is a Proc holding listeners the agent bound for it, closed |
| +// once its exit is proven: a nested unit's control, log and agent listeners. |
| +// A worker's crash restart closes a dead unit's before the next unit's |
| +// prepare binds the same paths (respawnWorkerLocked). |
| +type listenerCloser interface { |
| + closeListeners() |
| +} |
| + |
| func (p *unitProc) closeListeners() { |
| if p.closeOnce.Swap(true) { |
| return |
diff --git a/internal/activator/worker_gen_test.go b/internal/activator/worker_gen_test.go| index 7e3c02e58..11886417e 100644 |
| --- a/internal/activator/worker_gen_test.go |
| +++ b/internal/activator/worker_gen_test.go |
| @@ -3,12 +3,10 @@ package activator |
| import ( |
| "context" |
| "errors" |
| - "path/filepath" |
| "sync" |
| "testing" |
| "time" |
| - "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -47,23 +45,6 @@ func TestWorkerGenerations(t *testing.T) { |
| } |
| } |
| -// StartWorker spawns every worker into the flat host cgroup, whatever the |
| -// row's layout, so KillHost proves that cgroup for a nested-looking row too. |
| -// Here the cgroup is a fixture, not cgroup v2, so the proof is refused. |
| -// Answering nil for a nested layout skipped the proof for a worker that |
| -// runs in carlos-exec@<host> all the same. |
| -func TestKillHostProvesANestedLayoutWorkersHostCgroup(t *testing.T) { |
| - host := "nw.example.com" |
| - b, _ := delegatedBackend(t, host) |
| - rd := t.TempDir() |
| - b.RunDirOverride = rd |
| - rt := registry.Route{Host: host, Kind: "sidecar", DBPath: provision.NestedDBPath(b.DataDir, host), |
| - Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| - if err := b.KillHost(context.Background(), rt); err == nil { |
| - t.Fatal("KillHost answered a nested-layout worker's host cgroup as proven without proving it") |
| - } |
| -} |
| - |
| // hostKillBackend is fakeBackend plus KillHost, failing while bad. |
| type hostKillBackend struct { |
| *fakeBackend |
diff --git a/internal/activator/worker_nested_test.go b/internal/activator/worker_nested_test.go| index eb6423db0..b90c863bc 100644 |
| --- a/internal/activator/worker_nested_test.go |
| +++ b/internal/activator/worker_nested_test.go |
| @@ -22,6 +22,7 @@ import ( |
| "time" |
| "github.com/carlosframework/platform/internal/agentsock" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/instancectl" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -1053,3 +1054,258 @@ func TestOverlapReplaceWorkerFallsBackOnlyAfterRoleGone(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// ---- the end-to-end tests over the real control server ------------------ |
| + |
| +// stubScript is a staged build that execs this test binary as the worker |
| +// stub (fakeWorkerMain), with the manifest's args. |
| +func stubScript(t *testing.T, ver string) string { |
| + t.Helper() |
| + self, err := os.Executable() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return "#!/bin/sh\n# build " + ver + "\nACTIVATOR_FAKE_WORKER=1 exec '" + self + "' \"$@\"\n" |
| +} |
| + |
| +// lifecycleService is a real Service over the harness's ExecBackend with the |
| +// lease and restore faked (leaselessExec), its agent socket answered by the |
| +// Service itself, as the agent wires it, and the row in a real registry. |
| +func lifecycleService(t *testing.T, h *workerHarness, idle time.Duration) (*Service, *fakeClock, *registry.Registry, *fakeBackend) { |
| + t.Helper() |
| + fb := &fakeBackend{} |
| + reg, err := registry.Open(filepath.Join(h.root, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { reg.Close() }) |
| + a := New(Config{Bucket: "test-bucket", Idle: idle}, &leaselessExec{ExecBackend: h.b, fb: fb}, reg) |
| + clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| + a.now = clk.now |
| + a.Logs = h.sink |
| + h.b.AgentSock = agentsock.New(a) |
| + if err := reg.Upsert(h.rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + return a, clk, reg, fb |
| +} |
| + |
| +func (a *Service) workerReady(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + e := a.entries[host] |
| + return e != nil && e.state == actAwake && e.readyNoted |
| +} |
| + |
| +func (a *Service) workerProcExited(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + e := a.entries[host] |
| + return e != nil && e.proc != nil && e.proc.Exited() |
| +} |
| + |
| +// hasLine reports whether any of lines contains sub. |
| +func hasLine(lines []string, sub string) bool { |
| + for _, l := range lines { |
| + if strings.Contains(l, sub) { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// TestNestedWorkerLifecycle: a nested worker, end to end over the real |
| +// control server and a fake systemd running the trampoline's stub: wake, |
| +// ready over its own agent socket, next-due, idle, sleep (the requested |
| +// stop leaves the unit failed and is not read as a crash), the timer wake, |
| +// a crash restart (D6), and a finish that parks. |
| +func TestNestedWorkerLifecycle(t *testing.T) { |
| + h := newWorkerHarness(t, nil) |
| + h.stage("v1", stubScript(t, "v1")) |
| + a, clk, reg, fb := lifecycleService(t, h, 10*time.Second) |
| + host, unit := h.host, h.workerUnit() |
| + due := clk.now().Add(time.Minute).Unix() |
| + h.writeTenantEnv("STUB_MODE=serve\nSTUB_NEXT_DUE=" + strconv.FormatInt(due, 10) + "\n") |
| + |
| + // 1. Wake: the unit starts on the staged build and the stub reports |
| + // ready over the per-host agent socket, with its database where it was |
| + // told (D4: CARLOS_SIDECAR_DB, the nested path). |
| + if err := a.WakeWorker(h.rt); err != nil { |
| + t.Fatalf("WakeWorker: %v\nsystemd:\n%s", err, h.sd.joined()) |
| + } |
| + waitFor(t, "the worker to be ready", func() bool { return a.workerReady(host) }) |
| + if _, err := os.Stat(h.rt.DBPath); err != nil { |
| + t.Fatalf("the worker's database at CARLOS_SIDECAR_DB: %v", err) |
| + } |
| + if svc := h.sd.unitOf(unit); svc == nil || svc.active != "active" { |
| + t.Fatalf("worker unit %+v, want active", svc) |
| + } |
| + if h.sd.startCount() != 1 || strings.Contains(h.sd.joined(), "carlos-instance@") { |
| + t.Fatalf("systemd calls:\n%s", h.sd.joined()) |
| + } |
| + |
| + // 2. Next-due: recorded while the child is alive to say so. |
| + waitFor(t, "the next-due record", func() bool { got, err := reg.Get(host); return err == nil && got.NextDue == due }) |
| + |
| + // 3. Idle, then sleep: the requested stop goes through the fenced /stop; |
| + // the trampoline's 143 leaves the unit failed, and that is NOT a crash |
| + // to restart (D6): the entry parks and the lease goes back. |
| + clk.advance(11 * time.Second) |
| + a.sweepOnce() |
| + waitFor(t, "the worker to sleep", func() bool { return !a.Owns(host) && fb.releaseCount() == 1 }) |
| + if svc := h.sd.unitOf(unit); svc == nil || svc.active != "failed" || svc.result != "exit-code" { |
| + t.Fatalf("worker unit after a requested stop %+v, want failed/exit-code (the trampoline's 143)", svc) |
| + } |
| + if h.sd.startCount() != 1 { |
| + t.Fatalf("starts = %d after a requested stop: the stop was restarted as a crash", h.sd.startCount()) |
| + } |
| + if got, _ := reg.Get(host); got.NextDue != due { |
| + t.Fatalf("next_due = %d after the sleep, want %d kept", got.NextDue, due) |
| + } |
| + |
| + // 4. The timer wake: the due sweep wakes it, and the failed unit needs |
| + // no reset before its next start. This incarnation runs in finish mode: |
| + // ready, then exit 0. |
| + h.writeTenantEnv("STUB_MODE=finish\n") |
| + clk.advance(time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the timer wake", func() bool { return h.sd.startCount() == 2 }) |
| + waitFor(t, "the one-shot to finish", func() bool { return a.workerProcExited(host) }) |
| + waitFor(t, "the due time consumed", func() bool { got, err := reg.Get(host); return err == nil && got.NextDue == 0 }) |
| + if !hasLine(h.sink.eventLines(), "info: instance exited (exit 0)") { |
| + t.Fatalf("events %v, want the clean exit line", h.sink.eventLines()) |
| + } |
| + if svc := h.sd.unitOf(unit); svc.active != "inactive" || svc.result != "success" { |
| + t.Fatalf("worker unit after exit 0 %+v", svc) |
| + } |
| + // 5. Finish park: exit 0 is the only success (D6), and a finished |
| + // one-shot parks, releasing its lease; nothing restarts it. |
| + a.sweepOnce() |
| + waitFor(t, "the finished worker to park", func() bool { return !a.Owns(host) && fb.releaseCount() == 2 }) |
| + if h.sd.startCount() != 2 { |
| + t.Fatalf("starts = %d: a clean finish was restarted", h.sd.startCount()) |
| + } |
| + |
| + // 6. Crash restart: woken again, the serving stub is killed from |
| + // outside (no stop was asked for); the unit reads failed/exit-code with |
| + // our invocation, so Crashed is true and the sweep restarts it in place, |
| + // keeping the lease. |
| + h.writeTenantEnv("STUB_MODE=serve\n") |
| + if err := a.WakeWorker(h.rt); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + waitFor(t, "the worker to be ready again", func() bool { return a.workerReady(host) }) |
| + if h.sd.startCount() != 3 { |
| + t.Fatalf("starts = %d", h.sd.startCount()) |
| + } |
| + releasesBefore := fb.releaseCount() |
| + if err := h.sd.unitOf(unit).cmd.Process.Kill(); err != nil { |
| + t.Fatal(err) |
| + } |
| + waitFor(t, "the crash to be observed", func() bool { return a.workerProcExited(host) }) |
| + if !hasLine(h.sink.eventLines(), "warn: instance exited: exit 137 (exit-code)") { |
| + t.Fatalf("events %v, want the crash's exit line", h.sink.eventLines()) |
| + } |
| + a.sweepOnce() |
| + waitFor(t, "the crash restart", func() bool { return h.sd.startCount() == 4 && a.workerReady(host) }) |
| + if fb.releaseCount() != releasesBefore || !a.Owns(host) { |
| + t.Fatal("a crash restart released the lease or lost the entry") |
| + } |
| + if !hasLine(h.sink.eventLines(), "info: sidecar restarted") { |
| + t.Fatalf("events %v, want the restart event", h.sink.eventLines()) |
| + } |
| + a.Bounce(host) |
| + waitFor(t, "the final stop", func() bool { return !a.Owns(host) }) |
| +} |
| + |
| +// Review Focus 1: a worker that computes /data/<host>.db itself starts |
| +// nested and crashes at the open. It reads as a crash (D6), restarts with |
| +// the doubling backoff, keeps its lease, and `carlos sidecars logs` (the |
| +// app log sink) names the open failure. |
| +func TestNestedWorkerComputedFlatPathCrashLoops(t *testing.T) { |
| + old := sidecarSettle |
| + sidecarSettle = 0 // the wake does not wait on a child that is about to crash |
| + t.Cleanup(func() { sidecarSettle = old }) |
| + h := newWorkerHarness(t, nil) |
| + h.stage("v1", stubScript(t, "v1")) |
| + a, clk, _, fb := lifecycleService(t, h, time.Hour) |
| + host, unit := h.host, h.workerUnit() |
| + // /data, as the namespace shows it: nothing can be opened beneath it. |
| + // A regular file stands in: open(<file>/<host>.db) fails for everyone. |
| + dataRoot := filepath.Join(h.root, "data-ro") |
| + if err := os.WriteFile(dataRoot, nil, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + h.writeTenantEnv("STUB_MODE=flat-db\nSTUB_DATA_ROOT=" + dataRoot + "\n") |
| + if err := a.WakeWorker(h.rt); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + flat := filepath.Join(dataRoot, host+".db") |
| + crashed := func(n int) func() bool { |
| + return func() bool { |
| + return h.sd.startCount() == n && a.workerProcExited(host) |
| + } |
| + } |
| + waitFor(t, "the first crash", crashed(1)) |
| + if !hasLine(h.sink.eventLines(), "warn: instance exited: exit 1 (exit-code)") { |
| + t.Fatalf("events %v, want the crash read as exit 1", h.sink.eventLines()) |
| + } |
| + waitFor(t, "the open failure in the app log", func() bool { return hasLine(h.sink.appLines(), "open "+flat+": ") }) |
| + if svc := h.sd.unitOf(unit); svc.active != "failed" || svc.result != "exit-code" { |
| + t.Fatalf("unit %+v", svc) |
| + } |
| + // The sweep reads the crash and restarts at once, arming the backoff. |
| + a.sweepOnce() |
| + waitFor(t, "the first restart to crash", crashed(2)) |
| + // Same clock: inside the backoff, no restart. |
| + a.sweepOnce() |
| + time.Sleep(100 * time.Millisecond) |
| + if h.sd.startCount() != 2 { |
| + t.Fatalf("starts = %d: a crash loop restarted inside its backoff", h.sd.startCount()) |
| + } |
| + clk.advance(sidecarRestartBase + time.Second) |
| + a.sweepOnce() |
| + waitFor(t, "the paced restart", crashed(3)) |
| + if !a.Owns(host) || fb.releaseCount() != 0 { |
| + t.Fatalf("owns %v, releases %d: a crash loop must keep its entry and lease", a.Owns(host), fb.releaseCount()) |
| + } |
| + if _, err := os.Stat(h.rt.DBPath); err == nil { |
| + t.Fatal("the worker never opened its real database, yet it exists") |
| + } |
| + a.Bounce(host) |
| + waitFor(t, "the final stop", func() bool { return !a.Owns(host) }) |
| +} |
| + |
| +// Replaces TestKillHostProvesANestedLayoutWorkersHostCgroup (D19): a nested |
| +// worker runs in its unit's cgroup under its host slice, not in |
| +// carlos-exec@<host>, so KillHost's answer about that cgroup is no proof of |
| +// the worker. RoleGone for the worker role is: it refuses while the unit |
| +// runs and holds once the unit is terminal with its cgroup empty. |
| +func TestKillHostIsNoProofForANestedWorker(t *testing.T) { |
| + t.Parallel() |
| + ctl := &fakeControl{} |
| + unit := &fakeUnit{} |
| + unit.set("active", "inv1", true, true) // the worker unit runs |
| + b, rt, _ := nestedWorkerBackend(t, ctl, unit) |
| + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) |
| + defer cancel() |
| + // KillHost speaks for carlos-exec@<host>, which a nested worker never |
| + // enters: with no delegated cgroup it answers ErrNoDelegation, the |
| + // answer callers take as "the leader's exit is all there is". That says |
| + // nothing of the unit, which still runs. |
| + if err := b.KillHost(ctx, rt); err != nil && !errors.Is(err, cgroup.ErrNoDelegation) { |
| + t.Fatalf("KillHost = %v", err) |
| + } |
| + if err := b.RoleGone(ctx, rt, instancectl.RoleWorker); err == nil { |
| + t.Fatal("RoleGone proved a running worker unit gone") |
| + } |
| + unit.set("failed", "inv1", false, false) |
| + pctx, pcancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + defer pcancel() |
| + if err := b.RoleGone(pctx, rt, instancectl.RoleWorker); err != nil { |
| + t.Fatalf("RoleGone over a terminal unit with an empty cgroup: %v", err) |
| + } |
| + if calls := ctl.joined(); !strings.Contains(calls, "stop "+rt.Host+" worker") { |
| + t.Fatalf("calls:\n%s", calls) |
| + } |
| +} |
diff --git a/internal/activator/worker_stub_test.go b/internal/activator/worker_stub_test.go| new file mode 100644 |
| index 000000000..aed90713d |
| --- /dev/null |
| +++ b/internal/activator/worker_stub_test.go |
| @@ -0,0 +1,87 @@ |
| +package activator |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "fmt" |
| + "net" |
| + "net/http" |
| + "os" |
| + "path/filepath" |
| + "strconv" |
| + "time" |
| +) |
| + |
| +// fakeWorkerMain is the tenant program a nested worker test's build execs |
| +// (TestMain, ACTIVATOR_FAKE_WORKER=1): the worker's side of the contract, |
| +// reading only what its launch env hands it. CARLOS_SIDECAR_DB is the one |
| +// database path it is told (D4); the agent socket and token are its only |
| +// channel back (D3). STUB_MODE, from the tenant's own env file, chooses: |
| +// |
| +// - serve: open the database at CARLOS_SIDECAR_DB (created, to prove the |
| +// path is usable), report ready, record STUB_NEXT_DUE when set, then |
| +// wait for the stop signal; |
| +// - finish: the same, then exit 0 (a one-shot's work is done); |
| +// - flat-db: open STUB_DATA_ROOT/<host>.db, as a worker that computes the |
| +// flat path itself does (Review Focus 1). In the namespace /data is a |
| +// read-only tmpfs, so that open fails and the worker crashes before it |
| +// is ever ready; here STUB_DATA_ROOT is whatever the test makes |
| +// unopenable. |
| +func fakeWorkerMain() { |
| + host := os.Getenv("CARLOS_SIDECAR_HOST") |
| + mode := os.Getenv("STUB_MODE") |
| + if mode == "" && len(os.Args) > 1 { |
| + mode = os.Args[1] |
| + } |
| + open := func(path string) { |
| + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) |
| + if err != nil { |
| + fmt.Fprintf(os.Stderr, "worker: open %s: %v\n", path, err) |
| + os.Exit(1) |
| + } |
| + f.Close() |
| + } |
| + switch mode { |
| + case "flat-db": |
| + open(filepath.Join(os.Getenv("STUB_DATA_ROOT"), host+".db")) |
| + default: |
| + if db := os.Getenv("CARLOS_SIDECAR_DB"); db != "" { |
| + open(db) |
| + } |
| + } |
| + sock, tok := os.Getenv("CARLOS_AGENT_SOCKET"), os.Getenv("CARLOS_SIDECAR_TOKEN") |
| + cl := &http.Client{ |
| + Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&net.Dialer{}).DialContext(ctx, "unix", sock) |
| + }}, |
| + Timeout: 5 * time.Second, |
| + } |
| + post := func(path string, body any) { |
| + raw, _ := json.Marshal(body) |
| + req, _ := http.NewRequest("POST", "http://agent"+path, bytes.NewReader(raw)) |
| + req.Header.Set("Carlos-Sidecar-Host", host) |
| + req.Header.Set("Authorization", "Bearer "+tok) |
| + req.Header.Set("Content-Type", "application/json") |
| + resp, err := cl.Do(req) |
| + if err != nil { |
| + fmt.Fprintf(os.Stderr, "worker: %s: %v\n", path, err) |
| + os.Exit(2) |
| + } |
| + resp.Body.Close() |
| + if resp.StatusCode/100 != 2 { |
| + fmt.Fprintf(os.Stderr, "worker: %s: status %d\n", path, resp.StatusCode) |
| + os.Exit(2) |
| + } |
| + } |
| + post("/v1/ready", map[string]string{"version": "stub"}) |
| + if at := os.Getenv("STUB_NEXT_DUE"); at != "" { |
| + n, _ := strconv.ParseInt(at, 10, 64) |
| + post("/v1/next-due", map[string]int64{"at": n}) |
| + } |
| + fmt.Fprintln(os.Stdout, "worker: ready in mode "+mode) |
| + if mode == "finish" { |
| + os.Exit(0) |
| + } |
| + select {} |
| +} |