amadan
Sidecar isolation Task 3: the worker unit template (D1, D6)
carlos-instance-worker@.service is the sidecar template with five lines different: description, the role word on ExecStart= and ExecStopPost=, the worker log socket, and no SuccessExitStatus= so a signal death reads as a crash. Installed with the other embedded units, never enabled. Tests pin the five-line diff (non-comment lines), the shared namespace block, the hardening and own-build binds, and the embedded-unit list.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
4 files changed,
+272
−3
internal/host/host_test.go+1 −0internal/host/units/carlos-instance-worker@.service+183 −0internal/host/units_hardening_test.go+2 −1internal/host/units_isolation_test.go+86 −2
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (19 s)
=== gofmt (1 s)
gofmt -l .: clean
=== rest (172 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.858s
ok github.com/carlosframework/platform/cmd/moveprobe 11.689s
ok github.com/carlosframework/platform/cmd/zddprobe 0.753s
ok github.com/carlosframework/platform/internal/activator 58.566s
ok github.com/carlosframework/platform/internal/agentsock 0.020s
ok github.com/carlosframework/platform/internal/alerting 0.007s
ok github.com/carlosframework/platform/internal/analytics 0.020s
ok github.com/carlosframework/platform/internal/appconfig 3.120s
ok github.com/carlosframework/platform/internal/apperr 0.095s
ok github.com/carlosframework/platform/internal/applog 0.145s
ok github.com/carlosframework/platform/internal/awsinv 0.005s
ok github.com/carlosframework/platform/internal/boxchoice 0.005s
ok github.com/carlosframework/platform/internal/boxid 1.251s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.014s
ok github.com/carlosframework/platform/internal/certwarm 0.006s
ok github.com/carlosframework/platform/internal/cgroup 0.802s
ok github.com/carlosframework/platform/internal/changelog 0.433s
ok github.com/carlosframework/platform/internal/claimactivation 1.060s
ok github.com/carlosframework/platform/internal/clamd 0.058s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 7.954s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.011s
ok github.com/carlosframework/platform/internal/dbquery 0.010s
ok github.com/carlosframework/platform/internal/dbreads 2.095s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.010s
ok github.com/carlosframework/platform/internal/dnsrec 0.004s
ok github.com/carlosframework/platform/internal/docsite 0.009s
ok github.com/carlosframework/platform/internal/domains 3.166s
ok github.com/carlosframework/platform/internal/doorkey 0.008s
ok github.com/carlosframework/platform/internal/economics 0.008s
ok github.com/carlosframework/platform/internal/edge 45.772s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.133s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.016s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.155s
ok github.com/carlosframework/platform/internal/host 142.829s
ok github.com/carlosframework/platform/internal/hostactors 6.084s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.005s
ok github.com/carlosframework/platform/internal/hostbox 2.949s
ok github.com/carlosframework/platform/internal/hostclaim 0.757s
ok github.com/carlosframework/platform/internal/hostenv 0.009s
ok github.com/carlosframework/platform/internal/hostevidence 0.012s
ok github.com/carlosframework/platform/internal/hostexec 5.261s
ok github.com/carlosframework/platform/internal/hostfence 0.048s
ok github.com/carlosframework/platform/internal/hostfiles 10.303s
ok github.com/carlosframework/platform/internal/hostfloor 0.035s
ok github.com/carlosframework/platform/internal/hostguard 1.054s
ok github.com/carlosframework/platform/internal/hostops 2.044s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.002s
ok github.com/carlosframework/platform/internal/hostquarantine 0.005s
ok github.com/carlosframework/platform/internal/hostrelocate 0.677s
ok github.com/carlosframework/platform/internal/hostresolve 0.207s
ok github.com/carlosframework/platform/internal/hostwrite 0.003s
ok github.com/carlosframework/platform/internal/infraview 0.019s
ok github.com/carlosframework/platform/internal/instancectl 6.904s
ok github.com/carlosframework/platform/internal/instanceexec 2.807s
ok github.com/carlosframework/platform/internal/instancefs 0.012s
ok github.com/carlosframework/platform/internal/instancekey 0.020s
ok github.com/carlosframework/platform/internal/instanceupdates 0.022s
ok github.com/carlosframework/platform/internal/layout 1.795s
ok github.com/carlosframework/platform/internal/lbhealth 0.002s
ok github.com/carlosframework/platform/internal/lease 10.233s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.548s
ok github.com/carlosframework/platform/internal/ledger 0.019s
ok github.com/carlosframework/platform/internal/mail 0.210s
ok github.com/carlosframework/platform/internal/maintenance 0.081s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.589s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.121s
ok github.com/carlosframework/platform/internal/metrics 0.005s
ok github.com/carlosframework/platform/internal/moves 22.299s
ok github.com/carlosframework/platform/internal/moves/archive 0.973s
ok github.com/carlosframework/platform/internal/objstore 0.266s
ok github.com/carlosframework/platform/internal/ops 0.273s
ok github.com/carlosframework/platform/internal/park 0.312s
ok github.com/carlosframework/platform/internal/plans 0.007s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
2026/10/24 21:09:47 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:47 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:47 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:47 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 21:09:47 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:47 provision: an app's records could not be listed: they are skipped
2026/10/24 21:09:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:47 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:09:47 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 21:09:47 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:09:47 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:09:47 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:09:47 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 21:09:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:09:47 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:09:47 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:09:47 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:09:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:09:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:09:47 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:47 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:09:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:09:47 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:09:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 21:09:48 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:09:48 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 21:09:48 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 21:09:48 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:48 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:48 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:48 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:48 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:09:48 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 21:09:48 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 21:09:48 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 21:09:48 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:48 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:48 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.74s)
plan9_b8_census_test.go:372: record-write census rejected 2 site(s); update the explicit inventory only after reviewing the source:
changed site internal/instanceexec/exit.go:RunExit: source differs from the reviewed digest
changed site internal/ops/litestream.go:FleetDBsWithSkipped: source differs from the reviewed digest
2026/10/24 21:09:49 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:09:49 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:49 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 21:09:50 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 21:09:50 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 21:09:50 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 21:09:50 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:09:50 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:09:50 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:09:50 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 21:09:50 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 21:09:50 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:09:50 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 21:09:50 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 21:09:50 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 21:09:50 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:50 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 21:09:50 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 21:09:50 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 21:09:50 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:50 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:09:50 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:50 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 21:09:50 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:09:50 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:50 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 21:09:50 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:50 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 21:09:50 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:50 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:50 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 21:09:50 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 21:09:50 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 21:09:50 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:50 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:09:50 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 21:09:50 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 21:09:50 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:09:50 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 21:09:50 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 21:09:50 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 21:09:50 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:50 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 21:09:50 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 21:09:50 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 21:09:50 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 21:09:50 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 21:09:50 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 21:09:50 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 21:09:51 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 21:09:51 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:51 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:09:51 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 21:09:51 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 21:09:51 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:09:51 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 21:09:51 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:09:51 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:09:51 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 21:09:51 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:09:51 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 21:09:51 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 21:09:51 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 21:09:51 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 21:09:51 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 21:09:51 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 21:09:51 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 21:09:51 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 21:09:51 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:51 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:52 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:52 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 21:09:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:52 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 21:09:52 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 21:09:52 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 21:09:52 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:09:52 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:52 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 21:09:52 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:52 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:09:52 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:52 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:52 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 21:09:52 provision: a.shop.example.com: swapped for import sd_t7i2hpngs7rj7i5dr7q5dvswpa (the original route is journaled)
2026/10/24 21:09:52 provision: a.shop.example.com: import sd_t7i2hpngs7rj7i5dr7q5dvswpa committed: its gate is cleared and its journal row dropped
2026/10/24 21:09:52 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:52 provision: a.shop.example.com: import sd_x2gsqrd3jchuhh5wsnubtz7q6m rolled back to its original route
2026/10/24 21:09:52 provision: a.shop.example.com: import sd_x2gsqrd3jchuhh5wsnubtz7q6m: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:09:52 provision: a.shop.example.com: import sd_x2gsqrd3jchuhh5wsnubtz7q6m is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_6fs6nqmy2vmvl66whmpzyt7zmu rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_6fs6nqmy2vmvl66whmpzyt7zmu: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_6fs6nqmy2vmvl66whmpzyt7zmu is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_duu3bnyn5uazzzinficxoa3dsy
2026/10/24 21:09:53 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 21:09:53 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 21:09:53 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:53 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 21:09:53 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 21:09:53 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_tv4rv7zlav4vdv6qc523u5zsfa cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_tjtibu5smmu5eg7f3sklsad4uy cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_yfpychglfvgntuk2ybelzm4p7y is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_5y7cqlay2nep6yx2o5wejbr2te is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_mpadcjsjglt2e3dq7xnqqd75ea is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 21:09:53 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 21:09:53 provision: b.shop.example.com: import sd_nzsf62crfvlu23ndw3r52nitsq left alone this pass: the import journal could not be read
2026/10/24 21:09:53 provision: c.shop.example.com: import sd_u2cjkehtxdjdongwthiqdcxkam left alone this pass: the import journal could not be read
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_4sdvu4murwxt72gbp7wdkg525q committed: its gate is cleared and its journal row dropped
2026/10/24 21:09:53 provision: a.shop.example.com: not converging import sd_t2y2o7c4zzqvgbehfyh7lhax4e: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:09:53 provision: a.shop.example.com: not converging import sd_kg5sonsdubrj224sjtmeqzn4xu: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 21:09:53 provision: a.shop.example.com: not converging import sd_x2ey7bsznbka53anymv22gydli: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:09:53 provision: a.shop.example.com: not converging import sd_ielehtfze6wtwhybh4d7twhdnq: it is placed but not committed, and the row on this box has no import gate
2026/10/24 21:09:53 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:53 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 21:09:53 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_nllc2wh6q75yz66lvckbfk4vui committed: its gate is cleared and its journal row dropped
2026/10/24 21:09:53 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:53 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:53 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 21:09:53 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_qw7zg3b4vkqpulrpv2uyq4xi4q: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_qw7zg3b4vkqpulrpv2uyq4xi4q rolled back (no box had swapped it)
2026/10/24 21:09:53 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 21:09:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:53 provision: removed instance route acme.go.tito.io (its seed sd_bnafcmfq2tbfixf6a7w66nqgsi was aborted); the next pass rolls it back
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_bnafcmfq2tbfixf6a7w66nqgsi: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_bnafcmfq2tbfixf6a7w66nqgsi rolled back (no box had swapped it)
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_sw6jyojakpwi4pdqtdrkamhkyu has a row this pass may not remove; leaving the seed aborted
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_k76hxhuj6sokfiaprq44s5kyo4 rolled back (no box had swapped it)
2026/10/24 21:09:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:53 provision: removed instance route acme.go.tito.io (its seed sd_dgzvhynpeejmk6fjinnxupbcgm was aborted); the next pass rolls it back
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_dgzvhynpeejmk6fjinnxupbcgm has no route on this box: its local data is removed
2026/10/24 21:09:53 provision: acme.go.tito.io: aborted seed sd_dgzvhynpeejmk6fjinnxupbcgm rolled back (no box had swapped it)
2026/10/24 21:09:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_i2oaj4ucueiega2laxwhlyjpua (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_i2oaj4ucueiega2laxwhlyjpua rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_i2oaj4ucueiega2laxwhlyjpua is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_eipuuqathjjkhatjzz6hqvkaua (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_eipuuqathjjkhatjzz6hqvkaua rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_eipuuqathjjkhatjzz6hqvkaua is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_wkjx47tkoeahfxu7kuzs33ngva (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_wkjx47tkoeahfxu7kuzs33ngva rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_wkjx47tkoeahfxu7kuzs33ngva: the host's replica is left alone: its record no longer names this seed
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_wkjx47tkoeahfxu7kuzs33ngva is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_uwklngxpwhamryl7tcyqxrwzqe (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_uwklngxpwhamryl7tcyqxrwzqe rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_uwklngxpwhamryl7tcyqxrwzqe rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_uwklngxpwhamryl7tcyqxrwzqe is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_gb5ubjb3coxpngrfqw6e5su4rm (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_gb5ubjb3coxpngrfqw6e5su4rm rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_2764yg2cknrafzb55ulrsclcbi (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_2764yg2cknrafzb55ulrsclcbi is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_7lfot5i5eyw3fij5v2qoi62gla (the original route is journaled)
2026/10/24 21:09:53 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_7lfot5i5eyw3fij5v2qoi62gla left alone this pass: its record could not be read
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_tom5n2cx3z6ecsut6j2w3ghrti (the original route is journaled)
2026/10/24 21:09:53 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_tom5n2cx3z6ecsut6j2w3ghrti left alone this pass: its record could not be read
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_ibzeni77gc2xglynlfgkrdu7q4 (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_ibzeni77gc2xglynlfgkrdu7q4: close the import gate: constraint failed: injected (1811)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_ibzeni77gc2xglynlfgkrdu7q4 rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_ibzeni77gc2xglynlfgkrdu7q4 is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_txq7kyzyrqzgayfa7q6uimsf5q (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_txq7kyzyrqzgayfa7q6uimsf5q: close the import gate: constraint failed: injected (1811)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_txq7kyzyrqzgayfa7q6uimsf5q rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_txq7kyzyrqzgayfa7q6uimsf5q is cleaned up and its journal row retired
2026/10/24 21:09:53 provision: a.shop.example.com: swapped for import sd_eemgafkrgv34wknlrvue2csngm (the original route is journaled)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_eemgafkrgv34wknlrvue2csngm rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_eemgafkrgv34wknlrvue2csngm rolled back to its original route
2026/10/24 21:09:53 provision: a.shop.example.com: import sd_eemgafkrgv34wknlrvue2csngm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_nb6elmd7aowh5fy65cbfuufz6a (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_nb6elmd7aowh5fy65cbfuufz6a rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_nb6elmd7aowh5fy65cbfuufz6a is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_d3dlemne3ryq5ibsobwcvcgqc4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_d3dlemne3ryq5ibsobwcvcgqc4: retire its instance key: injected failure at retire-key
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_d3dlemne3ryq5ibsobwcvcgqc4 rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_d3dlemne3ryq5ibsobwcvcgqc4 is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_7benkywnoavhqknfowexw2abjm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_7benkywnoavhqknfowexw2abjm rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_7benkywnoavhqknfowexw2abjm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_iw3gokw4dutmv2dfgn5eq4atjm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_iw3gokw4dutmv2dfgn5eq4atjm: remove its local data: injected failure at remove-local
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_iw3gokw4dutmv2dfgn5eq4atjm rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_iw3gokw4dutmv2dfgn5eq4atjm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_wzonz6qvp6n4gomvvh5wj76w2q (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_wzonz6qvp6n4gomvvh5wj76w2q rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_wzonz6qvp6n4gomvvh5wj76w2q is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_5oi3ftjsl32hvrvp5wazaosrvq (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5oi3ftjsl32hvrvp5wazaosrvq: remove its local data: injected failure at remove-local-done
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5oi3ftjsl32hvrvp5wazaosrvq rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5oi3ftjsl32hvrvp5wazaosrvq is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_kflvmlvfw4p7kx3lahjimfnj2i (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_kflvmlvfw4p7kx3lahjimfnj2i rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_kflvmlvfw4p7kx3lahjimfnj2i is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_4wk6cawu7ht4o2syljo3jtimbq (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_4wk6cawu7ht4o2syljo3jtimbq: restore the original route: constraint failed: injected (1811)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_4wk6cawu7ht4o2syljo3jtimbq rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_4wk6cawu7ht4o2syljo3jtimbq is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_5tm6hmpuefuqqnbhlqhw2wbx34 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5tm6hmpuefuqqnbhlqhw2wbx34: restore the original route: constraint failed: injected (1811)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5tm6hmpuefuqqnbhlqhw2wbx34 rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_5tm6hmpuefuqqnbhlqhw2wbx34 is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_czyvzleg7jdwkdkydec5sv2q4q (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_czyvzleg7jdwkdkydec5sv2q4q is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_tk7on62taxnevngeg6a4d73eia (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tk7on62taxnevngeg6a4d73eia rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tk7on62taxnevngeg6a4d73eia: delete its snapshot: injected failure at delete-snapshot
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tk7on62taxnevngeg6a4d73eia is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_4r3hf223g72naoxtfeyqm5plla (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_4r3hf223g72naoxtfeyqm5plla rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_4r3hf223g72naoxtfeyqm5plla is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_vlwjgwj36pqbp4eeyjjzbupika (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_vlwjgwj36pqbp4eeyjjzbupika rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_vlwjgwj36pqbp4eeyjjzbupika: delete the host's replica: injected failure at delete-replica
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_vlwjgwj36pqbp4eeyjjzbupika is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_duqipi6g3m3uemti2cinnhhyky (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_duqipi6g3m3uemti2cinnhhyky rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_duqipi6g3m3uemti2cinnhhyky is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_tiqhfc4ywe5xmomalocur5qcla (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tiqhfc4ywe5xmomalocur5qcla rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tiqhfc4ywe5xmomalocur5qcla: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tiqhfc4ywe5xmomalocur5qcla is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ibbgp5el6gwyanqunjkojqnjs4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ibbgp5el6gwyanqunjkojqnjs4 rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ibbgp5el6gwyanqunjkojqnjs4: retire its journal row: constraint failed: injected (1811)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ibbgp5el6gwyanqunjkojqnjs4 is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_qqmhdel26tumtab56ge4mlirla (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_qqmhdel26tumtab56ge4mlirla rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_qqmhdel26tumtab56ge4mlirla is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: mark import sd_qqmhdel26tumtab56ge4mlirla rolled back: injected failure at cas
2026/10/24 21:09:54 provision: a.shop.example.com: aborted seed sd_qqmhdel26tumtab56ge4mlirla rolled back (no box had swapped it)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_6sg2rwlxe3p7du25ckyykk7vbe (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_6sg2rwlxe3p7du25ckyykk7vbe rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_6sg2rwlxe3p7du25ckyykk7vbe is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: aborted seed sd_6sg2rwlxe3p7du25ckyykk7vbe rolled back (no box had swapped it)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ol6annrgijdijsauthzxav3fzm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ol6annrgijdijsauthzxav3fzm rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ol6annrgijdijsauthzxav3fzm rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ol6annrgijdijsauthzxav3fzm rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ol6annrgijdijsauthzxav3fzm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_fjz3ivuayv3w7c6m6rlxutalfy (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_fjz3ivuayv3w7c6m6rlxutalfy rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_fjz3ivuayv3w7c6m6rlxutalfy is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_xpmez766tswmxdgt3fgasldbmy (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_xpmez766tswmxdgt3fgasldbmy rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_xpmez766tswmxdgt3fgasldbmy is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_fvawssf67cpqasssjxgxhne3su (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_fvawssf67cpqasssjxgxhne3su rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_fvawssf67cpqasssjxgxhne3su is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ztjdv3wvwiviwqeeqkyuhjde3a (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ztjdv3wvwiviwqeeqkyuhjde3a rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ztjdv3wvwiviwqeeqkyuhjde3a is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_otzal3xqwwdsgxdq4at7lwv4dm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_otzal3xqwwdsgxdq4at7lwv4dm rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_otzal3xqwwdsgxdq4at7lwv4dm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_dwq4f6qorlyuyak325q5eevnmq (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_dwq4f6qorlyuyak325q5eevnmq rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_dwq4f6qorlyuyak325q5eevnmq is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ihr7cejhke2qnteodqnlic3dve (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ihr7cejhke2qnteodqnlic3dve rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ihr7cejhke2qnteodqnlic3dve is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_hocv4sxs67mil25hj6dzyzuer4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_hocv4sxs67mil25hj6dzyzuer4 has no route on this box: its local data is removed and no route is put back
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_hocv4sxs67mil25hj6dzyzuer4 is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_v2kaiudfxow4xqqgqnlxcoimfm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_4wbo5ad5ghxcfbyhcjdedbtpwa (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_qniq5gr6yy2lp62stky52biqi4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_qniq5gr6yy2lp62stky52biqi4 cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_tihbmb6452aub2kfdk37njhdey (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_tihbmb6452aub2kfdk37njhdey is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 21:09:54 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 21:09:54 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_xrgkwbcyvlajila5266elgt2km (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_l5ijs5g6pg6vtsevrxrahpa3xq (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ppz6wfuss2gbm4j5u3ybnlwtfu (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_qwatjhzzd5utk3bgpfy6hhpaj4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_kvchvsmp6nakrt2fxqfm4ad2yi (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_kvchvsmp6nakrt2fxqfm4ad2yi rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_jcktcmllysyieqy3ij4jflctti (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_jcktcmllysyieqy3ij4jflctti has no route on this box: its local data is removed and no route is put back
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_jcktcmllysyieqy3ij4jflctti is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_wk2rcfwqzydyteivsezyatvmla (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_wk2rcfwqzydyteivsezyatvmla: remove its local data: injected failure at remove-local
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_wk2rcfwqzydyteivsezyatvmla is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_q2bfzrzhbnauwhq7l2vhjowm74 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_q2bfzrzhbnauwhq7l2vhjowm74: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_mobf5hbl7lxhqweiykuo3pnsj4 (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_mobf5hbl7lxhqweiykuo3pnsj4 rollback deferred: no fence generation: registry busy
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_em6vkgc2rg2btghjhkufzwbsba (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_em6vkgc2rg2btghjhkufzwbsba rollback deferred: no fence generation: the generation source answered 0
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_y4fy5vgqhinq6irr3lezy3tubi (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_6j3k6kme36dauinyol67b6676u (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_i2jmnek7dijbum6jr7wec5ujfu (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 21:09:54 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_aatu26knzskqgis6j7fk4r4s5e (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_aatu26knzskqgis6j7fk4r4s5e rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_aatu26knzskqgis6j7fk4r4s5e is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_nugj6mpz4pzmh2rxe3ks73c2iq (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_nugj6mpz4pzmh2rxe3ks73c2iq rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_nugj6mpz4pzmh2rxe3ks73c2iq is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_uhgrfedhcy4izxg3cyiv5l3qsm (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_uhgrfedhcy4izxg3cyiv5l3qsm rolled back to its original route
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_uhgrfedhcy4izxg3cyiv5l3qsm is cleaned up and its journal row retired
2026/10/24 21:09:54 provision: a.shop.example.com: swapped for import sd_ndjvwaem46r3timqc544bjxk5i (the original route is journaled)
2026/10/24 21:09:54 provision: a.shop.example.com: import sd_ndjvwaem46r3timqc544bjxk5i: retire its instance key: the env file is a symlink
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_ndjvwaem46r3timqc544bjxk5i rolled back to its original route
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_ndjvwaem46r3timqc544bjxk5i is cleaned up and its journal row retired
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_ydgzi6gyj76q7xljbanckemxuq (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_cmb7maybflfmcy63nrel64x6zq (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_s7wwzguxrtdwegwkye3r6je6w4 refused: there is no route at a.shop.example.com to replace
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_kah6lxdelspcksbocmkogrd7nm refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_kvji3i7ivblttxqofnj7imxecm refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_5vcbykwumsbep3xuminlowtg4a refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_xecjkhngdcijtx22sravmpqrmm refused: the route belongs to app other
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_lltxlgr7ezwqslidxzlooh5vj4 refused: the route is filed under another account
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_3vh2dcdcqmrqoi3iydj7k5kfeu refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_6hl245k4fyfjup375eslumzdfi refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_7meoqbudsbxiacqurwblr7kug4 refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_i7ksbeloevits3i6ouhnhkykdm refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_tgwlog4clxd5kbyspzzk5xezou refused: the route is not an address route this import can take over (socket)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_65xam2u3hrtp43e64hug3brenm refused: the route is not an address route this import can take over (no address)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_rxoaun2ksphv6uo2ajtbgalz3m refused: the route is not an address route this import can take over (provisioned)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_qs6ie7rltsshs4hlrm4lictitm refused: the route is not an address route this import can take over (kind static)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_eulxjwocpe6ksxnz52ptvbd3m4 refused: the route is not an address route this import can take over (backing)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_fhrr76vtnkt6emk23j32zmvaz4 refused: the route is not an address route this import can take over (tunnel)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_qdrd4d4wgedoxkypfbbvcedtra refused: the route is not an address route this import can take over (caps)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_fgqj6ms27cox74urhnideb7t6u refused: the route is not an address route this import can take over (platform)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_njiuplexgz6ukbdfjylrs7pzqa refused: the route is not an address route this import can take over (canary)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_m6klvehuhs7juqhiaokllczwua refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_5asx6lelqzb75bpziavpdzy3za refused: the route is not an address route this import can take over (region)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_vkq4zsn43bsxmn7tm2f7rzhsqi refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_k3dyqhtg64dv365s72b47dszi4 refused: the route is not an address route this import can take over (hibernate)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_xr62rixjwhgogkoaln4p2opqmq refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_inilyw2s3nxqsdtyn576byink4 refused: this box does not run instances nested
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_gdajxmhuuz6rwdiqzifurkehwu refused: this box does not run instances nested
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_su7cdpn342v7hkox6hd7takzwi refused: claim control is on for this box
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_y6iqftakf776thbgbafswyj5xa (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_c5ghfiach7vksa3jjnixmcthaq (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 21:09:55 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_6gx5n5y6tom3rgtdpxt3ngfape: its verify hash is not a sha256
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_qil6ucgkdtrbzf7yg5gz6zib6i: its verify hash is not a sha256
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_oebytdeo7o72x4e42n4oodisee: its verify hash is not a sha256
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 21:09:55 provision: A.shop.example.com/..: not swapping for import sd_brwc3nefm5r5wjtnhnpmc7q6sa: its host or seed id is malformed (seeds: not a host)
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_37upv5at3reb36mnho7u55voqy: its record does not describe a nested, hibernating instance
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_e2hi5g4kpjkijmzbjebdgosjqm: its record does not describe a nested, hibernating instance
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_jdm3qdb7oxnc4rhqro7t7jlloa: its record does not describe a nested, hibernating instance
2026/10/24 21:09:55 provision: a.shop.example.com: not swapping for import sd_3fsbjr4va5st2drm2wqu3mohaq: its record does not describe a nested, hibernating instance
2026/10/24 21:09:55 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_5krvc5vsomczwaalh3fkzpdovq (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_gys36wj3fkrepomutvggeyhxcm (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_spx5lks63bc5t4l32yv7va5y6u (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 21:09:55 provision: dry-run: would consider swapping a.shop.example.com for import sd_x7qwu7da5tougrjvnnk245wk3e
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_kiigdlokzgtkxkggw3iejiwffa rolled back to its original route
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_kiigdlokzgtkxkggw3iejiwffa: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_kiigdlokzgtkxkggw3iejiwffa is cleaned up and its journal row retired
2026/10/24 21:09:55 provision: a.shop.example.com: import sd_dzdv7ctqq5jojimcj5qfzm647m left alone: the import journal holds seed sd_syhzpkvjzkdeobssbrsgfkxrvu at swapped
2026/10/24 21:09:55 provision: a.shop.example.com: not minting import sd_mgbwjcveo2ucxy4n56v6pvkelm: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:09:55 provision: a.shop.example.com: not minting import sd_7zzr23oku4cg4y36lfzky7pe6u: it is placed but not committed, and only its swap makes its row
2026/10/24 21:09:55 provision: a.shop.example.com: swapped for import sd_m66hxd7qkqnluu74heiejkzvni (the original route is journaled)
2026/10/24 21:09:55 provision: a.shop.example.com: not minting import sd_m66hxd7qkqnluu74heiejkzvni: it is swapped but not committed, and only its swap makes its row
2026/10/24 21:09:55 provision: a.shop.example.com: not minting import sd_m66hxd7qkqnluu74heiejkzvni: it is placed but not committed, and only its swap makes its row
2026/10/24 21:09:55 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 21:09:55 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:09:55 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:55 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:55 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 21:09:55 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:55 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 21:09:55 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:55 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 21:09:55 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 21:09:55 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 21:09:55 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 21:09:55 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 21:09:55 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:09:55 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 21:09:55 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 21:09:55 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 21:09:55 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:09:55 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 21:09:56 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 21:09:56 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 21:09:56 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 21:09:56 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 21:09:56 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 21:09:56 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 21:09:56 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 21:09:56 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 21:09:56 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 21:09:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:56 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 21:09:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:56 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:09:56 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:09:56 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:09:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 21:09:56 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 21:09:56 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 21:09:56 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 21:09:56 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:56 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 21:09:56 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:56 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: record view: change index unusable (get refused): walking
2026/10/24 21:09:56 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 21:09:56 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:09:56 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 21:09:56 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 21:09:56 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 21:09:56 provision: an app's records could not be listed: they are skipped
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:09:56 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:09:56 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 21:09:56 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 21:09:56 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:09:56 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:09:56 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:09:56 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 21:09:56 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:09:56 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:09:56 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 21:09:56 provision: removed retired instance route one.example.com (record gone)
2026/10/24 21:09:56 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 21:09:56 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 21:09:56 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 21:09:56 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (15.17s)
plan14_root_reads_test.go:600: root read census: 17 discrepancies
changed root read internal/instancectl/server.go:parseRequest; review class, task, sites and digest
changed root read internal/instancectl/server.go:type Server; review class, task, sites and digest
changed root read internal/ops/hibernate_verify.go:HibernatingHosts; review class, task, sites and digest
changed root read internal/ops/litestream.go:FleetDBsWithSkipped; review class, task, sites and digest
changed root read internal/ops/ops.go:runLitestream; review class, task, sites and digest
changed root source envelope internal/host/units
changed root source envelope internal/instancectl
changed root source envelope internal/instanceexec
changed root source envelope internal/ops
unclassified root read internal/host/units/carlos-instance-worker@.service:external file:external@0
unclassified root read internal/ops/litestream.go:ActivatorReplicated registry:Backing@59,registry:DBPath@97,registry:Hibernate@117
unclassified root read internal/ops/litestream.go:ActivatorReplicatedPaths registry:DBPath@257,whole:Route@178,whole:Route@212
changed legacy declaration internal/instancectl/server.go:parseRequest
changed legacy declaration internal/instancectl/server.go:type Server
changed legacy declaration internal/ops/hibernate_verify.go:HibernatingHosts
changed legacy declaration internal/ops/litestream.go:FleetDBsWithSkipped
changed legacy declaration internal/ops/ops.go:runLitestream
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 23.941s
ok github.com/carlosframework/platform/internal/publicrelease 0.011s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.568s
ok github.com/carlosframework/platform/internal/relay 7.239s
ok github.com/carlosframework/platform/internal/release 25.846s
ok github.com/carlosframework/platform/internal/rollout 3.857s
ok github.com/carlosframework/platform/internal/rolloutwords 0.004s
ok github.com/carlosframework/platform/internal/rootaws 0.008s
ok github.com/carlosframework/platform/internal/rootparticipation 0.064s
ok github.com/carlosframework/platform/internal/rootstate 0.030s
ok github.com/carlosframework/platform/internal/runs 0.522s
ok github.com/carlosframework/platform/internal/s3compat 0.008s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.721s
ok github.com/carlosframework/platform/internal/seeds 0.013s
ok github.com/carlosframework/platform/internal/sim/engines 2.805s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.956s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 3.820s
ok github.com/carlosframework/platform/internal/status 0.104s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.300s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.006s
ok github.com/carlosframework/platform/internal/termfence 0.005s
ok github.com/carlosframework/platform/internal/tunnelclient 0.006s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.006s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.676s
FAIL
=== internal_console@0 (81 s)
ok github.com/carlosframework/platform/internal/console 79.129s
=== internal_console@1 (44 s)
ok github.com/carlosframework/platform/internal/console 42.407s
=== internal_console@2 (42 s)
ok github.com/carlosframework/platform/internal/console 40.240s
=== internal_console@3 (42 s)
ok github.com/carlosframework/platform/internal/console 40.655s
=== internal_console@4 (43 s)
ok github.com/carlosframework/platform/internal/console 40.837s
=== internal_console@5 (42 s)
ok github.com/carlosframework/platform/internal/console 40.432s
=== internal_console@6 (38 s)
ok github.com/carlosframework/platform/internal/console 36.152s
=== internal_console@7 (46 s)
ok github.com/carlosframework/platform/internal/console 44.175s
=== root@0 (110 s)
ok github.com/carlosframework/platform 108.142s
=== root@1 (84 s)
ok github.com/carlosframework/platform 82.606s
=== root@2 (93 s)
using http://127.0.0.1:45277 as a@keymail.dev
using http://127.0.0.1:32807 as vicky@keymail.dev
titogo (staging):
KEY VALUE
PORT 8080
--- FAIL: TestRunBuiltCopyIsApplied (0.14s)
approved_run_copy_test.go:139: RUN-C13: internal/instanceexec/exec.go does not contain the approved text "instance-exec: role <role> is not run, seed, sidecar or cmd"
approved_run_copy_test.go:139: RUN-C14: internal/instanceexec/exit.go does not contain the approved text "instance-exit: role <role> is not run, seed, sidecar or cmd"
2026/10/24 21:10:00 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:00 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A46239": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:00 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:00 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A46239": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:00 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:00 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:00 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A42051": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:00 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:02 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:02 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A35957": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:02 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:02 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:02 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A43943": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:02 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:04 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:04 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A39007": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:04 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:07 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:07 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A44837": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:07 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
using http://127.0.0.1:36365 as paul@keymail.dev
account: ops (bab)
2026/10/24 21:10:09 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A36365": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:09 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
set 1 env var(s) for titogo (default)
using http://127.0.0.1:36365 as paul@keymail.dev
account: ops (bab)
set 1 secret(s) for titogo (default)
account: ops (bab)
account: ops (bab)
using http://127.0.0.1:36365 as paul@keymail.dev
account: ops (bab)
config sync requested for titogo the files land within seconds; programs carlos owns restart themselves, and anything else keeps its old values until you restart it
2026/10/24 21:10:09 console: delivery to mallory@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=mallory%40keymail.dev&from=console%40127.0.0.1%3A36365": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:09 console: could not notify mallory@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:11 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:10:11 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:10:11 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:10:12 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:10:12 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:10:12 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:10:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:12 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A43661": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:12 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:12 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:10:12 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:10:12 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:10:13 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:10:13 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:10:13 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:10:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:14 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A37825": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:14 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:55 console: db read dbr_wyeh4rxsuuc7asui6ownv4e2wu: relayed 1 chunks (101 bytes) in 41.032s, ended done
2026/10/24 21:10:55 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:10:55 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:10:55 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:10:56 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:10:56 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:10:56 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:10:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:56 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A32779": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:56 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:10:57 console: db read dbr_f2rny7rax2tpssngluvpp4holy: relayed 1 chunks (1070 bytes) in 1s, ended done
2026/10/24 21:10:58 console: db read dbr_uc5wye6nf2dss3p4hbkwf6jfyu: relayed 1 chunks (805 bytes) in 1.001s, ended done
2026/10/24 21:10:58 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:10:58 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 21:10:58 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:10:59 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:10:59 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 21:10:59 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:10:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:10:59 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A44809": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:10:59 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 21:11:00 console: db read dbr_lqhc6b46wmrqgkujdm5fubqbpy: relayed 2 chunks (8386 bytes) in 1.042s, ended client gone
using http://127.0.0.1:41977 as paul@keymail.dev
account: ops (bab)
attached tickets.example.com -> default.go.tito.io
DNS: pointing
using http://127.0.0.1:41977 as paul@keymail.dev
account: ops (bab)
attached carloku.example.com -> default.go.tito.io
2026/10/24 21:11:01 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:11:01 console: delivery to rival@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=rival%40keymail.dev&from=console%40127.0.0.1%3A39395": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:11:01 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A39395": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:11:01 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:11:01 console: could not notify rival@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:11:10 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A39401": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:11:10 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:11:10 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 21:11:10 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A32943": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 21:11:10 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 21:11:11 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 21:11:11 domains: minted platform domain blog.mdbbndjqx.oncarlos.test (upstream blog.kass.test)
2026/10/24 21:11:11 domains: minted platform domain blog.gdpkwhwrw.oncarlos.test (upstream blog.woodstar.test)
2026/10/24 21:11:11 domains: minted platform domain oldsite.bab.oncarlos.test (upstream old.test)
2026/10/24 21:11:11 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 21:11:12 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
2026/10/24 21:11:12 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
FAIL
FAIL github.com/carlosframework/platform 91.068s
FAIL
FAILED: rest root@2
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index e1154574e..23c14e229 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -49,6 +49,7 @@ var expectedUnits = []string{ |
| "carlos-instance-probe.service", |
| "carlos-instance-seed@.service", |
| "carlos-instance-sidecar@.service", |
| + "carlos-instance-worker@.service", |
| "carlos-instance@.service", |
| "carlos-instances.slice", |
| "carlos-litestream.service", |
diff --git a/internal/host/units/carlos-instance-worker@.service b/internal/host/units/carlos-instance-worker@.service| new file mode 100644 |
| index 000000000..948040ef0 |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-worker@.service |
| @@ -0,0 +1,183 @@ |
| +# One nested native worker (sidecar isolation design D1, D6): the sidecar |
| +# template's twin for a worker that runs as its own host. Five directives |
| +# differ from carlos-instance-sidecar@.service: this description, the role |
| +# word on ExecStart= and ExecStopPost=, the worker log socket, and no |
| +# SuccessExitStatus= (see below). Everything else, the whole namespace |
| +# block included, is byte-identical, and a test holds it so. The template |
| +# is installed with the others and never enabled: the agent starts a worker |
| +# by name. The activator owns it exactly as it owned the exec child this |
| +# replaces: lease, restore, seed, replication, hibernation, control socket, |
| +# logs. What changed is that the process is a unit, still uid carlos, |
| +# inside a mount namespace in which nothing but the instance's own two |
| +# directories exists, on mounts where no symlink is ever followed. The |
| +# instance name is the escaped bounded identifier (%i); paths use the |
| +# unescaped one (%I). |
| +[Unit] |
| +Description=carlos exec-backed instance %I (worker) |
| +# Stop-only coupling: when the edge stops (including the stop half of a |
| +# restart) this unit stops, and nothing starts it again but the agent. |
| +# PartOf= would propagate the RESTART and race the agent's own startup |
| +# (codex R1-7). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=simple |
| +ExecStart=/opt/carlos/carlos instance-exec worker %I |
| +# Runs inside the same namespace, as carlos, after this invocation's |
| +# processes have been signalled: leaves an ADVISORY exit record in the |
| +# instance's own run directory (§6.5). Not "+": the "+" prefix drops User= |
| +# and the Private*/Protect* settings but keeps the explicit |
| +# TemporaryFileSystem=/BindPaths= mounts (codex R3-2), so nothing run from |
| +# here can see the host's /run/carlos anyway. |
| +ExecStopPost=/opt/carlos/carlos instance-exit worker %I |
| +User=carlos |
| +Group=carlos |
| +# One slice per host holds the instance, its seed and its sidecar, and |
| +# carries the tier (§4.3, §6.6). Slices referenced here are created on |
| +# demand; no per-host unit file exists. |
| +Slice=carlos-instances-%i.slice |
| +# The activator owns recycling: a crash stands the entry down and the next |
| +# request wakes it. systemd must not race that. |
| +Restart=no |
| +# SIGTERM to the app, SIGKILL to anything it left behind after |
| +# TimeoutStopSec; the activator's stop budget is 20s. |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# The main process is the trampoline, which stays as the namespace's init |
| +# and runs the app as its child (#126): it forwards the SIGTERM and exits |
| +# 128+n when the app dies by signal n. The sidecar template lists 129 130 |
| +# 141 143 as clean here; this one deliberately does not. A worker is its |
| +# own host, and a death by signal is a crash that must read as one, so |
| +# the agent restarts it with backoff and "carlos sidecars" shows it failed. |
| +# The log stream: stdout and stderr connect to the agent's per-role |
| +# listener (§6.4). PID 1 opens this as root, outside the namespace, on the |
| +# nosymfollow logs mount, at a path no tenant mount contains. |
| +StandardOutput=file:/run/carlos/logs/%I.worker.sock |
| +StandardError=inherit |
| +# The instance's own two directories, the only writable data paths. |
| +BindPaths=/data/instances/%I /run/carlos/instances/%I |
| +ReadWritePaths=/data/instances/%I /run/carlos/instances/%I |
| +# The working directory is the instance's own data directory, the one its |
| +# database is in (discussion #256 item D). A flat child ran with cwd /, |
| +# and an app writing relative paths (./data/blobs) wrote under /data; |
| +# without this line a unit's cwd is / on a read-only tmpfs and that app |
| +# exits at start with "read-only file system". systemd applies it after |
| +# the namespace is set up, so the bind above is what it chdirs into. Same |
| +# value in all three templates: the seed and the sidecar see the files |
| +# the instance made from the same place. |
| +WorkingDirectory=/data/instances/%I |
| +# Its own build, and no other app's (#214 item 2). The namespace block |
| +# empties /opt/carlos; these put back /opt/carlos/live/<host>, a link to |
| +# ../app, and /opt/carlos/app, which PID 1 binds by chasing the source |
| +# link to this host's live build at every start. The control service |
| +# writes both sources before it starts the unit. |
| +BindReadOnlyPaths=/run/carlos-control/exe/%I/app:/opt/carlos/app /run/carlos-control/exe/%I/live:/opt/carlos/live |
| + |
| +# --- the namespace (titogo@.service, 2026-07-31 design, on systemd 257) --- |
| +ProtectSystem=strict |
| +TemporaryFileSystem=/data:ro |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Every other tree a carlos-uid unit writes on the box is hidden the same |
| +# way (codex R9-2): the restore drills' cache under /var/cache, and |
| +# /var/lib as a class. An instance needs nothing from either. |
| +TemporaryFileSystem=/var/cache:ro |
| +TemporaryFileSystem=/var/lib:ro |
| +# The version store holds every app's build on the box, world-readable so |
| +# a unit can exec its own (#214 item 2). An instance sees none of it: an |
| +# empty /opt/carlos with the platform binary that ExecStart= and |
| +# ExecStopPost= run put back, and, outside this block because it is per |
| +# instance, its own build at /opt/carlos/live/<host>. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Root-written credentials a carlos-GROUP process may read live here: |
| +# /etc/carlos/relay-masters is root:carlos 0640 for carlos-relay.service, |
| +# and an instance's Group=carlos would read it too (codex R10-1). An |
| +# instance gets its configuration through its launch file and needs |
| +# nothing from /etc/carlos. /run/carlos-control is the root control |
| +# service's socket directory (design §5.2): hiding it here too means a |
| +# connect from inside any instance's namespace is impossible outright, |
| +# not merely refused by the peer check on the far end (final review, |
| +# plan 2, item 3). |
| +InaccessiblePaths=/etc/carlos /run/carlos-control |
| +ProtectHome=yes |
| +PrivateTmp=yes |
| +PrivateDevices=yes |
| +# PrivateDevices= bind-mounts the HOST's /dev/shm into the private /dev, |
| +# and every instance is uid carlos, so a sibling's shared memory would be |
| +# one open away. Each unit gets its own, charged to its own cgroup. |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# POSIX message queues and SysV IPC belong to an IPC namespace, and every |
| +# instance is uid carlos, so a sibling's queue would be one mq_open away |
| +# (#178). PrivateIPC= gives each unit its own namespace, and mq_open() |
| +# goes through that namespace's own internal mount. PrivateDevices= |
| +# carries the host's /dev/mqueue into the private /dev the way it carries |
| +# /dev/shm, and that mount would still list and open the host namespace's |
| +# queues, so it is replaced the same way. |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# /sys/fs/cgroup read-only: the edge's delegated subtree is carlos-owned, |
| +# and its cgroup.kill / cgroup.freeze would otherwise be one write away |
| +# for a same-uid tenant (codex R9-1). |
| +ProtectControlGroups=yes |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# An allow opens an address both ways and on every port, so a loopback |
| +# resolver can never be opened: a tenant could bind systemd-resolved's |
| +# 127.0.0.53 itself and reach every listener bound to all addresses |
| +# through it. Where resolved runs, instances resolve through its |
| +# upstreams instead, and the control service opens those. The "-" makes |
| +# this a no-op on a box without resolved. |
| +BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the tenant trees are nosuid, PrivateTmp's sit under a root 0700 |
| +# directory). |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). The sidecar |
| +# is a sibling service with its own family rule (§4.3). |
| +OOMPolicy=kill |
diff --git a/internal/host/units_hardening_test.go b/internal/host/units_hardening_test.go| index 776b93b7b..84a3f5ae0 100644 |
| --- a/internal/host/units_hardening_test.go |
| +++ b/internal/host/units_hardening_test.go |
| @@ -42,7 +42,7 @@ const instanceDeny = "IPAddressDeny=localhost multicast " + |
| // the four templates and the probe (TestInstanceTemplatesShareOneNamespace), |
| // so pinning it on carlos-instance@ pins it on all five. |
| func TestNamespaceBlockCarriesTheHardening(t *testing.T) { |
| - for _, name := range []string{"carlos-instance@.service", "carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| + for _, name := range []string{"carlos-instance@.service", "carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-worker@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| u := readUnit(t, name) |
| for _, want := range []string{ |
| // #214 item 2: no other app's build is visible. |
| @@ -110,6 +110,7 @@ func TestInstanceTemplatesBindOnlyTheirOwnBuild(t *testing.T) { |
| "carlos-instance@.service": {"%I", ""}, |
| "carlos-instance-seed@.service": {"%I", ""}, |
| "carlos-instance-sidecar@.service": {"%I", ""}, |
| + "carlos-instance-worker@.service": {"%I", ""}, |
| "carlos-instance-cmd@.service": {"%I", "cmd-"}, |
| "carlos-instance-probe.service": {"_probe", ""}, |
| } { |
diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go| index 21745e8a3..f6fe64ee2 100644 |
| --- a/internal/host/units_isolation_test.go |
| +++ b/internal/host/units_isolation_test.go |
| @@ -3,8 +3,11 @@ package host |
| import ( |
| "os" |
| "path/filepath" |
| + "slices" |
| "strings" |
| "testing" |
| + |
| + "github.com/carlosframework/platform/internal/instancectl" |
| ) |
| func readUnit(t *testing.T, name string) string { |
| @@ -239,7 +242,7 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| return u[start:end] |
| } |
| ref := block("carlos-instance@.service") |
| - for _, name := range []string{"carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| + for _, name := range []string{"carlos-instance-gen@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-worker@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| if block(name) != ref { |
| t.Errorf("%s namespace block differs from carlos-instance@.service", name) |
| } |
| @@ -288,6 +291,9 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| "carlos-instance-sidecar@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec sidecar %I", |
| "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock", |
| "SuccessExitStatus=129 130 141 143", "WorkingDirectory=/data/instances/%I"}, |
| + "carlos-instance-worker@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec worker %I", |
| + "ExecStopPost=/opt/carlos/carlos instance-exit worker %I", "StandardOutput=file:/run/carlos/logs/%I.worker.sock", |
| + "WorkingDirectory=/data/instances/%I"}, |
| "carlos-instance-probe.service": {"Type=oneshot", "ExecStart=/opt/carlos/carlos instance-probe", |
| "BindPaths=/data/instances/_probe /run/carlos/instances/_probe", "StandardOutput=truncate:/run/carlos-control/probe.out"}, |
| } { |
| @@ -307,7 +313,9 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| t.Errorf("%s uses ExecStopPost=+: the plus prefix keeps the unit's tmpfs (codex R3-2)", name) |
| } |
| // A seed a signal ended has not seeded (#126). |
| - if (name == "carlos-instance-seed@.service" || name == "carlos-instance-cmd@.service") && strings.Contains(u, "\nSuccessExitStatus=") { |
| + // A worker is its own host: a signal death must read as a crash |
| + // (sidecar isolation D6), so it carries no SuccessExitStatus= either. |
| + if (name == "carlos-instance-seed@.service" || name == "carlos-instance-cmd@.service" || name == "carlos-instance-worker@.service") && strings.Contains(u, "\nSuccessExitStatus=") { |
| t.Errorf("%s counts a signal's exit status as success", name) |
| } |
| if name != "carlos-instance-probe.service" && !strings.Contains(u, "User=carlos") { |
| @@ -497,3 +505,79 @@ func TestGenerationTemplateKeysNothingByItsInstanceString(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// The worker template is the sidecar template with five directives |
| +// different (sidecar isolation D1, D6): the description, the two commands' |
| +// role word, the log socket and the missing SuccessExitStatus=. The |
| +// comparison is over non-comment lines, so the worker's header can say what |
| +// it is without counting toward the five. |
| +func TestWorkerTemplateDiffersInFiveLines(t *testing.T) { |
| + lines := func(name string) []string { |
| + var out []string |
| + for _, l := range strings.Split(readUnit(t, name), "\n") { |
| + if l == "" || strings.HasPrefix(l, "#") { |
| + continue |
| + } |
| + out = append(out, l) |
| + } |
| + return out |
| + } |
| + side, work := lines("carlos-instance-sidecar@.service"), lines("carlos-instance-worker@.service") |
| + // Both sides keep their order, so a longest-common-subsequence walk is |
| + // a faithful line diff. |
| + n, m := len(side), len(work) |
| + lcs := make([][]int, n+1) |
| + for i := range lcs { |
| + lcs[i] = make([]int, m+1) |
| + } |
| + for i := n - 1; i >= 0; i-- { |
| + for j := m - 1; j >= 0; j-- { |
| + if side[i] == work[j] { |
| + lcs[i][j] = lcs[i+1][j+1] + 1 |
| + } else { |
| + lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) |
| + } |
| + } |
| + } |
| + var removed, added []string |
| + for i, j := 0, 0; i < n || j < m; { |
| + switch { |
| + case i < n && j < m && side[i] == work[j]: |
| + i++ |
| + j++ |
| + case i < n && (j == m || lcs[i+1][j] >= lcs[i][j+1]): |
| + removed = append(removed, side[i]) |
| + i++ |
| + default: |
| + added = append(added, work[j]) |
| + j++ |
| + } |
| + } |
| + wantRemoved := []string{ |
| + "Description=carlos exec-backed instance %I (sidecar)", |
| + "ExecStart=/opt/carlos/carlos instance-exec sidecar %I", |
| + "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", |
| + "SuccessExitStatus=129 130 141 143", |
| + "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock", |
| + } |
| + wantAdded := []string{ |
| + "Description=carlos exec-backed instance %I (worker)", |
| + "ExecStart=/opt/carlos/carlos instance-exec worker %I", |
| + "ExecStopPost=/opt/carlos/carlos instance-exit worker %I", |
| + "StandardOutput=file:/run/carlos/logs/%I.worker.sock", |
| + } |
| + slices.Sort(removed) |
| + slices.Sort(wantRemoved) |
| + slices.Sort(added) |
| + slices.Sort(wantAdded) |
| + if !slices.Equal(removed, wantRemoved) { |
| + t.Errorf("lines only in the sidecar template:\n got %q\nwant %q", removed, wantRemoved) |
| + } |
| + if !slices.Equal(added, wantAdded) { |
| + t.Errorf("lines only in the worker template:\n got %q\nwant %q", added, wantAdded) |
| + } |
| + // The unit name the Go side builds is the file name embedded here. |
| + if file := instancectl.UnitPrefixWorker + ".service"; file != "carlos-instance-worker@.service" { |
| + t.Errorf("UnitPrefixWorker %q names no embedded template (%s)", instancectl.UnitPrefixWorker, file) |
| + } |
| +} |