amadan
Sidecar isolation Task 2: the worker role at root, fenced like the run unit (D1, D19)
names.go gains RoleWorker and UnitPrefixWorker (carlos-instance-worker@), with prefixFor, ParseRole and ParseUnitName cases; the slice name is unchanged, a worker's slice being its own host's. The control service admits worker on /start, /stop and /reset and sweeps the new template; a worker start takes the run role's tier rule (always set-property: the populated-slice exemption stays the legacy sidecar's), may name its staged build, validated with validateStagedExe against the worker host's own app, and is refused with a generation (one unit per host, never a generation). clearLegacyClaimLaunch stays run-only. The #292 ordered start/stop fence is keyed by (role, host) instead of host and covers RoleRun and RoleWorker: a worker /stop cancels and joins the worker starts in flight, applies the accepted-connection watermark to that unit alone, and only then stops it, so a nested worker start that timed out on the agent's side is never submitted by a late handler after the /stop has answered (spec §3.5.1, D19). A worker's stop never fences the same host's run unit, nor the reverse. instance-exec and instance-exit accept worker (launch.worker.json, exit.worker.<inv>.json); a worker "<iid>:<n>" is refused like a seed's. Tests: TestParseUnitNameWorker, TestStartWorkerSetsTier, TestStartWorkerRefusesGen, TestStopResetAndSweepCoverTheWorkerRole, TestWorkerStartValidatesStagedExe, and the #292 shape parameterised by role: TestWorkerStopCancelsAndJoinsHeldStart, TestWorkerStartAcceptedBeforeStopIsRefused, TestWorkerJoinPastBudgetAnswersError, TestWorkerStopDoesNotFenceRunUnit, TestRunStopDoesNotFenceWorkerUnit; plus the trampoline's worker launch and exit-record cases. Census: two claim lines appended (server.go parseRequest, type Server); census-propose is Task 12's.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
13 files changed,
+510
−115
implementation/census-claims-sidecar-isolation.txt+2 −0internal/instancectl/legacystart.go+64 −46internal/instancectl/legacystart_live_test.go+28 −10internal/instancectl/legacystart_test.go+89 −8internal/instancectl/names.go+16 −7internal/instancectl/names_test.go+43 −3internal/instancectl/server.go+50 −37internal/instancectl/server_test.go+86 −0internal/instancectl/stagedexe_linux_test.go+87 −0internal/instanceexec/exec.go+2 −2internal/instanceexec/exec_test.go+30 −0internal/instanceexec/exit.go+2 −2internal/instanceexec/exit_test.go+11 −0
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok amadan.net/carlos/platform.git/launchctx 0.013s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (17 s)
=== gofmt (1 s)
gofmt -l .: clean
=== rest (171 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.638s
ok github.com/carlosframework/platform/cmd/moveprobe 12.029s
ok github.com/carlosframework/platform/cmd/zddprobe 0.757s
ok github.com/carlosframework/platform/internal/activator 58.618s
ok github.com/carlosframework/platform/internal/agentsock 0.030s
ok github.com/carlosframework/platform/internal/alerting 0.007s
ok github.com/carlosframework/platform/internal/analytics 0.042s
ok github.com/carlosframework/platform/internal/appconfig 2.944s
ok github.com/carlosframework/platform/internal/apperr 0.227s
ok github.com/carlosframework/platform/internal/applog 0.286s
ok github.com/carlosframework/platform/internal/awsinv 0.007s
ok github.com/carlosframework/platform/internal/boxchoice 0.010s
ok github.com/carlosframework/platform/internal/boxid 0.670s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.009s
ok github.com/carlosframework/platform/internal/caps 0.017s
ok github.com/carlosframework/platform/internal/certwarm 0.008s
ok github.com/carlosframework/platform/internal/cgroup 0.797s
ok github.com/carlosframework/platform/internal/changelog 0.595s
ok github.com/carlosframework/platform/internal/claimactivation 1.307s
ok github.com/carlosframework/platform/internal/clamd 0.067s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 7.954s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.021s
ok github.com/carlosframework/platform/internal/dbquery 0.005s
ok github.com/carlosframework/platform/internal/dbreads 2.130s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.008s
ok github.com/carlosframework/platform/internal/dnsrec 0.004s
ok github.com/carlosframework/platform/internal/docsite 0.009s
ok github.com/carlosframework/platform/internal/domains 2.975s
ok github.com/carlosframework/platform/internal/doorkey 0.008s
ok github.com/carlosframework/platform/internal/economics 0.009s
ok github.com/carlosframework/platform/internal/edge 45.450s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.135s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.017s
ok github.com/carlosframework/platform/internal/geo 0.020s
ok github.com/carlosframework/platform/internal/geoip 0.029s
ok github.com/carlosframework/platform/internal/historyreadback 0.153s
ok github.com/carlosframework/platform/internal/host 142.632s
ok github.com/carlosframework/platform/internal/hostactors 6.440s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.006s
ok github.com/carlosframework/platform/internal/hostbox 2.856s
ok github.com/carlosframework/platform/internal/hostclaim 0.755s
ok github.com/carlosframework/platform/internal/hostenv 0.009s
ok github.com/carlosframework/platform/internal/hostevidence 0.006s
ok github.com/carlosframework/platform/internal/hostexec 5.189s
ok github.com/carlosframework/platform/internal/hostfence 0.055s
ok github.com/carlosframework/platform/internal/hostfiles 10.256s
ok github.com/carlosframework/platform/internal/hostfloor 0.018s
ok github.com/carlosframework/platform/internal/hostguard 1.047s
ok github.com/carlosframework/platform/internal/hostops 1.943s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.005s
ok github.com/carlosframework/platform/internal/hostquarantine 0.005s
ok github.com/carlosframework/platform/internal/hostrelocate 0.597s
ok github.com/carlosframework/platform/internal/hostresolve 0.231s
ok github.com/carlosframework/platform/internal/hostwrite 0.005s
ok github.com/carlosframework/platform/internal/infraview 0.005s
ok github.com/carlosframework/platform/internal/instancectl 6.898s
ok github.com/carlosframework/platform/internal/instanceexec 2.809s
ok github.com/carlosframework/platform/internal/instancefs 0.020s
ok github.com/carlosframework/platform/internal/instancekey 0.005s
ok github.com/carlosframework/platform/internal/instanceupdates 0.008s
ok github.com/carlosframework/platform/internal/layout 1.783s
ok github.com/carlosframework/platform/internal/lbhealth 0.010s
ok github.com/carlosframework/platform/internal/lease 10.277s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.529s
ok github.com/carlosframework/platform/internal/ledger 0.033s
ok github.com/carlosframework/platform/internal/mail 0.224s
ok github.com/carlosframework/platform/internal/maintenance 0.079s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.567s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.107s
ok github.com/carlosframework/platform/internal/metrics 0.011s
ok github.com/carlosframework/platform/internal/moves 22.336s
ok github.com/carlosframework/platform/internal/moves/archive 0.996s
ok github.com/carlosframework/platform/internal/objstore 0.306s
ok github.com/carlosframework/platform/internal/ops 0.262s
ok github.com/carlosframework/platform/internal/park 0.273s
ok github.com/carlosframework/platform/internal/plans 0.009s
ok github.com/carlosframework/platform/internal/privatefile 0.007s
2026/10/24 20:55:52 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 20:55:52 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:55:52 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:55:52 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 20:55:52 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:55:52 provision: an app's records could not be listed: they are skipped
2026/10/24 20:55:52 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:55:52 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 20:55:52 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 20:55:52 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:55:52 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:55:52 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:55:52 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:55:52 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:55:52 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 20:55:52 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:55:52 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:52 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:55:52 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:55:52 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:55:53 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:55:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:55:53 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:55:53 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:55:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:55:53 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 20:55:53 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:53 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:55:53 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:53 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:53 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 20:55:53 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 20:55:53 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 20:55:53 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 20:55:53 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:53 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:53 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (0.74s)
plan9_b8_census_test.go:372: record-write census rejected 2 site(s); update the explicit inventory only after reviewing the source:
changed site internal/instanceexec/exit.go:RunExit: source differs from the reviewed digest
changed site internal/ops/litestream.go:FleetDBsWithSkipped: source differs from the reviewed digest
2026/10/24 20:55:54 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 20:55:55 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 20:55:55 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 20:55:55 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 20:55:55 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:55:55 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:55:55 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:55:55 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:55:55 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 20:55:55 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 20:55:55 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 20:55:55 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 20:55:55 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 20:55:55 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 20:55:55 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:55 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:55:55 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 20:55:55 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 20:55:55 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:55 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 20:55:55 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:55 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 20:55:55 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 20:55:55 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:55 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 20:55:55 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:55 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 20:55:55 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:55 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:55 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 20:55:55 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 20:55:55 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 20:55:55 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:55 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:55:55 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 20:55:55 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 20:55:55 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 20:55:55 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 20:55:55 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 20:55:55 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 20:55:55 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 20:55:55 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:55 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 20:55:55 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 20:55:55 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 20:55:55 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 20:55:55 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 20:55:55 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 20:55:55 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:55 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 20:55:56 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 20:55:56 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:56 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 20:55:56 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 20:55:56 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:55:56 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:55:56 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 20:55:56 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:55:56 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:55:56 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:55:56 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 20:55:56 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 20:55:56 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 20:55:56 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:55:56 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 20:55:56 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 20:55:56 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 20:55:56 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 20:55:56 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:55:56 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:56 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:55:57 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:57 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:57 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:55:57 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:57 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:55:57 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 20:55:57 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 20:55:57 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 20:55:57 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 20:55:57 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:55:57 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:55:57 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:55:57 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 20:55:57 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:55:57 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 20:55:57 provision: a.shop.example.com: swapped for import sd_juarxms4x2lgx4jzqyj5nih2du (the original route is journaled)
2026/10/24 20:55:57 provision: a.shop.example.com: import sd_juarxms4x2lgx4jzqyj5nih2du committed: its gate is cleared and its journal row dropped
2026/10/24 20:55:57 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_etkjs4be7jervzmr2o72tzveaa rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_etkjs4be7jervzmr2o72tzveaa: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_etkjs4be7jervzmr2o72tzveaa is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_ypuslmktc725qtg7agvuohemci rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_ypuslmktc725qtg7agvuohemci: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_ypuslmktc725qtg7agvuohemci is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_4obylbvyl5fqtwemla6a6twfxi
2026/10/24 20:55:58 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 20:55:58 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 20:55:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 20:55:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 20:55:58 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_skpuecg5xhrx723mzyfgrplngm cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_e3gbljgtjaeyi5zsrudluyf6ja cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_fsnsxbtouniogqdbv4ajtjcoxy is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_yi6n4glu2h7q5x6kz56tz3zqye is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_ggdq4jfvymwnvuujb46ocx2qhu is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:55:58 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 20:55:58 provision: b.shop.example.com: import sd_nw75ghuirx2xku46jpvcu2de5i left alone this pass: the import journal could not be read
2026/10/24 20:55:58 provision: c.shop.example.com: import sd_dkxnpasx6vtjeggqxm4rkd4v6y left alone this pass: the import journal could not be read
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_kim6fofehbhzxtc34klafii4di committed: its gate is cleared and its journal row dropped
2026/10/24 20:55:58 provision: a.shop.example.com: not converging import sd_77nusbhexh5l4zuqsl2atsu47m: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 20:55:58 provision: a.shop.example.com: not converging import sd_aewchxnewn2gdmq357soif372a: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 20:55:58 provision: a.shop.example.com: not converging import sd_7phbawoegb2uossf5sojyjudca: it is placed but not committed, and the row on this box has no import gate
2026/10/24 20:55:58 provision: a.shop.example.com: not converging import sd_zcq26cwyhodtkwxti7ubgb2mue: it is placed but not committed, and the row on this box has no import gate
2026/10/24 20:55:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 20:55:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_3kclquss7ygwftp4jzbfgyxnb4 committed: its gate is cleared and its journal row dropped
2026/10/24 20:55:58 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:58 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 20:55:58 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 20:55:58 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_yp2o3dzmbtuwnrrjvcsu6iwmrm: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_yp2o3dzmbtuwnrrjvcsu6iwmrm rolled back (no box had swapped it)
2026/10/24 20:55:58 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 20:55:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:58 provision: removed instance route acme.go.tito.io (its seed sd_pkvgj5ackc2xdcfi42hmqwq2we was aborted); the next pass rolls it back
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_pkvgj5ackc2xdcfi42hmqwq2we: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_pkvgj5ackc2xdcfi42hmqwq2we rolled back (no box had swapped it)
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_ffwpxs7dj4xj2avxucvxpee4vu has a row this pass may not remove; leaving the seed aborted
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_7kugwzy7he3s2xdr57mua6aktu rolled back (no box had swapped it)
2026/10/24 20:55:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:58 provision: removed instance route acme.go.tito.io (its seed sd_rhlhrvcwhn7gjrsxkvm4xefikq was aborted); the next pass rolls it back
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_rhlhrvcwhn7gjrsxkvm4xefikq has no route on this box: its local data is removed
2026/10/24 20:55:58 provision: acme.go.tito.io: aborted seed sd_rhlhrvcwhn7gjrsxkvm4xefikq rolled back (no box had swapped it)
2026/10/24 20:55:58 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_dcy2h2w5w3z747gypiphi6tgxq (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_dcy2h2w5w3z747gypiphi6tgxq rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_dcy2h2w5w3z747gypiphi6tgxq is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_kufmnu22w4cove7qa6ufjjzlu4 (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_kufmnu22w4cove7qa6ufjjzlu4 rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_kufmnu22w4cove7qa6ufjjzlu4 is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_rgp5i5sc7f2msn2pghic2drari (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_rgp5i5sc7f2msn2pghic2drari rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_rgp5i5sc7f2msn2pghic2drari: the host's replica is left alone: its record no longer names this seed
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_rgp5i5sc7f2msn2pghic2drari is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_hpmjwy2hzuo74iukj6ocmfmtau (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_hpmjwy2hzuo74iukj6ocmfmtau rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_hpmjwy2hzuo74iukj6ocmfmtau rolled back to its original route
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_hpmjwy2hzuo74iukj6ocmfmtau is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_addm7zhrzzzaye5ghzpuykonl4 (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_addm7zhrzzzaye5ghzpuykonl4 rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_dsihevxvlmqdevjjkxkbfp2ani (the original route is journaled)
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_dsihevxvlmqdevjjkxkbfp2ani is cleaned up and its journal row retired
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_oftsl4yd6cu4mpx5tlyu5s4dbq (the original route is journaled)
2026/10/24 20:55:58 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_oftsl4yd6cu4mpx5tlyu5s4dbq left alone this pass: its record could not be read
2026/10/24 20:55:58 provision: a.shop.example.com: swapped for import sd_rx7zmxycrapzcuribkkjh3z52y (the original route is journaled)
2026/10/24 20:55:58 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 20:55:58 provision: a.shop.example.com: import sd_rx7zmxycrapzcuribkkjh3z52y left alone this pass: its record could not be read
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_bofwbmdyr63mcebzm5xjwxrz2u (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_bofwbmdyr63mcebzm5xjwxrz2u: close the import gate: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_bofwbmdyr63mcebzm5xjwxrz2u rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_bofwbmdyr63mcebzm5xjwxrz2u is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_ylfbwbodkzilmd6bl7qdsaf7nm (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ylfbwbodkzilmd6bl7qdsaf7nm: close the import gate: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ylfbwbodkzilmd6bl7qdsaf7nm rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ylfbwbodkzilmd6bl7qdsaf7nm is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_rj3nfbthjwyvnm5yjk6twne2qm (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_rj3nfbthjwyvnm5yjk6twne2qm rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_rj3nfbthjwyvnm5yjk6twne2qm rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_rj3nfbthjwyvnm5yjk6twne2qm is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_i2r7nyzndt4bea7gsx6mrm35ei (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_i2r7nyzndt4bea7gsx6mrm35ei rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_i2r7nyzndt4bea7gsx6mrm35ei is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_kzvvkgszuzaidsry53ijc75q6u (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_kzvvkgszuzaidsry53ijc75q6u: retire its instance key: injected failure at retire-key
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_kzvvkgszuzaidsry53ijc75q6u rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_kzvvkgszuzaidsry53ijc75q6u is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_gbbpyvnq6ijydev6vyshizkuui (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_gbbpyvnq6ijydev6vyshizkuui rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_gbbpyvnq6ijydev6vyshizkuui is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_sstalqlnipjgs7hqozzgixvliu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_sstalqlnipjgs7hqozzgixvliu: remove its local data: injected failure at remove-local
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_sstalqlnipjgs7hqozzgixvliu rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_sstalqlnipjgs7hqozzgixvliu is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_x3slvipsynzmxgo7fdirkj2jnu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_x3slvipsynzmxgo7fdirkj2jnu rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_x3slvipsynzmxgo7fdirkj2jnu is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_qluffcltddo37telifggyhw5wm (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_qluffcltddo37telifggyhw5wm: remove its local data: injected failure at remove-local-done
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_qluffcltddo37telifggyhw5wm rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_qluffcltddo37telifggyhw5wm is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_dsvyomlcubehlrbewg3dyocjpu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_dsvyomlcubehlrbewg3dyocjpu rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_dsvyomlcubehlrbewg3dyocjpu is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_2eoqqzo3xdfvv22thjqnsrfsnm (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_2eoqqzo3xdfvv22thjqnsrfsnm: restore the original route: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_2eoqqzo3xdfvv22thjqnsrfsnm rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_2eoqqzo3xdfvv22thjqnsrfsnm is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_wb3yigi4omt6fk3wwu5gsihb7u (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_wb3yigi4omt6fk3wwu5gsihb7u: restore the original route: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_wb3yigi4omt6fk3wwu5gsihb7u rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_wb3yigi4omt6fk3wwu5gsihb7u is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_ahj6ov4x6tehkxqzlw37t46zm4 (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ahj6ov4x6tehkxqzlw37t46zm4 is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_x5okutmkxq37qsadfawgp3vczy (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_x5okutmkxq37qsadfawgp3vczy rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_x5okutmkxq37qsadfawgp3vczy: delete its snapshot: injected failure at delete-snapshot
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_x5okutmkxq37qsadfawgp3vczy is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_qskyqmvan7rhtsayjhq6iwurm4 (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_qskyqmvan7rhtsayjhq6iwurm4 rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_qskyqmvan7rhtsayjhq6iwurm4 is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_yjmsuclz4th66akwtewuiqa4xe (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_yjmsuclz4th66akwtewuiqa4xe rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_yjmsuclz4th66akwtewuiqa4xe: delete the host's replica: injected failure at delete-replica
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_yjmsuclz4th66akwtewuiqa4xe is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_k67vsz6qikrjedruzowgpsls5m (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_k67vsz6qikrjedruzowgpsls5m rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_k67vsz6qikrjedruzowgpsls5m is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_ezt4qxhysgvrmlphhyhftpsouu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ezt4qxhysgvrmlphhyhftpsouu rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ezt4qxhysgvrmlphhyhftpsouu: retire its journal row: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ezt4qxhysgvrmlphhyhftpsouu is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_36p2qaqjeqzyzcx446toarloci (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_36p2qaqjeqzyzcx446toarloci rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_36p2qaqjeqzyzcx446toarloci: retire its journal row: constraint failed: injected (1811)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_36p2qaqjeqzyzcx446toarloci is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_tj6gbow4jfwgz222c2ka4yqccu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_tj6gbow4jfwgz222c2ka4yqccu rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_tj6gbow4jfwgz222c2ka4yqccu is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: mark import sd_tj6gbow4jfwgz222c2ka4yqccu rolled back: injected failure at cas
2026/10/24 20:55:59 provision: a.shop.example.com: aborted seed sd_tj6gbow4jfwgz222c2ka4yqccu rolled back (no box had swapped it)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_3hplwrtesrxotkmbtatxrftf5a (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_3hplwrtesrxotkmbtatxrftf5a rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_3hplwrtesrxotkmbtatxrftf5a is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: aborted seed sd_3hplwrtesrxotkmbtatxrftf5a rolled back (no box had swapped it)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_xdmjy75pn4tfqwtqxxyh3nn4ce (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_xdmjy75pn4tfqwtqxxyh3nn4ce rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_xdmjy75pn4tfqwtqxxyh3nn4ce rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_xdmjy75pn4tfqwtqxxyh3nn4ce rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_xdmjy75pn4tfqwtqxxyh3nn4ce is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_p3yszwl5scnrecwlxltaq4m2si (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_p3yszwl5scnrecwlxltaq4m2si rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_p3yszwl5scnrecwlxltaq4m2si is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_v6hd6apasvdplsajoup5hjrcpq (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_v6hd6apasvdplsajoup5hjrcpq rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_v6hd6apasvdplsajoup5hjrcpq is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_ncklt65jjodz2qjpfsvxcdebmq (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ncklt65jjodz2qjpfsvxcdebmq rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_ncklt65jjodz2qjpfsvxcdebmq is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_wyksp4s2ygv523j4e2hbacnoh4 (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_wyksp4s2ygv523j4e2hbacnoh4 rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_wyksp4s2ygv523j4e2hbacnoh4 is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_4nmxd554f6tf3iws3tulrjdv3q (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_4nmxd554f6tf3iws3tulrjdv3q rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_4nmxd554f6tf3iws3tulrjdv3q is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_yuyolmhme632ovbtdj55jqkwei (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_yuyolmhme632ovbtdj55jqkwei rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_yuyolmhme632ovbtdj55jqkwei is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_efd32g3evq4wtzhzdogf6u3zyi (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_efd32g3evq4wtzhzdogf6u3zyi rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_efd32g3evq4wtzhzdogf6u3zyi is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_r3ispuknm3l6ofsscwzto25imy (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_r3ispuknm3l6ofsscwzto25imy has no route on this box: its local data is removed and no route is put back
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_r3ispuknm3l6ofsscwzto25imy is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_2czgur4ulisypdafdfyk7oml3i (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_wij6g4zjwwwbkxmn3lvltbfidy (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_w6vm7hslp6wypvzxovwwbdqqqa (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_w6vm7hslp6wypvzxovwwbdqqqa cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_oj76wdpzxgohbognzhiugejuge (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_oj76wdpzxgohbognzhiugejuge is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 20:55:59 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 20:55:59 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_xp5x6fnhce5pxd2zappvktrntu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_pyppj3bi54za3tzqhn4q3xfmke (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_rl6pg2k72dcojzeffwiufowrg4 (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_lwfl6tkzbedw5zvubuqvrsua3u (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_v3jteye4saxfmjx7wj4vsanulm (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_v3jteye4saxfmjx7wj4vsanulm rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_j3bhvtwo6u7gh7ieauxxyhkb2u (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_j3bhvtwo6u7gh7ieauxxyhkb2u has no route on this box: its local data is removed and no route is put back
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_j3bhvtwo6u7gh7ieauxxyhkb2u is cleaned up and its journal row retired
2026/10/24 20:55:59 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_peqmnnqmydnrxjml44yis4vlbu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_peqmnnqmydnrxjml44yis4vlbu: remove its local data: injected failure at remove-local
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_peqmnnqmydnrxjml44yis4vlbu is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_n2msnswvvtmbiven3z2dol3u3y (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_n2msnswvvtmbiven3z2dol3u3y: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_etf5cgqmkwed7ugmc3lr6y4guu (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_etf5cgqmkwed7ugmc3lr6y4guu rollback deferred: no fence generation: the generation source answered 0
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_3us26rfav6rs4r2rsm3khjo2cy (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_3us26rfav6rs4r2rsm3khjo2cy rollback deferred: no fence generation: registry busy
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_dqfyrhmjfavfjeolz4tew6gnam (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_onjptmuvanrseoi3zszjxejns4 (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_wlfpszkf4cpwwistpbxzzdihem (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 20:55:59 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 20:55:59 provision: a.shop.example.com: swapped for import sd_adivfiiksfm6x5fjbilo22nsyq (the original route is journaled)
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_adivfiiksfm6x5fjbilo22nsyq rolled back to its original route
2026/10/24 20:55:59 provision: a.shop.example.com: import sd_adivfiiksfm6x5fjbilo22nsyq is cleaned up and its journal row retired
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_j7yh3tx64fxoqsg4epnsy22kxi (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_j7yh3tx64fxoqsg4epnsy22kxi rolled back to its original route
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_j7yh3tx64fxoqsg4epnsy22kxi is cleaned up and its journal row retired
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_5eoe5r7g4hrr3g3pz33cno2zyu (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_5eoe5r7g4hrr3g3pz33cno2zyu rolled back to its original route
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_5eoe5r7g4hrr3g3pz33cno2zyu is cleaned up and its journal row retired
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_emzjqdtmmaov4nde2ivyipwiee (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_emzjqdtmmaov4nde2ivyipwiee: retire its instance key: the env file is a symlink
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_emzjqdtmmaov4nde2ivyipwiee rolled back to its original route
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_emzjqdtmmaov4nde2ivyipwiee is cleaned up and its journal row retired
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_pibpceoescahcwrtxr7himxsm4 (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_vzmnwcjoridtvcjbajux6yi2sm (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_rh5d4em5bw6f6vkmsw3h5k4hgi refused: there is no route at a.shop.example.com to replace
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_c3u6ndi4vfog2h4rhsbonczm6m refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_zeerl6ete3dh6sft7vyykaur7q refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_o4haap7papnqepm2um7tqbmkwy refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_jjowg7nhzkf3rtgjzj4fdbe7ne refused: the route belongs to app other
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_clklr24ct7xkhf5gv6epc5ppbi refused: the route is filed under another account
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_opz3xdemqgecu4lax33eii3p4y refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_lf7i3tada2icrbgiaeabwulpa4 refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_spdyjos7k6vxlozfvxrjjffm5u refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_rzma7qxodurwo35ho5jdtchtay refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_xm64dn2dl5dlyfmd4kvlbm2lma refused: the route is not an address route this import can take over (socket)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_d7z3b2wsvzl4yvfm3ztrse4iva refused: the route is not an address route this import can take over (no address)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_leib6e6kr5l56y3bcjvsshkxfi refused: the route is not an address route this import can take over (provisioned)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_qfzgjqqsegdsdpwkobntyjgfam refused: the route is not an address route this import can take over (kind static)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_sbyewjxrslnde3y2tyout7th4y refused: the route is not an address route this import can take over (backing)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_s52qrzl4rj3eorgjqozd7eyw5y refused: the route is not an address route this import can take over (tunnel)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_xn5ayokhjq3ptundtpphbh7h2y refused: the route is not an address route this import can take over (caps)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_46nkrt4rh4kjbky2xgldltb6i4 refused: the route is not an address route this import can take over (platform)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_bxhrz2v62cdqqzuelrzp4paely refused: the route is not an address route this import can take over (canary)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_j7bud7krhcnqhj2u7bvn6zdgse refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_4yxjtnlyl3ykke77irgn2ovtlq refused: the route is not an address route this import can take over (region)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_7f65ouga2d2mjoosbrmy2ppq3m refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_37bmu33pp65tftdzl7vvc5gfwe refused: the route is not an address route this import can take over (hibernate)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_kpybkyvuj6ddh4nxlq6b2qcswq refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_rh3uffeov4vln5cliq47sztsma refused: this box does not run instances nested
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_objrxf6jirl5u6gxgovokujqa4 refused: this box does not run instances nested
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_l64vgd4h6jv23he2qcgvundjrq refused: claim control is on for this box
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_jmt4zy3rkajuirpznsvkkxbd4a (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_ussrww25mncldsnlb4dwhlosoe (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 20:56:00 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_7cxcctl3hflevh2kehs3hmzpc4: its verify hash is not a sha256
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_zjpdq43y6zv7sz427wbnuw7k2y: its verify hash is not a sha256
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_ohctgdy5f6d2oej6v3yayusea4: its verify hash is not a sha256
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 20:56:00 provision: A.shop.example.com/..: not swapping for import sd_6z7ndvy627tfaefwgpcwxk26ti: its host or seed id is malformed (seeds: not a host)
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_yoxx6l6cue7py5irzc23cztjjy: its record does not describe a nested, hibernating instance
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_34llnaojvn75uyvdd4yizoc3ce: its record does not describe a nested, hibernating instance
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_67em7cromjm65q46gdng5qxlsu: its record does not describe a nested, hibernating instance
2026/10/24 20:56:00 provision: a.shop.example.com: not swapping for import sd_4vksu7t2o3m5eb2rowrrwfd2ka: its record does not describe a nested, hibernating instance
2026/10/24 20:56:00 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_n2jyo5dnur7us5qrbnmkmtnpqm (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_hcdaboxqokxq4v2iwrtkdnrd5m (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_qzprlsiezde7qs4tdrwqrpyksu (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 20:56:00 provision: dry-run: would consider swapping a.shop.example.com for import sd_f6f6xfilspryr7llpvi76apwre
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_7tmeddnvs7y5njj6rk2qglnve4 rolled back to its original route
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_7tmeddnvs7y5njj6rk2qglnve4: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_7tmeddnvs7y5njj6rk2qglnve4 is cleaned up and its journal row retired
2026/10/24 20:56:00 provision: a.shop.example.com: import sd_rtxfgabh3qpozxejo7xj7f7spq left alone: the import journal holds seed sd_uw64cijkw6leiucbmjhyi237sq at swapped
2026/10/24 20:56:00 provision: a.shop.example.com: not minting import sd_x7t2ghfemungdgm5lfdwf442ey: it is swapped but not committed, and only its swap makes its row
2026/10/24 20:56:00 provision: a.shop.example.com: not minting import sd_opq3sjyrljz57vbpl75aptma4q: it is placed but not committed, and only its swap makes its row
2026/10/24 20:56:00 provision: a.shop.example.com: swapped for import sd_3hbjgy3yp3weuxhg7usqf3zfy4 (the original route is journaled)
2026/10/24 20:56:00 provision: a.shop.example.com: not minting import sd_3hbjgy3yp3weuxhg7usqf3zfy4: it is swapped but not committed, and only its swap makes its row
2026/10/24 20:56:00 provision: a.shop.example.com: not minting import sd_3hbjgy3yp3weuxhg7usqf3zfy4: it is placed but not committed, and only its swap makes its row
2026/10/24 20:56:00 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:56:00 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:56:00 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:00 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:00 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 20:56:00 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:00 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 20:56:00 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:00 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 20:56:00 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 20:56:00 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 20:56:00 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 20:56:00 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 20:56:00 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 20:56:00 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 20:56:00 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 20:56:00 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 20:56:00 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:56:01 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 20:56:01 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 20:56:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 20:56:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:56:01 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 20:56:01 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:56:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:56:01 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 20:56:01 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:56:01 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:56:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:56:01 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 20:56:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:56:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:56:01 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:56:01 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:56:01 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 20:56:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:56:01 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:56:01 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 20:56:01 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:56:01 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:56:01 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:56:01 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 20:56:01 provision: record view: change index unusable (get refused): walking
2026/10/24 20:56:01 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 20:56:01 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 20:56:01 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 20:56:01 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:56:01 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:56:01 provision: an app's records could not be listed: they are skipped
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:56:01 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:56:01 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 20:56:01 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:56:01 provision: removed retired instance route one.example.com (record gone)
2026/10/24 20:56:01 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:56:01 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:56:01 provision: removed retired instance route one.example.com (record gone)
2026/10/24 20:56:01 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 20:56:01 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 20:56:01 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 20:56:01 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 20:56:01 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:56:01 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 20:56:01 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (14.55s)
plan14_root_reads_test.go:600: root read census: 15 discrepancies
changed root read internal/instancectl/server.go:parseRequest; review class, task, sites and digest
changed root read internal/instancectl/server.go:type Server; review class, task, sites and digest
changed root read internal/ops/hibernate_verify.go:HibernatingHosts; review class, task, sites and digest
changed root read internal/ops/litestream.go:FleetDBsWithSkipped; review class, task, sites and digest
changed root read internal/ops/ops.go:runLitestream; review class, task, sites and digest
changed root source envelope internal/instancectl
changed root source envelope internal/instanceexec
changed root source envelope internal/ops
unclassified root read internal/ops/litestream.go:ActivatorReplicated registry:Backing@59,registry:DBPath@97,registry:Hibernate@117
unclassified root read internal/ops/litestream.go:ActivatorReplicatedPaths registry:DBPath@257,whole:Route@178,whole:Route@212
changed legacy declaration internal/instancectl/server.go:parseRequest
changed legacy declaration internal/instancectl/server.go:type Server
changed legacy declaration internal/ops/hibernate_verify.go:HibernatingHosts
changed legacy declaration internal/ops/litestream.go:FleetDBsWithSkipped
changed legacy declaration internal/ops/ops.go:runLitestream
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 23.193s
ok github.com/carlosframework/platform/internal/publicrelease 0.010s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.551s
ok github.com/carlosframework/platform/internal/relay 7.231s
ok github.com/carlosframework/platform/internal/release 25.679s
ok github.com/carlosframework/platform/internal/rollout 3.782s
ok github.com/carlosframework/platform/internal/rolloutwords 0.004s
ok github.com/carlosframework/platform/internal/rootaws 0.009s
ok github.com/carlosframework/platform/internal/rootparticipation 0.060s
ok github.com/carlosframework/platform/internal/rootstate 0.027s
ok github.com/carlosframework/platform/internal/runs 0.619s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.599s
ok github.com/carlosframework/platform/internal/seeds 0.015s
ok github.com/carlosframework/platform/internal/sim/engines 2.560s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.231s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.647s
ok github.com/carlosframework/platform/internal/status 0.077s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.281s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.005s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.002s
ok github.com/carlosframework/platform/internal/updatewords 0.006s
ok github.com/carlosframework/platform/internal/usage 0.008s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.695s
FAIL
=== internal_console@0 (82 s)
ok github.com/carlosframework/platform/internal/console 80.501s
=== internal_console@1 (42 s)
ok github.com/carlosframework/platform/internal/console 41.297s
=== internal_console@2 (41 s)
ok github.com/carlosframework/platform/internal/console 39.756s
=== internal_console@3 (42 s)
ok github.com/carlosframework/platform/internal/console 40.487s
=== internal_console@4 (42 s)
ok github.com/carlosframework/platform/internal/console 40.666s
=== internal_console@5 (41 s)
ok github.com/carlosframework/platform/internal/console 39.519s
=== internal_console@6 (35 s)
ok github.com/carlosframework/platform/internal/console 34.364s
=== internal_console@7 (46 s)
ok github.com/carlosframework/platform/internal/console 44.797s
=== root@0 (109 s)
ok github.com/carlosframework/platform 107.770s
=== root@1 (83 s)
ok github.com/carlosframework/platform 82.103s
=== root@2 (92 s)
using http://127.0.0.1:32907 as a@keymail.dev
using http://127.0.0.1:36779 as vicky@keymail.dev
titogo (staging):
KEY VALUE
PORT 8080
--- FAIL: TestRunBuiltCopyIsApplied (0.12s)
approved_run_copy_test.go:139: RUN-C13: internal/instanceexec/exec.go does not contain the approved text "instance-exec: role <role> is not run, seed, sidecar or cmd"
approved_run_copy_test.go:139: RUN-C14: internal/instanceexec/exit.go does not contain the approved text "instance-exit: role <role> is not run, seed, sidecar or cmd"
2026/10/24 20:56:05 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:05 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A41899": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:05 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:05 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A41899": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:05 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:05 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:05 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A45057": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:05 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:07 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:07 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A40233": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:07 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:07 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:07 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A45545": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:07 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:09 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:09 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A35399": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:09 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:11 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:11 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A43625": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:11 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
using http://127.0.0.1:45213 as paul@keymail.dev
account: ops (bab)
2026/10/24 20:56:14 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A45213": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:14 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
set 1 env var(s) for titogo (default)
using http://127.0.0.1:45213 as paul@keymail.dev
account: ops (bab)
set 1 secret(s) for titogo (default)
account: ops (bab)
account: ops (bab)
using http://127.0.0.1:45213 as paul@keymail.dev
account: ops (bab)
config sync requested for titogo the files land within seconds; programs carlos owns restart themselves, and anything else keeps its old values until you restart it
2026/10/24 20:56:14 console: delivery to mallory@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=mallory%40keymail.dev&from=console%40127.0.0.1%3A45213": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:14 console: could not notify mallory@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:16 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:56:16 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 20:56:16 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:56:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:56:17 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 20:56:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:56:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:17 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A33815": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:17 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 20:56:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:56:17 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 20:56:17 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:56:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:56:18 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 20:56:18 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:56:19 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:56:19 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A46157": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:56:19 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:00 console: db read dbr_gk6p3qoabk67zgrkcxhm6eczm4: relayed 1 chunks (101 bytes) in 41.03s, ended done
2026/10/24 20:57:00 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:57:00 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 20:57:00 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:57:01 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:57:01 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 20:57:01 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:57:01 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:57:01 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A38263": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:01 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:02 console: db read dbr_6ljax5twyqpva3zw7zmhixtgxe: relayed 1 chunks (1070 bytes) in 1.001s, ended done
2026/10/24 20:57:03 console: db read dbr_6bsavm5qgph7yneffoksorzske: relayed 1 chunks (805 bytes) in 1.001s, ended done
2026/10/24 20:57:03 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:57:03 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=708
2026/10/24 20:57:03 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:57:04 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:57:04 INFO ltx file uploaded system=store db=shop.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=265
2026/10/24 20:57:04 INFO replica sync system=store db=shop.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:57:04 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:57:04 console: delivery to member@example.com: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=member%40example.com&from=console%40127.0.0.1%3A43597": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:04 console: could not notify member@example.com of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:05 console: db read dbr_aspfdnahcfpzkbeceel3pz2a7a: relayed 2 chunks (8386 bytes) in 1.044s, ended client gone
using http://127.0.0.1:38015 as paul@keymail.dev
account: ops (bab)
attached tickets.example.com -> default.go.tito.io
DNS: pointing
using http://127.0.0.1:38015 as paul@keymail.dev
account: ops (bab)
attached carloku.example.com -> default.go.tito.io
2026/10/24 20:57:06 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:57:06 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A35727": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:06 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:06 console: delivery to rival@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=rival%40keymail.dev&from=console%40127.0.0.1%3A35727": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:06 console: could not notify rival@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:15 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A46081": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:15 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/24 20:57:15 console: delivery to paul@keymail.dev: keymail lookup unreachable, trying email: keymail: lookup unreachable: Get "http://keymail.invalid/api/federation/lookup?addr=paul%40keymail.dev&from=console%40127.0.0.1%3A34375": dial tcp: lookup keymail.invalid on 127.0.0.53:53: no such host
2026/10/24 20:57:15 console: could not notify paul@keymail.dev of a CLI token issue: console: no email sender is configured
2026/10/24 20:57:16 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 20:57:16 domains: minted platform domain blog.mdbbndjqx.oncarlos.test (upstream blog.kass.test)
2026/10/24 20:57:16 domains: minted platform domain blog.gdpkwhwrw.oncarlos.test (upstream blog.woodstar.test)
2026/10/24 20:57:16 domains: minted platform domain oldsite.bab.oncarlos.test (upstream old.test)
2026/10/24 20:57:16 edge: static oldsite/stable: resolve channel: dirstore: apps/bab/oldsite/channels/stable.json: file does not exist — if this deployment predates account-scoped bucket keys its objects are still at apps/oldsite/, which the runtime no longer reads; move them once with: carlos accounts migrate --app oldsite --from legacy --to bab
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/24 20:57:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/24 20:57:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
FAIL
FAIL github.com/carlosframework/platform 90.774s
FAIL
FAILED: rest root@2
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/implementation/census-claims-sidecar-isolation.txt b/implementation/census-claims-sidecar-isolation.txt| index babd959e4..73431fbaf 100644 |
| --- a/implementation/census-claims-sidecar-isolation.txt |
| +++ b/implementation/census-claims-sidecar-isolation.txt |
| @@ -1,3 +1,5 @@ |
| internal/ops/ops.go:runLitestream shrink guard accepts and names a drop because the row is ActivatorReplicated (CoverageShortfallExcept); other drops still refuse |
| internal/ops/hibernate_verify.go:HibernatingHosts selects by ActivatorReplicated, so an always-on worker is drilled; reads the whole route through the predicate |
| internal/ops/litestream.go:FleetDBsWithSkipped the activator-owned exclusion tests ActivatorReplicated (workers included) instead of Hibernate alone; reads the whole route through the predicate |
| +internal/instancectl/server.go:parseRequest admits the worker role on /start, /stop and /reset and lets a worker start name its staged exe (validated by host, D1/D5); a worker with a generation is still refused; the role error names worker; no new read |
| +internal/instancectl/server.go:type Server the #292 start fence maps (legacyStarts, legacyStopping, legacyBarrier) are keyed by roleHost{role, host} instead of host, so the worker unit is fenced apart from the run unit (D19); no new read |
diff --git a/internal/instancectl/legacystart.go b/internal/instancectl/legacystart.go| index 35355f2b6..d6c823039 100644 |
| --- a/internal/instancectl/legacystart.go |
| +++ b/internal/instancectl/legacystart.go |
| @@ -8,29 +8,34 @@ import ( |
| "time" |
| ) |
| -// Legacy run start attempts (#292, Astra round 2 P1, the owner's ruling). |
| +// Legacy start attempts (#292, Astra round 2 P1, the owner's ruling; |
| +// widened to the worker role by sidecar isolation D19, #370). |
| // |
| // The agent proves an aborted start's unit stopped through /stop before it |
| // gives the lease back. Cancelling its /start does not join this side's |
| // handler, which could still submit the systemd start after that /stop had |
| // answered, if it had not yet seen the disconnect; systemd's replacement of |
| // a queued start by a stop only covers jobs that already exist. So, for the |
| -// original run unit (carlos-instance@, generation 0) only, start and stop |
| -// are ordered per host here: |
| +// original run unit (carlos-instance@, generation 0) and the worker unit |
| +// (carlos-instance-worker@, which never has a generation), start and stop |
| +// are ordered here, keyed by (role, host): |
| // |
| // - every /start registers an attempt with a context /stop can cancel, |
| // and checks that context immediately before it submits the start; |
| -// - a /stop fences the host: new starts are refused while it runs, every |
| -// attempt in flight is cancelled and joined (bounded), and only then is |
| -// the unit stopped, which replaces any job a joined attempt submitted; |
| +// - a /stop fences its (role, host): new starts of that unit are refused |
| +// while it runs, every attempt in flight is cancelled and joined |
| +// (bounded), and only then is the unit stopped, which replaces any job |
| +// a joined attempt submitted; |
| // - a start whose request was accepted at or before the moment a stop |
| -// fences the host (the accepted-connection watermark, connSeq) is an |
| +// fences the unit (the accepted-connection watermark, connSeq) is an |
| // earlier request, and is refused, however late its handler registers; |
| // one accepted after it runs normally. Serve answers one request per |
| // connection, so a connection's accept order is its request's. |
| // |
| -// Once /stop has answered, no start from an earlier request is in flight or |
| -// can be submitted afterwards. Generation units, seeds, sidecars and the |
| +// Once /stop has answered, no start of that unit from an earlier request is |
| +// in flight or can be submitted afterwards. The key is the role and the |
| +// host together (D19): a worker's stop never fences the same host's run |
| +// unit, nor the reverse. Generation units, seeds, legacy sidecars and the |
| // claim verbs are untouched. |
| // legacyJoinBudget bounds how long a /stop waits for the start attempts it |
| @@ -38,7 +43,18 @@ import ( |
| // the stop answers an error and the agent keeps the lease. |
| const legacyJoinBudget = 10 * time.Second |
| -// legacyAttempt is one in-flight legacy run start. |
| +// roleHost keys the fence: one unit, the role's template instance for the |
| +// host. Only RoleRun and RoleWorker are ever fenced (fencedRole). |
| +type roleHost struct { |
| + role Role |
| + host string |
| +} |
| + |
| +// fencedRole reports whether a role's start and stop are ordered here: the |
| +// original run unit (#292) and the worker unit (D19). |
| +func fencedRole(role Role) bool { return role == RoleRun || role == RoleWorker } |
| + |
| +// legacyAttempt is one in-flight legacy start. |
| type legacyAttempt struct { |
| cancel context.CancelFunc |
| done chan struct{} |
| @@ -64,33 +80,34 @@ func requestSeq(ctx context.Context) uint64 { |
| return n |
| } |
| -// errLegacyFenced refuses a start while a stop of its host runs, or one |
| -// whose request came before a stop of its host. |
| -var errLegacyFenced = errors.New("a stop of this host's run unit has run since this start was asked for") |
| +// errLegacyFenced refuses a start while a stop of its unit runs, or one |
| +// whose request came before a stop of its unit. |
| +var errLegacyFenced = errors.New("a stop of this unit has run since this start was asked for") |
| -// beginLegacyStart registers a legacy run start of host for a request of |
| -// accept order seq. Its context is parent's, cancellable by a /stop of the |
| -// host; finish must be called when the handler returns. |
| -func (s *Server) beginLegacyStart(parent context.Context, host string, seq uint64) (context.Context, func(), error) { |
| +// beginLegacyStart registers a start of role's unit for host, for a request |
| +// of accept order seq. Its context is parent's, cancellable by a /stop of |
| +// the same role and host; finish must be called when the handler returns. |
| +func (s *Server) beginLegacyStart(parent context.Context, role Role, host string, seq uint64) (context.Context, func(), error) { |
| + key := roleHost{role, host} |
| s.legacyMu.Lock() |
| defer s.legacyMu.Unlock() |
| - if s.legacyStopping[host] > 0 || (seq != 0 && seq <= s.legacyBarrier[host]) { |
| - return nil, nil, errLegacyFenced |
| + if s.legacyStopping[key] > 0 || (seq != 0 && seq <= s.legacyBarrier[key]) { |
| + return nil, nil, fmt.Errorf("%s unit of %s: %w", role, host, errLegacyFenced) |
| } |
| ctx, cancel := context.WithCancel(parent) |
| a := &legacyAttempt{cancel: cancel, done: make(chan struct{})} |
| if s.legacyStarts == nil { |
| - s.legacyStarts = make(map[string]map[*legacyAttempt]struct{}) |
| + s.legacyStarts = make(map[roleHost]map[*legacyAttempt]struct{}) |
| } |
| - if s.legacyStarts[host] == nil { |
| - s.legacyStarts[host] = make(map[*legacyAttempt]struct{}) |
| + if s.legacyStarts[key] == nil { |
| + s.legacyStarts[key] = make(map[*legacyAttempt]struct{}) |
| } |
| - s.legacyStarts[host][a] = struct{}{} |
| + s.legacyStarts[key][a] = struct{}{} |
| finish := func() { |
| s.legacyMu.Lock() |
| - delete(s.legacyStarts[host], a) |
| - if len(s.legacyStarts[host]) == 0 { |
| - delete(s.legacyStarts, host) |
| + delete(s.legacyStarts[key], a) |
| + if len(s.legacyStarts[key]) == 0 { |
| + delete(s.legacyStarts, key) |
| } |
| s.legacyMu.Unlock() |
| cancel() |
| @@ -99,37 +116,38 @@ func (s *Server) beginLegacyStart(parent context.Context, host string, seq uint6 |
| return ctx, finish, nil |
| } |
| -// fenceLegacyStarts is a /stop's first step for host: refuse new starts |
| -// while the stop runs, refuse every start accepted at or before this moment |
| -// (the accepted-connection watermark, read under legacyMu, so no start can |
| -// register between the watermark and the snapshot), cancel the attempts in |
| -// flight and join them, bounded by the join budget and ctx. The fence is |
| -// per host and in memory only: release lifts the refusal of new starts when |
| -// the stop is done, and only connections accepted before the watermark |
| -// stay refused. An error means an attempt did not return in time and may |
| -// still submit. |
| -func (s *Server) fenceLegacyStarts(ctx context.Context, host string) (func(), error) { |
| +// fenceLegacyStarts is a /stop's first step for role's unit of host: refuse |
| +// new starts of that unit while the stop runs, refuse every start of it |
| +// accepted at or before this moment (the accepted-connection watermark, |
| +// read under legacyMu, so no start can register between the watermark and |
| +// the snapshot), cancel the attempts in flight and join them, bounded by |
| +// the join budget and ctx. The fence is per (role, host) and in memory |
| +// only: release lifts the refusal of new starts when the stop is done, and |
| +// only connections accepted before the watermark stay refused. An error |
| +// means an attempt did not return in time and may still submit. |
| +func (s *Server) fenceLegacyStarts(ctx context.Context, role Role, host string) (func(), error) { |
| + key := roleHost{role, host} |
| s.legacyMu.Lock() |
| seq := s.connSeq.Load() |
| if s.legacyStopping == nil { |
| - s.legacyStopping = make(map[string]int) |
| + s.legacyStopping = make(map[roleHost]int) |
| } |
| if s.legacyBarrier == nil { |
| - s.legacyBarrier = make(map[string]uint64) |
| + s.legacyBarrier = make(map[roleHost]uint64) |
| } |
| - s.legacyStopping[host]++ |
| - if seq > s.legacyBarrier[host] { |
| - s.legacyBarrier[host] = seq |
| + s.legacyStopping[key]++ |
| + if seq > s.legacyBarrier[key] { |
| + s.legacyBarrier[key] = seq |
| } |
| var inFlight []*legacyAttempt |
| - for a := range s.legacyStarts[host] { |
| + for a := range s.legacyStarts[key] { |
| inFlight = append(inFlight, a) |
| } |
| s.legacyMu.Unlock() |
| release := func() { |
| s.legacyMu.Lock() |
| - if s.legacyStopping[host]--; s.legacyStopping[host] <= 0 { |
| - delete(s.legacyStopping, host) |
| + if s.legacyStopping[key]--; s.legacyStopping[key] <= 0 { |
| + delete(s.legacyStopping, key) |
| } |
| s.legacyMu.Unlock() |
| } |
| @@ -144,9 +162,9 @@ func (s *Server) fenceLegacyStarts(ctx context.Context, host string) (func(), er |
| select { |
| case <-a.done: |
| case <-timer.C: |
| - return release, fmt.Errorf("a start of %s is still in flight after %s", host, budget) |
| + return release, fmt.Errorf("a %s start of %s is still in flight after %s", role, host, budget) |
| case <-ctx.Done(): |
| - return release, fmt.Errorf("a start of %s is still in flight: %w", host, ctx.Err()) |
| + return release, fmt.Errorf("a %s start of %s is still in flight: %w", role, host, ctx.Err()) |
| } |
| } |
| return release, nil |
diff --git a/internal/instancectl/legacystart_live_test.go b/internal/instancectl/legacystart_live_test.go| index 617e9a0ff..08da47684 100644 |
| --- a/internal/instancectl/legacystart_live_test.go |
| +++ b/internal/instancectl/legacystart_live_test.go |
| @@ -20,12 +20,12 @@ func TestADelayedStartAcceptedBeforeTheFenceIsRefusedWhenItRegisters(t *testing. |
| s, _ := newServer(t, &fakeCtl{}) |
| _ = s.nextConnSeq() // connection 1: will carry the /stop |
| startConn := s.nextConnSeq() // connection 2: the delayed /start |
| - release, err := s.fenceLegacyStarts(context.Background(), host) |
| + release, err := s.fenceLegacyStarts(context.Background(), RoleRun, host) |
| if err != nil { |
| t.Fatalf("fence: %v", err) |
| } |
| release() |
| - if _, finish, err := s.beginLegacyStart(context.Background(), host, startConn); err == nil { |
| + if _, finish, err := s.beginLegacyStart(context.Background(), RoleRun, host, startConn); err == nil { |
| finish() |
| t.Fatal("a start accepted before the fence registered after it and was admitted") |
| } |
| @@ -77,7 +77,7 @@ func TestTheFenceCoversOnlyStartsAcceptedBeforeTheStop(t *testing.T) { |
| t.Fatalf("a start accepted after the stop returned: status %d, want 200", start.StatusCode) |
| } |
| s.legacyMu.Lock() |
| - stopping := s.legacyStopping[host] |
| + stopping := s.legacyStopping[roleHost{RoleRun, host}] |
| s.legacyMu.Unlock() |
| if stopping != 0 { |
| t.Fatalf("the stop's refusal of new starts outlived it (%d)", stopping) |
| @@ -91,10 +91,16 @@ func TestTheFenceCoversOnlyStartsAcceptedBeforeTheStop(t *testing.T) { |
| } |
| } |
| -// heldStart parks host's legacy /start before its tier step until release |
| -// is closed, and returns once the handler is parked there. The handler does |
| -// not see its own cancellation (unseenCancel). |
| +// heldStart parks host's legacy run /start before its tier step until |
| +// release is closed, and returns once the handler is parked there. The |
| +// handler does not see its own cancellation (unseenCancel). |
| func heldStart(t *testing.T, ctl *fakeCtl, c *http.Client, host string) (release chan struct{}) { |
| + t.Helper() |
| + return heldStartRole(t, ctl, c, RoleRun, host) |
| +} |
| + |
| +// heldStartRole is heldStart for a fenced role (run or worker, D19). |
| +func heldStartRole(t *testing.T, ctl *fakeCtl, c *http.Client, role Role, host string) (release chan struct{}) { |
| t.Helper() |
| entered, gate := make(chan struct{}), make(chan struct{}) |
| ctl.mu.Lock() |
| @@ -107,7 +113,7 @@ func heldStart(t *testing.T, ctl *fakeCtl, c *http.Client, host string) (release |
| } |
| ctl.unseenCancel = true |
| ctl.mu.Unlock() |
| - postAsync(c, "/start?host="+host+"&role=run&memory_max=536870912") |
| + postAsync(c, "/start?host="+host+"&role="+string(role)+"&memory_max=536870912") |
| select { |
| case <-entered: |
| case <-time.After(5 * time.Second): |
| @@ -147,13 +153,25 @@ func TestAStuckJoinDelaysNoOtherHost(t *testing.T) { |
| // /stop of the host does not hang behind the stuck attempt: it answers the |
| // same error within the bound, and joins it once it returns. |
| func TestAStuckStopAnswersAndTheNextStopDoesNotHang(t *testing.T) { |
| - host := "wedged.example.com" |
| + stuckStopAnswersWithinTheBound(t, RoleRun, "wedged.example.com") |
| +} |
| + |
| +// D19: a worker /stop whose held attempt does not return within the join |
| +// budget answers an error (502 naming the start in flight) rather than |
| +// 204, so the agent keeps the lease; the next stop does not hang either, |
| +// and once the attempt returns a stop answers 204. |
| +func TestWorkerJoinPastBudgetAnswersError(t *testing.T) { |
| + stuckStopAnswersWithinTheBound(t, RoleWorker, "wedged-worker.example.com") |
| +} |
| + |
| +func stuckStopAnswersWithinTheBound(t *testing.T, role Role, host string) { |
| + t.Helper() |
| ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "inv1\n"}} |
| _, c := newServer(t, ctl, func(s *Server) { s.legacyJoin = 200 * time.Millisecond }) |
| - gate := heldStart(t, ctl, c, host) |
| + gate := heldStartRole(t, ctl, c, role, host) |
| stopOnce := func() (int, string, time.Duration) { |
| begun := time.Now() |
| - resp := post(t, c, "/stop?host="+host+"&role=run") |
| + resp := post(t, c, "/stop?host="+host+"&role="+string(role)) |
| body, _ := io.ReadAll(resp.Body) |
| resp.Body.Close() |
| return resp.StatusCode, string(body), time.Since(begun) |
diff --git a/internal/instancectl/legacystart_test.go b/internal/instancectl/legacystart_test.go| index ad8b3acf9..fd3f7ae05 100644 |
| --- a/internal/instancectl/legacystart_test.go |
| +++ b/internal/instancectl/legacystart_test.go |
| @@ -17,8 +17,20 @@ import ( |
| // does not see that cancellation (unseenCancel): no start may be submitted |
| // once the /stop has returned. |
| func TestALegacyStartDelayedPastItsCancellationNeverSubmitsAfterAStop(t *testing.T) { |
| - host := "late.example.com" |
| - unit := UnitName(RoleRun, host) |
| + heldStartNeverSubmitsAfterAStop(t, RoleRun, "late.example.com") |
| +} |
| + |
| +// Sidecar isolation D19 (#370): the worker unit gets the run unit's |
| +// ordering. A worker /start whose client gave up while the handler was |
| +// held is cancelled and joined by the worker /stop that follows, and the |
| +// held handler never submits once that stop has answered. |
| +func TestWorkerStopCancelsAndJoinsHeldStart(t *testing.T) { |
| + heldStartNeverSubmitsAfterAStop(t, RoleWorker, "late-worker.example.com") |
| +} |
| + |
| +func heldStartNeverSubmitsAfterAStop(t *testing.T, role Role, host string) { |
| + t.Helper() |
| + unit := UnitName(role, host) |
| entered, gate := make(chan struct{}), make(chan struct{}) |
| ctl := &fakeCtl{ |
| answers: map[string]string{"show -p InvocationID --value": "inv1\n"}, |
| @@ -34,7 +46,7 @@ func TestALegacyStartDelayedPastItsCancellationNeverSubmitsAfterAStop(t *testing |
| startDone := make(chan struct{}) |
| go func() { |
| defer close(startDone) |
| - req, _ := http.NewRequestWithContext(startCtx, http.MethodPost, "http://unix/start?host="+host+"&role=run&memory_max=536870912", nil) |
| + req, _ := http.NewRequestWithContext(startCtx, http.MethodPost, "http://unix/start?host="+host+"&role="+string(role)+"&memory_max=536870912", nil) |
| if resp, err := c.Do(req); err == nil { |
| resp.Body.Close() |
| } |
| @@ -49,7 +61,7 @@ func TestALegacyStartDelayedPastItsCancellationNeverSubmitsAfterAStop(t *testing |
| stopped := make(chan int, 1) |
| go func() { |
| - resp := post(t, c, "/stop?host="+host+"&role=run") |
| + resp := post(t, c, "/stop?host="+host+"&role="+string(role)) |
| resp.Body.Close() |
| stopped <- resp.StatusCode |
| }() |
| @@ -92,23 +104,92 @@ func TestALegacyStartDelayedPastItsCancellationNeverSubmitsAfterAStop(t *testing |
| // earlier request: refused once that stop has run, even if its handler only |
| // gets going afterwards. |
| func TestALegacyStartFromBeforeAStopIsRefused(t *testing.T) { |
| - host := "early.example.com" |
| + startFromBeforeAStopIsRefused(t, RoleRun, "early.example.com") |
| +} |
| + |
| +// D19: the same watermark rule for the worker role. |
| +func TestWorkerStartAcceptedBeforeStopIsRefused(t *testing.T) { |
| + startFromBeforeAStopIsRefused(t, RoleWorker, "early-worker.example.com") |
| +} |
| + |
| +func startFromBeforeAStopIsRefused(t *testing.T, role Role, host string) { |
| + t.Helper() |
| ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "inv1\n"}} |
| s, _ := newServer(t, ctl) |
| // Stamp two connections in accept order: the start's first. |
| startSeq := s.nextConnSeq() |
| _ = s.nextConnSeq() // the stop's own connection |
| - release, err := s.fenceLegacyStarts(context.Background(), host) |
| + release, err := s.fenceLegacyStarts(context.Background(), role, host) |
| if err != nil { |
| t.Fatalf("fence: %v", err) |
| } |
| release() |
| - if _, _, err := s.beginLegacyStart(context.Background(), host, startSeq); err == nil { |
| + if _, _, err := s.beginLegacyStart(context.Background(), role, host, startSeq); err == nil { |
| t.Fatal("a start accepted before the stop was admitted after it") |
| } |
| - if _, finish, err := s.beginLegacyStart(context.Background(), host, s.nextConnSeq()); err != nil { |
| + if _, finish, err := s.beginLegacyStart(context.Background(), role, host, s.nextConnSeq()); err != nil { |
| t.Fatalf("a start accepted after the stop was refused: %v", err) |
| } else { |
| finish() |
| } |
| } |
| + |
| +// D19: the fence is keyed by host AND role. A worker's stop never fences |
| +// the same host's run unit: a run start accepted before the worker stop is |
| +// still admitted afterwards, and a run start or stop answers at once while |
| +// the worker stop is joining a stuck worker attempt. |
| +func TestWorkerStopDoesNotFenceRunUnit(t *testing.T) { |
| + stopOfOneRoleDoesNotFenceTheOther(t, RoleWorker, RoleRun, "worker-then-run.example.com") |
| +} |
| + |
| +// The reverse: a run unit's stop never fences the same host's worker. |
| +func TestRunStopDoesNotFenceWorkerUnit(t *testing.T) { |
| + stopOfOneRoleDoesNotFenceTheOther(t, RoleRun, RoleWorker, "run-then-worker.example.com") |
| +} |
| + |
| +func stopOfOneRoleDoesNotFenceTheOther(t *testing.T, stopping, other Role, host string) { |
| + t.Helper() |
| + ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "inv1\n"}} |
| + s, c := newServer(t, ctl) |
| + |
| + // The watermark: a start of the other role accepted before this role's |
| + // stop is not an earlier request of THIS fence. |
| + otherSeq := s.nextConnSeq() |
| + _ = s.nextConnSeq() |
| + release, err := s.fenceLegacyStarts(context.Background(), stopping, host) |
| + if err != nil { |
| + t.Fatalf("fence: %v", err) |
| + } |
| + release() |
| + if _, finish, err := s.beginLegacyStart(context.Background(), other, host, otherSeq); err != nil { |
| + t.Fatalf("a %s start accepted before a %s stop was refused: %v", other, stopping, err) |
| + } else { |
| + finish() |
| + } |
| + if _, _, err := s.beginLegacyStart(context.Background(), stopping, host, otherSeq); err == nil { |
| + t.Fatalf("a %s start accepted before a %s stop was admitted", stopping, stopping) |
| + } |
| + |
| + // The join: while this role's stop joins a held attempt, the other |
| + // role's verbs on the same host pass through. |
| + gate := heldStartRole(t, ctl, c, stopping, host) |
| + stop := postAsync(c, "/stop?host="+host+"&role="+string(stopping)) |
| + defer func() { close(gate); <-stop }() |
| + time.Sleep(50 * time.Millisecond) // the stop is joining |
| + for _, path := range []string{"/start?host=" + host + "&role=" + string(other) + "&memory_max=536870912", "/stop?host=" + host + "&role=" + string(other)} { |
| + done := postAsync(c, path) |
| + select { |
| + case code := <-done: |
| + if code/100 != 2 { |
| + t.Fatalf("%s during a %s stop: status %d", path, stopping, code) |
| + } |
| + case <-time.After(2 * time.Second): |
| + t.Fatalf("%s waited behind a %s stop of the same host", path, stopping) |
| + } |
| + } |
| + select { |
| + case <-stop: |
| + t.Fatalf("the %s stop answered while its attempt was still held", stopping) |
| + default: |
| + } |
| +} |
diff --git a/internal/instancectl/names.go b/internal/instancectl/names.go| index 2dd8a4f1c..e1f3f3e80 100644 |
| --- a/internal/instancectl/names.go |
| +++ b/internal/instancectl/names.go |
| @@ -22,7 +22,7 @@ import ( |
| "github.com/carlosframework/platform/internal/provision" |
| ) |
| -// Role is which of the four template units a request or a file is about. |
| +// Role is which of the five template units a request or a file is about. |
| type Role string |
| const ( |
| @@ -32,12 +32,18 @@ const ( |
| // RoleCmd is one `carlos run` command (run spec §5.4). "run" is the |
| // serving role already, so the command's role is cmd. |
| RoleCmd Role = "cmd" |
| + // RoleWorker is a native sidecar's own unit (sidecar isolation D1, |
| + // #370): a worker IS its host, so its unit and slice are keyed by the |
| + // worker's host, not an instance's. The sidecar role stays the legacy |
| + // co-spawn's, keyed by the instance it accompanies. A worker has one |
| + // unit per host and never a generation. |
| + RoleWorker Role = "worker" |
| ) |
| -// ParseRole accepts exactly the four literals. |
| +// ParseRole accepts exactly the five literals. |
| func ParseRole(s string) (Role, bool) { |
| switch Role(s) { |
| - case RoleRun, RoleSeed, RoleSidecar, RoleCmd: |
| + case RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker: |
| return Role(s), true |
| } |
| return "", false |
| @@ -68,13 +74,14 @@ func Escape(s string) string { |
| return b.String() |
| } |
| -// The four template prefixes. A unit under any of them is the platform's |
| +// The five template prefixes. A unit under any of them is the platform's |
| // to start, stop and sweep; nothing else is (design §5.3). |
| const ( |
| UnitPrefixRun = "carlos-instance@" |
| UnitPrefixSeed = "carlos-instance-seed@" |
| UnitPrefixSidecar = "carlos-instance-sidecar@" |
| UnitPrefixCmd = "carlos-instance-cmd@" |
| + UnitPrefixWorker = "carlos-instance-worker@" |
| ) |
| // CmdExeApp and CmdExeLive are the cmd template's executable mount |
| @@ -94,12 +101,14 @@ func prefixFor(role Role) string { |
| return UnitPrefixSidecar |
| case RoleCmd: |
| return UnitPrefixCmd |
| + case RoleWorker: |
| + return UnitPrefixWorker |
| } |
| return UnitPrefixRun |
| } |
| // UnitName is the template instance for a role and a host: |
| -// carlos-instance[-seed|-sidecar|-cmd]@<Escape(InstanceID(host))>.service. |
| +// carlos-instance[-seed|-sidecar|-cmd|-worker]@<Escape(InstanceID(host))>.service. |
| func UnitName(role Role, host string) string { |
| return prefixFor(role) + Escape(provision.InstanceID(host)) + ".service" |
| } |
| @@ -217,7 +226,7 @@ func Unescape(s string) (string, bool) { |
| return b.String(), true |
| } |
| -// ParseUnitName recognises the four templates' instances and returns the |
| +// ParseUnitName recognises the five templates' instances and returns the |
| // role and the ESCAPED identifier (systemd reports names escaped). Anything |
| // else, including a template with no instance, is not ours. |
| func ParseUnitName(unit string) (Role, string, bool) { |
| @@ -232,7 +241,7 @@ func ParseUnitName(unit string) (Role, string, bool) { |
| for _, p := range []struct { |
| prefix string |
| role Role |
| - }{{UnitPrefixSeed, RoleSeed}, {UnitPrefixSidecar, RoleSidecar}, {UnitPrefixCmd, RoleCmd}, {UnitPrefixRun, RoleRun}} { |
| + }{{UnitPrefixSeed, RoleSeed}, {UnitPrefixSidecar, RoleSidecar}, {UnitPrefixCmd, RoleCmd}, {UnitPrefixWorker, RoleWorker}, {UnitPrefixRun, RoleRun}} { |
| if strings.HasPrefix(body, p.prefix) { |
| esc := strings.TrimPrefix(body, p.prefix) |
| if esc == "" { |
diff --git a/internal/instancectl/names_test.go b/internal/instancectl/names_test.go| index 50b33f0f8..9f12cc697 100644 |
| --- a/internal/instancectl/names_test.go |
| +++ b/internal/instancectl/names_test.go |
| @@ -53,8 +53,48 @@ func TestUnitAndSliceNamesAreKeyedByEscapedInstanceID(t *testing.T) { |
| } |
| } |
| +// Sidecar isolation D1 (#370): the fifth original role, worker, has its |
| +// own template, carlos-instance-worker@, keyed by the worker's own host |
| +// (a worker IS its host), in that host's slice. The sidecar role and its |
| +// template are untouched. |
| +func TestParseUnitNameWorker(t *testing.T) { |
| + host := "bill-2.example.com" |
| + esc := `bill\x2d2.example.com` |
| + if UnitPrefixWorker != "carlos-instance-worker@" { |
| + t.Fatalf("UnitPrefixWorker = %q", UnitPrefixWorker) |
| + } |
| + if got := UnitName(RoleWorker, host); got != "carlos-instance-worker@"+esc+".service" { |
| + t.Errorf("worker unit = %q", got) |
| + } |
| + if got := UnitName(RoleWorker, host); got != UnitPrefixWorker+Escape(provision.InstanceID(host))+".service" { |
| + t.Errorf("worker unit = %q is not keyed by the escaped instance id", got) |
| + } |
| + gotRole, gotEsc, ok := ParseUnitName("carlos-instance-worker@" + esc + ".service") |
| + if !ok || gotRole != RoleWorker || gotEsc != esc { |
| + t.Errorf("ParseUnitName(worker unit) = %q %q %v", gotRole, gotEsc, ok) |
| + } |
| + if _, _, ok := ParseUnitName("carlos-instance-worker@.service"); ok { |
| + t.Error("a worker template with no instance was accepted") |
| + } |
| + if r, ok := ParseRole("worker"); !ok || r != RoleWorker || string(RoleWorker) != "worker" { |
| + t.Errorf("ParseRole(worker) = %q %v", r, ok) |
| + } |
| + if got := UnitName(RoleSidecar, host); got != "carlos-instance-sidecar@"+esc+".service" { |
| + t.Errorf("the sidecar unit changed: %q", got) |
| + } |
| + if got := SliceName(host); got != "carlos-instances-"+esc+".slice" { |
| + t.Errorf("slice = %q, want the host's slice unchanged", got) |
| + } |
| + if got := LaunchFileName(RoleWorker); got != "launch.worker.json" { |
| + t.Errorf("LaunchFileName(worker) = %q", got) |
| + } |
| + if got := ExitRecordName(RoleWorker, "abc123"); got != "exit.worker.abc123.json" { |
| + t.Errorf("ExitRecordName(worker) = %q", got) |
| + } |
| +} |
| + |
| func TestParseUnitNameRoundTrips(t *testing.T) { |
| - for _, role := range []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd} { |
| + for _, role := range []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker} { |
| u := UnitName(role, "bill-2.example.com") |
| gotRole, esc, ok := ParseUnitName(u) |
| if !ok || gotRole != role || esc != `bill\x2d2.example.com` { |
| @@ -72,12 +112,12 @@ func TestParseUnitNameRoundTrips(t *testing.T) { |
| } |
| func TestParseRole(t *testing.T) { |
| - for _, s := range []string{"run", "seed", "sidecar", "cmd"} { |
| + for _, s := range []string{"run", "seed", "sidecar", "cmd", "worker"} { |
| if r, ok := ParseRole(s); !ok || string(r) != s { |
| t.Errorf("ParseRole(%q) = %q %v", s, r, ok) |
| } |
| } |
| - for _, s := range []string{"", "Run", "exec", "run ", "Cmd", "cmd-err"} { |
| + for _, s := range []string{"", "Run", "exec", "run ", "Cmd", "cmd-err", "Worker", "workers"} { |
| if _, ok := ParseRole(s); ok { |
| t.Errorf("ParseRole(%q) accepted", s) |
| } |
diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go| index 6ccff14ff..25ed98a3c 100644 |
| --- a/internal/instancectl/server.go |
| +++ b/internal/instancectl/server.go |
| @@ -28,7 +28,7 @@ import ( |
| ) |
| // Server is the root instance-control service (design §5). It does four |
| -// things and nothing else: starts, stops and sweeps the four templates' |
| +// things and nothing else: starts, stops and sweeps the five templates' |
| // instances, sets a host slice's memory tier before a start, runs the |
| // readiness probe, and answers /ready. Every systemctl argument it passes |
| // is built from a validated host by UnitName/SliceName; nothing from a |
| @@ -128,14 +128,14 @@ type Server struct { |
| genMu sync.Mutex |
| genStarting map[string]bool |
| - // Legacy run start attempts per host (legacystart.go, #292): the |
| - // starts in flight, the stops in progress, and each host's barrier, |
| - // the accept order of the latest stop's request. connSeq numbers |
| - // connections in accept order. |
| + // Legacy start attempts per (role, host) (legacystart.go, #292; the |
| + // worker role too, sidecar isolation D19): the starts in flight, the |
| + // stops in progress, and each unit's barrier, the accept order of the |
| + // latest stop's request. connSeq numbers connections in accept order. |
| legacyMu sync.Mutex |
| - legacyStarts map[string]map[*legacyAttempt]struct{} |
| - legacyStopping map[string]int |
| - legacyBarrier map[string]uint64 |
| + legacyStarts map[roleHost]map[*legacyAttempt]struct{} |
| + legacyStopping map[roleHost]int |
| + legacyBarrier map[roleHost]uint64 |
| connSeq atomic.Uint64 |
| // legacyJoin overrides legacyJoinBudget; tests shorten it. |
| legacyJoin time.Duration |
| @@ -368,7 +368,7 @@ func parseRequest(r *http.Request, v verbShape) (request, error) { |
| if len(v.roles) > 0 { |
| role, ok := ParseRole(q.Get("role")) |
| if !ok { |
| - return request{}, fmt.Errorf("role %q is not run, seed, sidecar or cmd", q.Get("role")) |
| + return request{}, fmt.Errorf("role %q is not run, seed, sidecar, cmd or worker", q.Get("role")) |
| } |
| allowed := false |
| for _, r := range v.roles { |
| @@ -418,12 +418,14 @@ func parseRequest(r *http.Request, v verbShape) (request, error) { |
| if x := q.Get("exe"); x != "" { |
| // A generation start; the legacy sidecar (zero-downtime spec |
| // §7.5: it runs its instance's serving artifact); a run's |
| - // command, which runs that same artifact; or the original |
| + // command, which runs that same artifact; the original |
| // template's run unit and the seed unit, which run the build a |
| - // cold wake recorded (settings spec §4.2). Every one is |
| - // validated before anything is laid out (refuseStagedExe). |
| - if req.gen == 0 && !v.seed && !v.cmd && req.role != RoleRun && req.role != RoleSidecar { |
| - return request{}, errors.New("exe is only for a run, seed or sidecar unit, a run's command, or a generation start") |
| + // cold wake recorded (settings spec §4.2); or a worker unit, |
| + // which always runs the staged build its wake resolved |
| + // (sidecar isolation D5). Every one is validated before |
| + // anything is laid out (refuseStagedExe). |
| + if req.gen == 0 && !v.seed && !v.cmd && req.role != RoleRun && req.role != RoleSidecar && req.role != RoleWorker { |
| + return request{}, errors.New("exe is only for a run, seed, sidecar or worker unit, a run's command, or a generation start") |
| } |
| req.exe = StagedExe{Path: x, Version: q.Get("version"), Digest: q.Get("digest")} |
| } else { |
| @@ -435,7 +437,7 @@ func parseRequest(r *http.Request, v verbShape) (request, error) { |
| // setTier writes the whole tier onto the host slice in one call: the |
| // memory caps, and the CPU and IO shares that follow from the same tier |
| -// (#214 item 1). Seed, instance and sidecar share all of it. |
| +// (#214 item 1). Seed, instance, sidecar and worker share all of it. |
| func (s *Server) setTier(ctx context.Context, host string, tier cgroup.Limit) error { |
| return sliceTier(ctx, s.Systemctl, host, tier) |
| } |
| @@ -459,20 +461,23 @@ func writeJSON(w http.ResponseWriter, status int, v any) { |
| } |
| func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { |
| - req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar}, tier: true, gen: true, exe: true}) |
| + req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar, RoleWorker}, tier: true, gen: true, exe: true}) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusBadRequest) |
| return |
| } |
| if req.gen != 0 { |
| + // parseRequest admits a generation for the run role only; a worker |
| + // has one unit per host and never a generation (D1). |
| s.startGen(w, r, req) |
| return |
| } |
| ctx := r.Context() |
| - if req.role == RoleRun { |
| - // An attempt a /stop of this host can cancel and join, and that a |
| - // stop which has already run refuses (legacystart.go, #292). |
| - attempt, finish, err := s.beginLegacyStart(ctx, req.host, requestSeq(ctx)) |
| + if fencedRole(req.role) { |
| + // An attempt a /stop of this unit can cancel and join, and that a |
| + // stop which has already run refuses (legacystart.go, #292; the |
| + // worker unit too, sidecar isolation D19). |
| + attempt, finish, err := s.beginLegacyStart(ctx, req.role, req.host, requestSeq(ctx)) |
| if err != nil { |
| http.Error(w, "start: "+err.Error(), http.StatusConflict) |
| return |
| @@ -481,13 +486,17 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { |
| ctx = attempt |
| } |
| unit := UnitName(req.role, req.host) |
| - // A sidecar naming its generation's staged build, or a run unit naming |
| - // the build its cold wake recorded, is validated before anything is |
| - // laid out, under the generation rule (validateStagedExe). |
| + // A sidecar naming its generation's staged build, a run unit naming |
| + // the build its cold wake recorded, or a worker naming the build its |
| + // wake resolved, is validated before anything is laid out, under the |
| + // generation rule (validateStagedExe, keyed by the request's host: for |
| + // a worker that is the worker host's own app). |
| if s.refuseStagedExe(w, req, unit) { |
| return |
| } |
| if req.role == RoleRun { |
| + // The claim launch snapshot belongs to the run unit alone; a |
| + // worker has none. |
| if err := s.clearLegacyClaimLaunch(ctx, req.host); err != nil { |
| http.Error(w, "start: "+err.Error(), http.StatusBadGateway) |
| return |
| @@ -507,15 +516,17 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { |
| // alone keeps the right tier too. A start without exe (a live-link |
| // sidecar) sets the tier as before. The exemption is the sidecar's |
| // alone: a run unit naming its recorded build starts a cold instance, |
| - // and sets its tier as a run start always has. |
| + // and sets its tier as a run start always has; a worker's slice is its |
| + // own (its host has no run unit), so a worker start always sets the |
| + // tier too (sidecar isolation D1). |
| if req.role == RoleSidecar && req.exe.Path != "" && s.slicePopulated(req.host) { |
| s.logf("instance-control: %s: slice populated, leaving its limit to the generation owner", req.host) |
| } else if err := s.setTier(ctx, req.host, req.tier); err != nil { |
| http.Error(w, "set-property: "+err.Error(), http.StatusBadGateway) |
| return |
| } |
| - // Never submitted once cancelled (#292): a /stop of this host cancels |
| - // the attempt before it joins it, and the client may have gone. |
| + // Never submitted once cancelled (#292, D19): a /stop of this unit |
| + // cancels the attempt before it joins it, and the client may have gone. |
| if err := ctx.Err(); err != nil { |
| http.Error(w, "start: cancelled before it was submitted: "+err.Error(), http.StatusConflict) |
| return |
| @@ -658,7 +669,7 @@ func (s *Server) stopUnit(ctx context.Context, unit string) { |
| } |
| func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { |
| - req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd}, gen: true, kill: true}) |
| + req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker}, gen: true, kill: true}) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusBadRequest) |
| return |
| @@ -667,13 +678,15 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { |
| s.stopGen(w, r, req) |
| return |
| } |
| - if req.role == RoleRun { |
| - // Every start of this host's run unit in flight is cancelled and |
| - // joined first, and earlier requests' starts are refused from here |
| - // on (legacystart.go, #292): once this answers, no start from an |
| - // earlier request can still be submitted, and the stop below |
| - // replaces any job a joined start had submitted. |
| - release, err := s.fenceLegacyStarts(r.Context(), req.host) |
| + if fencedRole(req.role) { |
| + // Every start of this unit (the host's run unit, or its worker |
| + // unit, D19) in flight is cancelled and joined first, and earlier |
| + // requests' starts are refused from here on (legacystart.go, |
| + // #292): once this answers, no start from an earlier request can |
| + // still be submitted, and the stop below replaces any job a |
| + // joined start had submitted. The fence is the unit's own: a |
| + // worker stop fences no run start of the host, nor the reverse. |
| + release, err := s.fenceLegacyStarts(r.Context(), req.role, req.host) |
| defer release() |
| if err != nil { |
| // Stopped anyway, but not proven final: a start may follow. |
| @@ -699,7 +712,7 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { |
| } |
| func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) { |
| - req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd}, gen: true}) |
| + req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker}, gen: true}) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusBadRequest) |
| return |
| @@ -715,7 +728,7 @@ func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) { |
| w.WriteHeader(http.StatusNoContent) |
| } |
| -// handleSweep stops every instance of the four templates that is not |
| +// handleSweep stops every instance of the five templates that is not |
| // inactive or has a job, keyed on the names systemd reports, so a unit |
| // whose identifier is hashed and whose row is gone is still stopped |
| // (design §6.7, codex R7-1). |
| @@ -723,7 +736,7 @@ func (s *Server) handleSweep(w http.ResponseWriter, r *http.Request) { |
| ctx := r.Context() |
| targets := map[string]bool{} |
| out, err := s.Systemctl(ctx, "list-units", "--all", "--plain", "--no-legend", "--no-pager", |
| - UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service", UnitPrefixCmd+"*.service", UnitPrefixGen+"*.service") |
| + UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service", UnitPrefixCmd+"*.service", UnitPrefixWorker+"*.service", UnitPrefixGen+"*.service") |
| if err != nil { |
| http.Error(w, "list-units: "+err.Error(), http.StatusBadGateway) |
| return |
diff --git a/internal/instancectl/server_test.go b/internal/instancectl/server_test.go| index c2e12c2b3..dd2e313bf 100644 |
| --- a/internal/instancectl/server_test.go |
| +++ b/internal/instancectl/server_test.go |
| @@ -708,6 +708,92 @@ func TestReadyRefusesWhenTheRunningBinaryIsNotTheInstalledOne(t *testing.T) { |
| } |
| } |
| +// Sidecar isolation D1 (#370): a worker start on the shared verbs takes |
| +// the run role's tier rule. A worker's slice is its own, so the tier is |
| +// always set, and the start is the one systemd call sequence the run unit |
| +// gets: set-property, start, show. |
| +func TestStartWorkerSetsTier(t *testing.T) { |
| + ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "w1\n"}} |
| + _, c := newServer(t, ctl) |
| + resp := post(t, c, "/start?host=bill-2.example.com&role=worker&memory_max=536870912") |
| + if resp.StatusCode != 200 { |
| + t.Fatalf("status %d", resp.StatusCode) |
| + } |
| + var out StartResponse |
| + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil || out.InvocationID != "w1" { |
| + t.Errorf("response = %+v, %v", out, err) |
| + } |
| + want := "set-property --runtime carlos-instances-bill\\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100\n" + |
| + "start carlos-instance-worker@bill\\x2d2.example.com.service\n" + |
| + "show -p InvocationID --value carlos-instance-worker@bill\\x2d2.example.com.service" |
| + if got := ctl.joined(); got != want { |
| + t.Errorf("systemctl calls:\n%s\nwant:\n%s", got, want) |
| + } |
| +} |
| + |
| +// A worker has one unit per host and no generation units (D1): /start |
| +// with the worker role and a generation is 400, before systemctl is asked |
| +// anything; so are the generation-only parameters on its stop. |
| +func TestStartWorkerRefusesGen(t *testing.T) { |
| + ctl := &fakeCtl{} |
| + _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() }) |
| + for _, q := range []string{ |
| + "/start?host=h.example.com&role=worker&memory_max=536870912&gen=1", |
| + "/start?host=h.example.com&role=worker&memory_max=536870912&gen=7", |
| + "/stop?host=h.example.com&role=worker&gen=1", |
| + "/stop?host=h.example.com&role=worker&kill=1", |
| + "/reset?host=h.example.com&role=worker&gen=1", |
| + } { |
| + resp := post(t, c, q) |
| + resp.Body.Close() |
| + if resp.StatusCode != 400 { |
| + t.Errorf("%s: status %d, want 400", q, resp.StatusCode) |
| + } |
| + } |
| + if ctl.joined() != "" { |
| + t.Errorf("systemctl was called: %s", ctl.joined()) |
| + } |
| +} |
| + |
| +// The worker role is admitted on /stop, /reset and /sweep (D1): its unit |
| +// is stopped and reset by name, and the sweep lists and stops the worker |
| +// template's instances as it does the other four. |
| +func TestStopResetAndSweepCoverTheWorkerRole(t *testing.T) { |
| + ctl := &fakeCtl{answers: map[string]string{ |
| + "list-units": "carlos-instance-worker@a.example.com.service loaded active running carlos exec-backed instance a.example.com (worker)\n", |
| + "show -p ActiveState --value": "inactive\n", |
| + }} |
| + s, c := newServer(t, ctl) |
| + s.cgroupWaitBudget = time.Second |
| + if resp := post(t, c, "/stop?host=a.example.com&role=worker"); resp.StatusCode != http.StatusNoContent { |
| + t.Fatalf("stop status %d", resp.StatusCode) |
| + } |
| + if resp := post(t, c, "/reset?host=a.example.com&role=worker"); resp.StatusCode/100 != 2 { |
| + t.Fatalf("reset status %d", resp.StatusCode) |
| + } |
| + resp := post(t, c, "/sweep") |
| + if resp.StatusCode != http.StatusOK { |
| + t.Fatalf("sweep status %d", resp.StatusCode) |
| + } |
| + var swept SweepResponse |
| + if err := json.NewDecoder(resp.Body).Decode(&swept); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(swept.Units) != 1 || swept.Units[0] != "carlos-instance-worker@a.example.com.service" { |
| + t.Errorf("swept = %v, want the listed worker unit", swept.Units) |
| + } |
| + calls := ctl.joined() |
| + for _, want := range []string{ |
| + "stop carlos-instance-worker@a.example.com.service", |
| + "reset-failed carlos-instance-worker@a.example.com.service", |
| + "carlos-instance-worker@*.service", |
| + } { |
| + if !strings.Contains(calls, want) { |
| + t.Errorf("missing %q in\n%s", want, calls) |
| + } |
| + } |
| +} |
| + |
| func TestStopResetAndSweepCoverTheCmdRole(t *testing.T) { |
| ctl := &fakeCtl{answers: map[string]string{ |
| "list-units": "carlos-instance-cmd@a.example.com.service loaded active exited carlos exec-backed instance a.example.com (cmd)\n", |
diff --git a/internal/instancectl/stagedexe_linux_test.go b/internal/instancectl/stagedexe_linux_test.go| index da863eb5f..96f21a192 100644 |
| --- a/internal/instancectl/stagedexe_linux_test.go |
| +++ b/internal/instancectl/stagedexe_linux_test.go |
| @@ -245,6 +245,93 @@ func TestTheRunUnitBindsAValidatedStagedExe(t *testing.T) { |
| } |
| } |
| +// Sidecar isolation D1/D5 (#370): a worker start always names the staged |
| +// build the agent resolved, and root validates it with validateStagedExe |
| +// against the WORKER HOST's own app (HostApp is keyed by host), as it does |
| +// a run unit's. The validated build is what the host's app mount points |
| +// at; a wrong digest or another app's build is 403 before systemd or the |
| +// mount is touched. The tier is always set, populated slice or not: the |
| +// populated-slice exemption is the legacy sidecar's alone. |
| +func TestWorkerStartValidatesStagedExe(t *testing.T) { |
| + const worker = "worker-1.example.com" |
| + opt, exeRoot := t.TempDir(), t.TempDir() |
| + staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| + digest := writeStaged(t, staged, "worker build v2") |
| + shop := release.VersionPath(opt, release.OpsAccount, "v2", "shop") |
| + shopDigest := writeStaged(t, shop, "shop build v2") |
| + repointLiveLink(t, opt, worker, staged) |
| + ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "w1\n"}} |
| + var cgroupRoot string |
| + asked := map[string]int{} |
| + _, sock := serveOnSocket(t, ctl, func(s *Server) { |
| + s.DropInDir, s.ExeRoot, s.OptDir = t.TempDir(), exeRoot, opt |
| + cgroupRoot = s.CgroupRoot |
| + s.HostApp = func(host string) (string, string, error) { |
| + asked[host]++ |
| + switch host { |
| + case worker: |
| + return release.OpsAccount, "blog", nil |
| + case "shop.example.com": |
| + return release.OpsAccount, "shop", nil |
| + } |
| + return "", "", errors.New("unknown host") |
| + } |
| + }) |
| + c := NewClient(sock) |
| + x := StagedExe{Path: staged, Version: "v2", Digest: digest} |
| + app := filepath.Join(exeRoot, provision.InstanceID(worker), "app") |
| + |
| + if _, err := c.StartExe(context.Background(), worker, RoleWorker, cgroup.TierDefault.MemoryMax, x); err != nil { |
| + t.Fatalf("the worker host's own staged build was refused: %v", err) |
| + } |
| + if asked[worker] == 0 { |
| + t.Fatal("the registry was not asked about the worker host") |
| + } |
| + if !ctlCalled(ctl, "start "+UnitName(RoleWorker, worker)) { |
| + t.Fatalf("the worker unit was not started:\n%s", ctl.joined()) |
| + } |
| + if !ctlCalled(ctl, "set-property --runtime "+SliceName(worker)) { |
| + t.Fatalf("a worker start set no tier:\n%s", ctl.joined()) |
| + } |
| + if got, err := os.Readlink(app); err != nil || got != staged { |
| + t.Fatalf("the worker's app mount = %q (%v), want the staged %q", got, err, staged) |
| + } |
| + |
| + for name, bad := range map[string]StagedExe{ |
| + "a wrong digest": {Path: staged, Version: "v2", Digest: strings.Repeat("0", 64)}, |
| + "another app's build": {Path: shop, Version: "v2", Digest: shopDigest}, |
| + "a path outside": {Path: "/etc/passwd", Version: "v2", Digest: digest}, |
| + } { |
| + ctl.reset() |
| + if _, err := c.StartExe(context.Background(), worker, RoleWorker, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| + t.Errorf("%s: %v, want ErrStagedExe", name, err) |
| + } |
| + if ctl.joined() != "" { |
| + t.Errorf("%s: a refused start reached systemctl:\n%s", name, ctl.joined()) |
| + } |
| + if got, _ := os.Readlink(app); got != staged { |
| + t.Errorf("%s: a refused start moved the mount to %q", name, got) |
| + } |
| + } |
| + |
| + // A populated slice: the worker still gets its tier (the exemption at |
| + // server.go's sidecar branch is sidecar-only). |
| + events := filepath.Join(cgroupRoot, parentSlices, SliceName(worker), "cgroup.events") |
| + if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + ctl.reset() |
| + if _, err := c.StartExe(context.Background(), worker, RoleWorker, cgroup.TierDefault.MemoryMax, x); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !ctlCalled(ctl, "set-property --runtime "+SliceName(worker)) { |
| + t.Fatal("a staged worker start on a populated slice set no tier") |
| + } |
| +} |
| + |
| // The seed unit binds the recorded staged file the same way, through |
| // /seed, which names no role. A wrong digest is 403 before anything runs, |
| // and a seed that names no executable binds the live link, as today. |
diff --git a/internal/instanceexec/exec.go b/internal/instanceexec/exec.go| index 4e0391df8..4e76adec5 100644 |
| --- a/internal/instanceexec/exec.go |
| +++ b/internal/instanceexec/exec.go |
| @@ -41,7 +41,7 @@ func dataDir() string { |
| // RunExec is `carlos instance-exec <role> <iid>`. |
| func RunExec(args []string) { |
| fs := flag.NewFlagSet("instance-exec", flag.ExitOnError) |
| - fs.Usage = cliflag.Usage(fs, "usage: carlos instance-exec <run|seed|sidecar|cmd> <instance-id>") |
| + fs.Usage = cliflag.Usage(fs, "usage: carlos instance-exec <run|seed|sidecar|cmd|worker> <instance-id>") |
| claimSnapshot := fs.String("claim-reservation", "", "root launch reservation") |
| _ = fs.Parse(args) |
| if fs.NArg() != 2 { |
| @@ -50,7 +50,7 @@ func RunExec(args []string) { |
| } |
| role, ok := instancectl.ParseRole(fs.Arg(0)) |
| if !ok { |
| - log.Fatalf("instance-exec: role %q is not run, seed, sidecar or cmd", fs.Arg(0)) |
| + log.Fatalf("instance-exec: role %q is not run, seed, sidecar, cmd or worker", fs.Arg(0)) |
| } |
| // The tenant is scoped (#178, see ScopeAbstractUnix) and runs under |
| // the trampoline as its init (#126, see init_linux.go). |
diff --git a/internal/instanceexec/exec_test.go b/internal/instanceexec/exec_test.go| index 3d94b51db..c2fa1d85f 100644 |
| --- a/internal/instanceexec/exec_test.go |
| +++ b/internal/instanceexec/exec_test.go |
| @@ -61,6 +61,36 @@ func TestExecLaunchExecsExactlyWhatTheAgentWrote(t *testing.T) { |
| } |
| } |
| +// Sidecar isolation D1 (#370): the trampoline accepts the worker role and |
| +// reads launch.worker.json, keyed by the worker's own host. A worker has |
| +// no generations: "<iid>:<n>" under the worker role is refused as it is |
| +// for the seed and sidecar roles. |
| +func TestExecLaunchRunsTheWorkerRolesLaunchFile(t *testing.T) { |
| + data := t.TempDir() |
| + host := "worker-1.example.com" |
| + iid := provision.InstanceID(host) |
| + dir := filepath.Join(data, "instances", iid) |
| + writeLaunch(t, dir, instancectl.RoleWorker, instancectl.Launch{ |
| + Schema: 1, Host: host, |
| + Argv: []string{"/opt/carlos/live/" + host, "worker", "run"}, |
| + Env: []string{"CARLOS_SIDECAR_DB=/data/instances/" + host + "/" + host + ".db"}, |
| + }) |
| + var c captured |
| + if err := execLaunch(data, instancectl.RoleWorker, iid, c.exec); err != nil { |
| + t.Fatalf("execLaunch(worker): %v", err) |
| + } |
| + if c.calls != 1 || c.argv0 != "/opt/carlos/live/"+host || len(c.argv) != 3 || c.argv[1] != "worker" { |
| + t.Errorf("exec = %+v", c) |
| + } |
| + if role, ok := instancectl.ParseRole("worker"); !ok || role != instancectl.RoleWorker { |
| + t.Errorf("ParseRole(worker) = %q %v", role, ok) |
| + } |
| + var gen captured |
| + if err := execLaunch(data, instancectl.RoleWorker, iid+":7", gen.exec); err == nil || gen.calls != 0 { |
| + t.Errorf("a worker generation: %v after %d execs; want a refusal", err, gen.calls) |
| + } |
| +} |
| + |
| func TestExecLaunchRefusesAnArgv0OutsideTheLiveDir(t *testing.T) { |
| data := t.TempDir() |
| iid := "h.example.com" |
diff --git a/internal/instanceexec/exit.go b/internal/instanceexec/exit.go| index 9c5aa6f2d..3da028c71 100644 |
| --- a/internal/instanceexec/exit.go |
| +++ b/internal/instanceexec/exit.go |
| @@ -32,7 +32,7 @@ var invocationShape = regexp.MustCompile(`^[0-9a-f]{32}$`) |
| func RunExit(args []string) { |
| fs := flag.NewFlagSet("instance-exit", flag.ExitOnError) |
| runDir := fs.String("run-dir", provision.SocketDir, "run directory holding instances/<id>") |
| - fs.Usage = cliflag.Usage(fs, "usage: carlos instance-exit <run|seed|sidecar|cmd> <instance-id>") |
| + fs.Usage = cliflag.Usage(fs, "usage: carlos instance-exit <run|seed|sidecar|cmd|worker> <instance-id>") |
| _ = fs.Parse(args) |
| if fs.NArg() != 2 { |
| fs.Usage() |
| @@ -40,7 +40,7 @@ func RunExit(args []string) { |
| } |
| role, ok := instancectl.ParseRole(fs.Arg(0)) |
| if !ok { |
| - log.Printf("instance-exit: role %q is not run, seed, sidecar or cmd", fs.Arg(0)) |
| + log.Printf("instance-exit: role %q is not run, seed, sidecar, cmd or worker", fs.Arg(0)) |
| return |
| } |
| if err := writeExitRecord(*runDir, role, fs.Arg(1), os.Getenv, time.Now()); err != nil { |
diff --git a/internal/instanceexec/exit_test.go b/internal/instanceexec/exit_test.go| index 217f62bfe..216fdf5a3 100644 |
| --- a/internal/instanceexec/exit_test.go |
| +++ b/internal/instanceexec/exit_test.go |
| @@ -103,4 +103,15 @@ func TestWriteExitRecordKeysAGenerationRecordByGeneration(t *testing.T) { |
| if err := writeExitRecord(run, instancectl.RoleSidecar, iid+":7", env, now); err == nil { |
| t.Error("a sidecar generation was accepted") |
| } |
| + // Sidecar isolation D1 (#370): the worker role writes exit.worker.<inv> |
| + // and, having no generations, refuses "<iid>:<n>". |
| + if err := writeExitRecord(run, instancectl.RoleWorker, iid, env, now); err != nil { |
| + t.Fatalf("writeExitRecord(worker): %v", err) |
| + } |
| + if _, err := os.Stat(filepath.Join(run, "instances", iid, "exit.worker.deadbeefdeadbeefdeadbeefdeadbeef.json")); err != nil { |
| + t.Errorf("worker record missing: %v", err) |
| + } |
| + if err := writeExitRecord(run, instancectl.RoleWorker, iid+":7", env, now); err == nil { |
| + t.Error("a worker generation was accepted") |
| + } |
| } |