amadan
Merge remote-tracking branch 'origin/main' into feat/claim-root-trusted-state
6 files changed,
+372
−43
internal/activator/instancekey_test.go+65 −0internal/console/layoutguard_test.go+7 −4internal/host/agent.go+7 −3internal/host/instancekeys.go+72 −11internal/host/instancekeys_test.go+185 −11internal/host/layoutleg.go+36 −14
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 354.854s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.215s
ok github.com/carlosframework/platform/cmd/moveprobe 93.260s
ok github.com/carlosframework/platform/internal/activator 227.109s
ok github.com/carlosframework/platform/internal/agentsock 0.015s
ok github.com/carlosframework/platform/internal/alerting 0.004s
ok github.com/carlosframework/platform/internal/analytics 0.019s
ok github.com/carlosframework/platform/internal/appconfig 2.397s
ok github.com/carlosframework/platform/internal/apperr 0.074s
ok github.com/carlosframework/platform/internal/applog 0.131s
ok github.com/carlosframework/platform/internal/awsinv 0.004s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.649s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.201s
ok github.com/carlosframework/platform/internal/changelog 0.304s
ok github.com/carlosframework/platform/internal/clamd 0.054s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 367.867s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 5.774s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.005s
ok github.com/carlosframework/platform/internal/dnsrec 0.002s
ok github.com/carlosframework/platform/internal/docsite 0.004s
ok github.com/carlosframework/platform/internal/domains 2.217s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 26.089s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.126s
ok github.com/carlosframework/platform/internal/gate 1.657s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.016s
ok github.com/carlosframework/platform/internal/geoip 0.029s
ok github.com/carlosframework/platform/internal/host 90.942s
ok github.com/carlosframework/platform/internal/hostactors 0.552s
ok github.com/carlosframework/platform/internal/hostbox 2.218s
ok github.com/carlosframework/platform/internal/hostclaim 0.516s
ok github.com/carlosframework/platform/internal/hostevidence 0.015s
ok github.com/carlosframework/platform/internal/hostexec 4.972s
ok github.com/carlosframework/platform/internal/hostfence 0.034s
ok github.com/carlosframework/platform/internal/hostfiles 10.490s
ok github.com/carlosframework/platform/internal/hostfloor 0.014s
ok github.com/carlosframework/platform/internal/hostguard 0.870s
ok github.com/carlosframework/platform/internal/hostops 1.509s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.014s
ok github.com/carlosframework/platform/internal/hostquarantine 0.014s
ok github.com/carlosframework/platform/internal/hostrelocate 0.453s
ok github.com/carlosframework/platform/internal/hostresolve 0.139s
ok github.com/carlosframework/platform/internal/hostwrite 0.014s
ok github.com/carlosframework/platform/internal/infraview 0.014s
ok github.com/carlosframework/platform/internal/instancectl 1.730s
ok github.com/carlosframework/platform/internal/instanceexec 2.726s
ok github.com/carlosframework/platform/internal/instancefs 0.018s
ok github.com/carlosframework/platform/internal/instancekey 0.017s
ok github.com/carlosframework/platform/internal/instanceupdates 0.030s
ok github.com/carlosframework/platform/internal/layout 1.445s
ok github.com/carlosframework/platform/internal/lease 9.473s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.007s
ok github.com/carlosframework/platform/internal/ledger 0.017s
ok github.com/carlosframework/platform/internal/mail 0.168s
ok github.com/carlosframework/platform/internal/maintenance 0.089s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.564s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.081s
ok github.com/carlosframework/platform/internal/metrics 0.006s
ok github.com/carlosframework/platform/internal/moves 21.165s
ok github.com/carlosframework/platform/internal/moves/archive 0.769s
ok github.com/carlosframework/platform/internal/objstore 0.173s
ok github.com/carlosframework/platform/internal/ops 0.167s
ok github.com/carlosframework/platform/internal/park 0.204s
ok github.com/carlosframework/platform/internal/privatefile 0.004s
2026/10/13 14:38:50 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/13 14:38:50 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/13 14:38:50 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:38:50 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/13 14:38:50 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/13 14:38:50 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/13 14:38:50 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/13 14:38:50 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 14:38:50 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 14:38:50 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/13 14:38:50 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 14:38:50 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 14:38:50 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 14:38:50 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 14:38:50 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 14:38:50 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/13 14:38:50 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:38:50 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:38:50 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 14:38:50 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 14:38:50 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:38:50 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 14:38:50 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 14:38:50 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:50 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/13 14:38:51 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/13 14:38:51 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/13 14:38:51 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/13 14:38:51 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:38:51 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/13 14:38:51 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (10.32s)
plan14_root_reads_test.go:627: root read census: 7 discrepancies
changed root read internal/host/agent.go:runAgent; review class, task, sites and digest
changed root read internal/host/instancekeys.go:backfillInstanceKeys; review class, task, sites and digest
changed root read internal/host/layoutleg.go:activatorStart.Run; review class, task, sites and digest
changed root source envelope internal/host
unclassified root read internal/host/instancekeys.go:keyBackfillInSlot.BackfillInstanceKey registry:Host@199,registry:Host@258,whole:Route@351,whole:Route@386,whole:Route@550
changed legacy declaration internal/host/instancekeys.go:backfillInstanceKeys
changed legacy declaration internal/host/layoutleg.go:activatorStart.Run
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 14:39:02 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/13 14:39:02 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/13 14:39:02 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/13 14:39:02 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io hibernate false -> true
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged acme.go.tito.io status-named false -> true
2026/10/13 14:39:02 provision: converged acme.go.tito.io status-named true -> false
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 14:39:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 14:39:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 14:39:02 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 14:39:02 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/13 14:39:02 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/13 14:39:02 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/13 14:39:02 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/13 14:39:02 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/13 14:39:02 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 14:39:02 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 14:39:02 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/13 14:39:02 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/13 14:39:02 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:39:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:39:02 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/13 14:39:02 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:39:02 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/13 14:39:02 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/13 14:39:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:39:02 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/13 14:39:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 14:39:02 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/13 14:39:02 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:39:02 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 14:39:02 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:39:02 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/13 14:39:02 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/13 14:39:02 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/13 14:39:02 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:39:02 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 14:39:02 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/13 14:39:02 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/13 14:39:02 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/13 14:39:02 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/13 14:39:02 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/13 14:39:02 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/13 14:39:02 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/13 14:39:02 provision: dry-run: would converge owned.go.tito.io seed
2026/10/13 14:39:02 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:39:03 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/13 14:39:03 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/13 14:39:03 provision: converged owned.go.tito.io channel stable -> edge
2026/10/13 14:39:03 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/13 14:39:03 provision: converged owned.go.tito.io hibernate false -> true
2026/10/13 14:39:03 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/13 14:39:03 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/13 14:39:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: removed retired static route site.go.tito.io (record gone)
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/13 14:39:03 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/13 14:39:03 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/13 14:39:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/13 14:39:03 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/13 14:39:03 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 14:39:03 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/13 14:39:03 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/13 14:39:03 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/13 14:39:03 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 14:39:03 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/13 14:39:03 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/13 14:39:03 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/13 14:39:03 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 14:39:03 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/13 14:39:03 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/13 14:39:03 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/13 14:39:03 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/13 14:39:03 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/13 14:39:03 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/13 14:39:03 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/13 14:39:03 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/13 14:39:03 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/13 14:39:03 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/13 14:39:03 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/13 14:39:03 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/13 14:39:03 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/13 14:39:03 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/13 14:39:03 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/13 14:39:03 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 14:39:03 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/13 14:39:03 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 14:39:03 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 14:39:03 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/13 14:39:03 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/13 14:39:03 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:03 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/13 14:39:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:04 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 14:39:04 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 14:39:04 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/13 14:39:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 14:39:04 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 14:39:04 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/13 14:39:04 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL github.com/carlosframework/platform/internal/provision 13.691s
ok github.com/carlosframework/platform/internal/publicrelease 0.007s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 4.084s
ok github.com/carlosframework/platform/internal/relay 7.252s
ok github.com/carlosframework/platform/internal/release 20.370s
ok github.com/carlosframework/platform/internal/rollout 3.785s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/rootparticipation 0.008s
ok github.com/carlosframework/platform/internal/rootstate 0.025s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.326s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.061s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.370s
ok github.com/carlosframework/platform/internal/status 0.039s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.132s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.002s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.001s
ok github.com/carlosframework/platform/internal/vet 1.563s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/activator/instancekey_test.go b/internal/activator/instancekey_test.go| index 5bc223dfe..c632af017 100644 |
| --- a/internal/activator/instancekey_test.go |
| +++ b/internal/activator/instancekey_test.go |
| @@ -1002,3 +1002,68 @@ func TestBackfillInstanceKeyRefusesFencedAndConflictRows(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// v0.26.1 hotfix: the agent's backfill now runs beside the activator |
| +// rather than before it, so a backfill and a start's own mint can reach |
| +// the same host at once. Both read the files and ask a slow bucket outside |
| +// adminTokenMu, and both are held there until each has done so, so each |
| +// decides from a first read that found no key. They still decide and |
| +// append under the lock, create-only, so the host ends with exactly one |
| +// key, appended whole after the bytes that were there, and exactly one of |
| +// the two reports it minted. |
| +func TestBackfillAndAStartsMintRacingOnOneHostWriteOneKey(t *testing.T) { |
| + const rounds = 40 |
| + tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n" |
| + for i := range rounds { |
| + b := &ExecBackend{DataDir: t.TempDir()} |
| + b.ConfiguredKey = func(ctx context.Context, _ registry.Route) (string, error) { |
| + select { |
| + case <-time.After(5 * time.Millisecond): // a slow bucket that sets no key |
| + return "", nil |
| + case <-ctx.Done(): |
| + return "", ctx.Err() |
| + } |
| + } |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.sock"} |
| + own := filepath.Join(b.DataDir, rt.Host+".env") |
| + if err := os.WriteFile(own, []byte(tok), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Both callers pass phase one before either takes the lock. |
| + var phase sync.WaitGroup |
| + phase.Add(2) |
| + instanceKeyHooks.betweenPhases = func(string) { phase.Done(); phase.Wait() } |
| + var wg sync.WaitGroup |
| + var backfilled, started InstanceKeyOutcome |
| + var berr, serr error |
| + wg.Add(2) |
| + go func() { |
| + defer wg.Done() |
| + backfilled, berr = b.BackfillInstanceKey(t.Context(), rt) |
| + }() |
| + go func() { |
| + defer wg.Done() |
| + started, serr = b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt)) |
| + }() |
| + wg.Wait() |
| + instanceKeyHooks.betweenPhases = nil |
| + if berr != nil || serr != nil { |
| + t.Fatalf("round %d: backfill %v, %v; start %v, %v", i, backfilled, berr, started, serr) |
| + } |
| + minted := 0 |
| + for _, o := range []InstanceKeyOutcome{backfilled, started} { |
| + switch o { |
| + case InstanceKeyMinted: |
| + minted++ |
| + case InstanceKeyPresent: |
| + default: |
| + t.Fatalf("round %d: outcome %v, want minted or present", i, o) |
| + } |
| + } |
| + raw, _ := os.ReadFile(own) |
| + keys := keyLines(raw) |
| + if minted != 1 || len(keys) != 1 || !wellFormedInstanceKey(keys[0]) || string(raw) != tok+InstanceKeyVar+"="+keys[0]+"\n" { |
| + t.Fatalf("round %d: %d minted, %d keys, file %d bytes; want one whole key appended once", i, minted, len(keys), len(raw)) |
| + } |
| + } |
| +} |
diff --git a/internal/console/layoutguard_test.go b/internal/console/layoutguard_test.go| index 2154ddcb3..b05056134 100644 |
| --- a/internal/console/layoutguard_test.go |
| +++ b/internal/console/layoutguard_test.go |
| @@ -106,10 +106,13 @@ func TestSystemDoorsRefuseAPreLayoutMoveWhileABoxIsNested(t *testing.T) { |
| t.Fatalf("update to v0.19.0 with a nested row = %d %v, want 409", st, out) |
| } |
| // A version at the floor or above is not the guard's business; the |
| - // request goes on to the job (and fails there, on this fixture's |
| - // absent feed, which is not what this test is about). |
| - if st, _ := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v0.26.0"}, owner); st == http.StatusConflict { |
| - t.Error("update to v0.26.0 was refused by the layout guard") |
| + // request goes on to the job (and fails there, which is not what this |
| + // test is about). The fixture has no SystemFeed, so the job asks the |
| + // vendor feed: the version must be one that can never be published, or |
| + // the job succeeds once it is (v0.26.0 was, on 2026-10-13), moves the |
| + // pointer, and the rollback below lands on the wrong version. |
| + if st, _ := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v99.0.0"}, owner); st == http.StatusConflict { |
| + t.Error("update to v99.0.0 was refused by the layout guard") |
| } |
| // Every row flat: the rollback goes through. |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 50343c9e7..a817678d8 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -548,9 +548,11 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // worse off than before this existed (a leftover unit that keeps |
| // running is exactly what an agent restart left before this pass). |
| // |
| - // Then, on a legacy box, layout recovery and the instance-key |
| - // backfill, before Start can wake anything; on a claim box, Start |
| - // alone. The order and the branch are activatorStart's. |
| + // Then, on a legacy box, layout recovery before Start can wake |
| + // anything, and the instance-key backfill in the background after |
| + // it, which this does not wait for (the edge below must listen |
| + // within the update gate, whatever the box's size); on a claim |
| + // box, Start alone. The order and the branch are activatorStart's. |
| activatorStart{ |
| Claims: claims.Writes.Enabled, |
| Sweep: func(ctx context.Context) error { |
| @@ -569,7 +571,9 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| return pass |
| }, |
| Routes: reg.List, |
| + Route: reg.Get, |
| Keys: eb, |
| + Slot: svc.PrepareScheduledToken, |
| Publish: armedLayoutPass.Store, |
| Start: svc.Start, |
| }.Run(context.Background()) |
diff --git a/internal/host/instancekeys.go b/internal/host/instancekeys.go| index d79ea90f4..74b19c6c0 100644 |
| --- a/internal/host/instancekeys.go |
| +++ b/internal/host/instancekeys.go |
| @@ -3,6 +3,7 @@ package host |
| import ( |
| "context" |
| "errors" |
| + "fmt" |
| "log" |
| "time" |
| @@ -13,9 +14,11 @@ import ( |
| // The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md |
| // ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the |
| // env, so no process is ever started without one. The backfill gives every |
| -// instance already on a box its key now, at agent start, rather than at its |
| -// next start, so the rollout can be checked box by box before any app adopts |
| -// the key: a host that the backfill names "present" or "minted" is ready. |
| +// instance already on a box its key now, just after agent start, rather |
| +// than at its next start, so the rollout can be checked box by box before |
| +// any app adopts the key: a host that the backfill names "present" or |
| +// "minted" is ready. It runs in the background once the activator has |
| +// started (activatorStart.Run), never on the path the edge waits for. |
| // |
| // It runs only where the legacy activator writes <host>.env itself. With |
| // host claims on, every write to a host's installation runs under the |
| @@ -31,12 +34,64 @@ type instanceKeyBackfiller interface { |
| } |
| // instanceKeyBackfillBudget bounds one pass. Each keyless row costs one |
| -// read of the bucket's configuration before its key is minted, and the |
| -// pass runs before the activator starts, so with the bucket unreachable |
| -// the rows after the budget fail fast rather than each waiting out its |
| -// own read; every one of them mints at its next start. |
| +// read of the bucket's configuration before its key is minted. The pass |
| +// runs in the background after the activator starts (activatorStart.Run), |
| +// so it never holds the edge up, but with the bucket unreachable the rows |
| +// after the budget still fail fast rather than each waiting out its own |
| +// read; every one of them mints at its next start. |
| const instanceKeyBackfillBudget = time.Minute |
| +// hostPreparationSlot runs prepare in host's preparation slot, ordered |
| +// with the host's maintenance fence: (*activator.Service). |
| +// PrepareScheduledToken. It answers activator.ErrFenced without running |
| +// prepare when a fence is held for host, and a fence asked for while |
| +// prepare runs is installed only after it returns. Wakes do not wait on |
| +// the slot. |
| +type hostPreparationSlot func(host string, prepare func() (string, error)) (string, error) |
| + |
| +// errInstanceKeyRowSkipped is a row the backfill listed and, when its turn |
| +// came, did not write for: an operation held its host fenced, or its row |
| +// had changed (a move, a layout flip, a removal) so it is no longer one |
| +// the backfill selects. Counted with the skipped rows; its next start |
| +// mints the key if it still needs one. |
| +var errInstanceKeyRowSkipped = errors.New("skipped") |
| + |
| +// keyBackfillInSlot is the backfill's step for one row once the activator |
| +// is serving (the v0.26.1 hotfix moved the pass off the startup path). The |
| +// layout pass and every move fence a host before they move its files, and |
| +// install that fence only after any preparation already in the slot has |
| +// returned; the slot refuses a host already fenced. So inside the slot no |
| +// file of the host moves, and the row is read again there, so the write |
| +// goes to the paths the host has now, not those of the list taken before. |
| +// A start's own mint is not ordered by the slot: it and the backfill both |
| +// decide and append under the activator's adminTokenMu, create-only, so |
| +// whichever is second finds the key present. |
| +type keyBackfillInSlot struct { |
| + keys instanceKeyBackfiller |
| + slot hostPreparationSlot |
| + route func(string) (*registry.Route, error) |
| +} |
| + |
| +func (g keyBackfillInSlot) BackfillInstanceKey(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + got, err := activator.InstanceKeyFailed, error(nil) |
| + _, serr := g.slot(rt.Host, func() (string, error) { |
| + fresh, rerr := g.route(rt.Host) |
| + switch { |
| + case rerr != nil: |
| + err = fmt.Errorf("read its row: %w", rerr) |
| + case fresh == nil || !instanceKeyRoute(*fresh): |
| + err = fmt.Errorf("%w: its row changed after the backfill listed it", errInstanceKeyRowSkipped) |
| + default: |
| + got, err = g.keys.BackfillInstanceKey(ctx, *fresh) |
| + } |
| + return "", nil |
| + }) |
| + if serr != nil { |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: an operation holds it (%v)", errInstanceKeyRowSkipped, serr) |
| + } |
| + return got, err |
| +} |
| + |
| // instanceKeyRoute selects the rows the backfill writes for: ready, |
| // activator-spawned instances, the rows execSyncable delivers config to, |
| // without sidecars and without replicas (a replica is never minted a key, |
| @@ -58,10 +113,12 @@ type instanceKeyBackfill struct { |
| } |
| // backfillInstanceKeys runs the backfill over routes, within |
| -// instanceKeyBackfillBudget. It never fails the agent: a host it cannot |
| -// write is logged and counted, and that host's next start mints its key (or |
| -// refuses, loudly, if the same fault is still there). Nothing logged names |
| -// a key. |
| +// instanceKeyBackfillBudget, and logs one count line when it ends, |
| +// "agent: instance keys: ...": on a legacy box that is after the activator |
| +// has started, within the budget plus one row's read. It never fails the |
| +// agent: a host it cannot write is logged and counted, and that host's |
| +// next start mints its key (or refuses, loudly, if the same fault is |
| +// still there). Nothing logged names a key. |
| // |
| // The error is read before the outcome: a refused or failed row answers |
| // InstanceKeyFailed (the zero outcome) beside its error, and counting an |
| @@ -77,6 +134,10 @@ func backfillInstanceKeys(ctx context.Context, routes []registry.Route, b instan |
| } |
| got, err := b.BackfillInstanceKey(ctx, rt) |
| switch { |
| + case errors.Is(err, errInstanceKeyRowSkipped): |
| + st.Skipped++ |
| + log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err) |
| + continue |
| case errors.Is(err, activator.ErrInstanceKeyDeferred): |
| st.Deferred++ |
| log.Printf("agent: instance key for %s: %v", rt.Host, err) |
diff --git a/internal/host/instancekeys_test.go b/internal/host/instancekeys_test.go| index 1b6c63339..f09c8ac12 100644 |
| --- a/internal/host/instancekeys_test.go |
| +++ b/internal/host/instancekeys_test.go |
| @@ -5,6 +5,8 @@ import ( |
| "errors" |
| "fmt" |
| "reflect" |
| + "sync" |
| + "sync/atomic" |
| "testing" |
| "time" |
| @@ -121,14 +123,29 @@ func (f keyBackfillFunc) BackfillInstanceKey(ctx context.Context, rt registry.Ro |
| return f(ctx, rt) |
| } |
| -// Task 6 ruling (K1): the start sequence is activatorStart's, so the two |
| -// guards runAgent cannot be tested for are pinned here. A claim box starts |
| -// and nothing else: no sweep, no recovery, no backfill (ruling 7). A legacy |
| -// box sweeps, recovers, backfills, publishes the pass and then starts, in |
| -// that order: the backfill after every journal is settled and before the |
| -// pass is published or anything wakes. A route list that fails skips the |
| -// backfill only; the pass is still published and the service still starts. |
| -func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testing.T) { |
| +// openSlot is a preparation slot with no fence held: it runs prepare. |
| +func openSlot(_ string, prepare func() (string, error)) (string, error) { return prepare() } |
| + |
| +// routeOf answers the registry's Get from a fixed list. |
| +func routeOf(routes ...registry.Route) func(string) (*registry.Route, error) { |
| + return func(host string) (*registry.Route, error) { |
| + for _, rt := range routes { |
| + if rt.Host == host { |
| + return &rt, nil |
| + } |
| + } |
| + return nil, nil |
| + } |
| +} |
| + |
| +// Task 6 ruling (K1), as the v0.26.1 hotfix changed it: the start sequence |
| +// is activatorStart's, so the two guards runAgent cannot be tested for are |
| +// pinned here. A claim box starts and nothing else: no sweep, no recovery, |
| +// no backfill (ruling 7). A legacy box sweeps, recovers, publishes the |
| +// pass and starts, in that order, and the backfill runs after Start, once |
| +// every journal is settled. A route list that fails skips the backfill |
| +// only. |
| +func TestActivatorStartBackfillsOnlyALegacyBoxAfterRecoveryAndStart(t *testing.T) { |
| ready := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| for _, tc := range []struct { |
| name string |
| @@ -136,28 +153,185 @@ func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testi |
| listErr error |
| want []string |
| }{ |
| - {"legacy box", false, nil, []string{"sweep", "recover", "backfill", "publish", "start"}}, |
| + {"legacy box", false, nil, []string{"sweep", "recover", "publish", "start", "backfill"}}, |
| {"legacy box, routes unreadable", false, errors.New("db locked"), []string{"sweep", "recover", "publish", "start"}}, |
| {"claim box", true, nil, []string{"start"}}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
| + var mu sync.Mutex |
| var events []string |
| - note := func(e string) { events = append(events, e) } |
| - activatorStart{ |
| + note := func(e string) { mu.Lock(); events = append(events, e); mu.Unlock() } |
| + done := activatorStart{ |
| Claims: tc.claims, |
| Sweep: func(context.Context) error { note("sweep"); return nil }, |
| Recover: func(context.Context, bool) *layout.Pass { note("recover"); return nil }, |
| Routes: func() ([]registry.Route, error) { return []registry.Route{ready}, tc.listErr }, |
| + Route: routeOf(ready), |
| Keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| note("backfill") |
| return activator.InstanceKeyMinted, nil |
| }), |
| + Slot: openSlot, |
| Publish: func(*layout.Pass) { note("publish") }, |
| Start: func() { note("start") }, |
| }.Run(context.Background()) |
| + select { |
| + case <-done: |
| + case <-time.After(10 * time.Second): |
| + t.Fatal("the backfill never finished") |
| + } |
| + mu.Lock() |
| + defer mu.Unlock() |
| if !reflect.DeepEqual(events, tc.want) { |
| t.Fatalf("events = %v, want %v", events, tc.want) |
| } |
| }) |
| } |
| } |
| + |
| +// fenceCtx is a context with the deadline FenceAndStop requires. |
| +func fenceCtx(t *testing.T, d time.Duration) context.Context { |
| + ctx, cancel := context.WithTimeout(t.Context(), d) |
| + t.Cleanup(cancel) |
| + return ctx |
| +} |
| + |
| +// Off the startup path, the backfill shares the box with the layout pass |
| +// and moves, which fence a host before they move its files. A host that |
| +// is fenced when its turn comes is skipped and nothing is asked of it, and |
| +// a row whose status changed after the list (it began a move) is skipped |
| +// the same way; its next start mints the key. The rest are written from |
| +// the row as it stands, not as it was listed. |
| +func TestBackfillInSlotSkipsAFencedHostAndReadsTheRowAgain(t *testing.T) { |
| + ready := func(host string) registry.Route { |
| + return registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"} |
| + } |
| + fenced, moved, shop := ready("fenced.example.com"), ready("moved.example.com"), ready("shop.example.com") |
| + svc := &activator.Service{} |
| + if _, err := svc.FenceAndStop(fenceCtx(t, time.Second), fenced.Host, 1); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Fatalf("fence: %v", err) |
| + } |
| + nowMoving := moved |
| + nowMoving.Status = "moving" |
| + nowShop := shop |
| + nowShop.Socket = "/run/carlos/nested/shop.example.com.sock" |
| + f := &fakeKeyBackfiller{got: map[string]activator.InstanceKeyOutcome{shop.Host: activator.InstanceKeyMinted}} |
| + var sockets []string |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + sockets = append(sockets, rt.Socket) |
| + return f.BackfillInstanceKey(ctx, rt) |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(fenced, nowMoving, nowShop), |
| + } |
| + st := backfillInstanceKeys(t.Context(), []registry.Route{fenced, moved, shop}, g) |
| + if !reflect.DeepEqual(f.asked, []string{shop.Host}) { |
| + t.Fatalf("asked %v, want only %s", f.asked, shop.Host) |
| + } |
| + if !reflect.DeepEqual(sockets, []string{nowShop.Socket}) { |
| + t.Fatalf("written from %v, want the row as it stands now", sockets) |
| + } |
| + if st != (instanceKeyBackfill{Minted: 1, Skipped: 2}) { |
| + t.Fatalf("stats = %+v, want one minted and two skipped", st) |
| + } |
| +} |
| + |
| +// The other half of "nothing moves a file under it": a fence asked for |
| +// while a row is being written (the layout pass or a move about to move |
| +// the host's files) is installed only once the write has returned. |
| +func TestBackfillInSlotHoldsOffAFenceUntilTheWriteReturns(t *testing.T) { |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| + svc := &activator.Service{} |
| + writing, release := make(chan struct{}), make(chan struct{}) |
| + var wrote atomic.Bool |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| + close(writing) |
| + <-release |
| + wrote.Store(true) |
| + return activator.InstanceKeyMinted, nil |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(rt), |
| + } |
| + done := make(chan instanceKeyBackfill) |
| + go func() { done <- backfillInstanceKeys(t.Context(), []registry.Route{rt}, g) }() |
| + <-writing |
| + fenced := make(chan bool) |
| + go func() { |
| + _, err := svc.FenceAndStopIfUnfenced(fenceCtx(t, 10*time.Second), rt.Host, 1) |
| + if err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Errorf("fence: %v", err) |
| + } |
| + fenced <- wrote.Load() |
| + }() |
| + select { |
| + case <-fenced: |
| + t.Fatal("the fence was installed while the backfill was writing the host") |
| + case <-time.After(200 * time.Millisecond): |
| + } |
| + close(release) |
| + if !<-fenced { |
| + t.Fatal("the fence was installed before the write returned") |
| + } |
| + if st := <-done; st != (instanceKeyBackfill{Minted: 1}) { |
| + t.Fatalf("stats = %+v", st) |
| + } |
| +} |
| + |
| +// v0.26.1 hotfix: the v0.26.0 roll to a 127-instance box failed its update |
| +// gate because the backfill ran serially before Start, one bucket read per |
| +// keyless row, and the edge never listened inside the gate's 20 seconds. |
| +// On a box with hundreds of keyless rows and a slow bucket, Start is |
| +// called, and Run returns (runAgent goes on to the agent socket and the |
| +// edge), without waiting for the backfill. |
| +func TestActivatorStartDoesNotWaitForTheBackfillOnALargeBox(t *testing.T) { |
| + const rows, perRow, bound = 300, 30 * time.Millisecond, 2 * time.Second |
| + var routes []registry.Route |
| + for i := range rows { |
| + host := fmt.Sprintf("shop%03d.example.com", i) |
| + routes = append(routes, registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"}) |
| + } |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + defer cancel() |
| + slowBucket := keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + select { |
| + case <-time.After(perRow): |
| + return activator.InstanceKeyMinted, nil |
| + case <-ctx.Done(): |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: %w", activator.ErrInstanceKeyDeferred, ctx.Err()) |
| + } |
| + }) |
| + began := time.Now() |
| + var startedAfter time.Duration |
| + done := activatorStart{ |
| + Sweep: func(context.Context) error { return nil }, |
| + Recover: func(context.Context, bool) *layout.Pass { return nil }, |
| + Routes: func() ([]registry.Route, error) { return routes, nil }, |
| + Route: routeOf(routes...), |
| + Keys: slowBucket, |
| + Slot: (&activator.Service{}).PrepareScheduledToken, |
| + Publish: func(*layout.Pass) {}, |
| + Start: func() { startedAfter = time.Since(began) }, |
| + }.Run(ctx) |
| + returned := time.Since(began) |
| + if startedAfter == 0 || startedAfter > bound { |
| + t.Errorf("Start was called %v after Run began, want within %v (the backfill must not hold it)", startedAfter, bound) |
| + } |
| + if returned > bound { |
| + t.Errorf("Run returned after %v, want within %v: the edge waits for Run", returned, bound) |
| + } |
| + select { |
| + case <-done: |
| + t.Fatal("the backfill had finished when Run returned; it should still be running") |
| + default: |
| + } |
| + // Agent shutdown is not this test's: cancel the pass and let it end. |
| + cancel() |
| + select { |
| + case <-done: |
| + case <-time.After(30 * time.Second): |
| + t.Fatal("the backfill did not end once its context was cancelled") |
| + } |
| +} |
diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go| index 2abec0341..fabd8861d 100644 |
| --- a/internal/host/layoutleg.go |
| +++ b/internal/host/layoutleg.go |
| @@ -354,36 +354,58 @@ func startActivator(ctx context.Context, sweep func(context.Context) error, reco |
| // |
| // On a legacy box it is the sweep, then layout recovery (design §8.2, |
| // "Recovery is the journal": every journal a previous agent left is |
| -// completed and the host converged to the fleet's intent), then the |
| -// instance-key backfill (L21), then the pass's publication, then Start. The |
| -// backfill runs after recovery has settled every journal and before the |
| -// pass is published or the activator wakes anything, so nothing moves a |
| -// file under it; the pass is published only after recovery, so the serial |
| -// leg never runs beside it. |
| +// completed and the host converged to the fleet's intent), then the pass's |
| +// publication, then Start. The pass is published only after recovery, so |
| +// the serial leg never runs beside it. |
| +// |
| +// The instance-key backfill (L21) runs after Start, in the background, and |
| +// Run does not wait for it (the v0.26.1 hotfix). Before, it ran between |
| +// recovery and the publication, one bucket read per keyless row, and on a |
| +// box with 127 instances the edge, which comes up after Run returns, did |
| +// not listen within the update gate's 20 seconds. The backfill still runs |
| +// after recovery has settled every journal. What kept anything from moving |
| +// a file under it while it ran first, nothing else running yet, is now |
| +// the activator's per-host preparation slot (keyBackfillInSlot): each row |
| +// is read again and written inside it; a host that an operation holds |
| +// fenced (a layout pass, a move) is skipped, and a fence asked for while |
| +// the row is written waits until the write is done. A start's own mint |
| +// beside the backfill is serialized by the activator's adminTokenMu, and |
| +// both are create-only, so the host ends with one key. |
| type activatorStart struct { |
| Claims bool // claims.Writes.Enabled: a claim box |
| Sweep func(context.Context) error // the instance-unit sweep |
| Recover func(context.Context, bool) *layout.Pass // builds the pass and recovers under it; the bool is whether the sweep succeeded |
| Routes func() ([]registry.Route, error) // the registry's List |
| + Route func(string) (*registry.Route, error) // the registry's Get: each row is read again in its host's slot |
| Keys instanceKeyBackfiller // the exec backend |
| + Slot hostPreparationSlot // the service's PrepareScheduledToken |
| Publish func(*layout.Pass) // armedLayoutPass.Store |
| Start func() // the service's Start |
| } |
| -func (s activatorStart) Run(ctx context.Context) { |
| +// Run starts the activator and returns. The channel closes when the |
| +// instance-key backfill has finished; on a claim box it is closed already. |
| +// runAgent does not wait on it. |
| +func (s activatorStart) Run(ctx context.Context) <-chan struct{} { |
| + done := make(chan struct{}) |
| if s.Claims { |
| s.Start() |
| - return |
| + close(done) |
| + return done |
| } |
| startActivator(ctx, s.Sweep, func(ctx context.Context, swept bool) { |
| - pass := s.Recover(ctx, swept) |
| - if routes, err := s.Routes(); err != nil { |
| + s.Publish(s.Recover(ctx, swept)) |
| + }, s.Start) |
| + go func() { |
| + defer close(done) |
| + routes, err := s.Routes() |
| + if err != nil { |
| log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err) |
| - } else { |
| - backfillInstanceKeys(ctx, routes, s.Keys) |
| + return |
| } |
| - s.Publish(pass) |
| - }, s.Start) |
| + backfillInstanceKeys(ctx, routes, keyBackfillInSlot{keys: s.Keys, slot: s.Slot, route: s.Route}) |
| + }() |
| + return done |
| } |
| // layoutIntentTimeout bounds the one intent read at startup, which the |