amadan
host: config-gen, canary pointers and telemetry read only on change or at the backstop age (#336)
config-gen.json, the fast sweep's canary pointers and telemetry.json are read only when their scope generation moved, the last read failed, or it is the backstop's age. A nil or unusable watch makes every read due. release.WithPointerGate lets the sweep skip settled canary groups; telemetry reads each account once per pass and keeps the last good value.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
8 files changed,
+844
−15
internal/host/agent.go+103 −3internal/host/changeindex.go+15 −2internal/host/changeindex_test.go+11 −1internal/host/readgate_test.go+464 −0internal/host/telemetry_pass.go+108 −4internal/release/adopt.go+69 −0internal/release/adopt_test.go+62 −0internal/telemetry/telemetry.go+12 −5
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok amadan.net/carlos/platform.git/launchctx 0.012s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (23 s)
=== gofmt (1 s)
gofmt -l .: clean
=== rest (164 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.840s
ok github.com/carlosframework/platform/cmd/moveprobe 12.388s
ok github.com/carlosframework/platform/cmd/zddprobe 0.888s
ok github.com/carlosframework/platform/internal/activator 63.938s
ok github.com/carlosframework/platform/internal/agentsock 0.016s
ok github.com/carlosframework/platform/internal/alerting 0.003s
ok github.com/carlosframework/platform/internal/analytics 0.033s
ok github.com/carlosframework/platform/internal/appconfig 3.127s
ok github.com/carlosframework/platform/internal/apperr 0.192s
ok github.com/carlosframework/platform/internal/applog 0.380s
ok github.com/carlosframework/platform/internal/awsinv 0.008s
ok github.com/carlosframework/platform/internal/boxchoice 0.005s
ok github.com/carlosframework/platform/internal/boxid 1.383s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.014s
ok github.com/carlosframework/platform/internal/caps 0.024s
ok github.com/carlosframework/platform/internal/certwarm 0.006s
ok github.com/carlosframework/platform/internal/cgroup 1.063s
ok github.com/carlosframework/platform/internal/changelog 0.889s
ok github.com/carlosframework/platform/internal/claimactivation 1.665s
ok github.com/carlosframework/platform/internal/clamd 0.062s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 5.811s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.012s
ok github.com/carlosframework/platform/internal/dbverify 0.017s
ok github.com/carlosframework/platform/internal/dnsrec 0.007s
ok github.com/carlosframework/platform/internal/docsite 0.012s
ok github.com/carlosframework/platform/internal/domains 3.145s
ok github.com/carlosframework/platform/internal/doorkey 0.008s
ok github.com/carlosframework/platform/internal/economics 0.008s
ok github.com/carlosframework/platform/internal/edge 45.824s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.131s
ok github.com/carlosframework/platform/internal/gate 1.659s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.017s
ok github.com/carlosframework/platform/internal/geoip 0.029s
ok github.com/carlosframework/platform/internal/historyreadback 0.155s
ok github.com/carlosframework/platform/internal/host 129.782s
ok github.com/carlosframework/platform/internal/hostactors 7.251s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.007s
ok github.com/carlosframework/platform/internal/hostbox 3.921s
ok github.com/carlosframework/platform/internal/hostclaim 1.223s
ok github.com/carlosframework/platform/internal/hostenv 0.006s
ok github.com/carlosframework/platform/internal/hostevidence 0.006s
ok github.com/carlosframework/platform/internal/hostexec 7.122s
ok github.com/carlosframework/platform/internal/hostfence 0.083s
ok github.com/carlosframework/platform/internal/hostfiles 11.231s
ok github.com/carlosframework/platform/internal/hostfloor 0.048s
ok github.com/carlosframework/platform/internal/hostguard 1.098s
ok github.com/carlosframework/platform/internal/hostops 3.156s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.007s
ok github.com/carlosframework/platform/internal/hostquarantine 0.004s
ok github.com/carlosframework/platform/internal/hostrelocate 0.978s
ok github.com/carlosframework/platform/internal/hostresolve 0.290s
ok github.com/carlosframework/platform/internal/hostwrite 0.005s
ok github.com/carlosframework/platform/internal/infraview 0.006s
ok github.com/carlosframework/platform/internal/instancectl 5.303s
ok github.com/carlosframework/platform/internal/instanceexec 3.007s
ok github.com/carlosframework/platform/internal/instancefs 0.037s
ok github.com/carlosframework/platform/internal/instancekey 0.004s
ok github.com/carlosframework/platform/internal/instanceupdates 0.014s
ok github.com/carlosframework/platform/internal/layout 2.771s
ok github.com/carlosframework/platform/internal/lbhealth 0.008s
ok github.com/carlosframework/platform/internal/lease 10.281s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.742s
ok github.com/carlosframework/platform/internal/ledger 0.058s
ok github.com/carlosframework/platform/internal/mail 0.462s
ok github.com/carlosframework/platform/internal/maintenance 0.108s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.776s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.220s
ok github.com/carlosframework/platform/internal/metrics 0.008s
ok github.com/carlosframework/platform/internal/moves 23.964s
ok github.com/carlosframework/platform/internal/moves/archive 2.072s
ok github.com/carlosframework/platform/internal/objstore 0.454s
ok github.com/carlosframework/platform/internal/ops 0.286s
ok github.com/carlosframework/platform/internal/park 0.431s
ok github.com/carlosframework/platform/internal/plans 0.023s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
2026/10/22 10:36:05 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:05 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:05 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:05 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:05 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/22 10:36:05 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:05 provision: an app's records could not be listed: they are skipped
2026/10/22 10:36:05 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:05 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/22 10:36:05 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/22 10:36:05 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/22 10:36:05 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/22 10:36:05 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/22 10:36:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:05 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/22 10:36:05 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/22 10:36:05 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/22 10:36:05 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/22 10:36:05 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/22 10:36:05 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/22 10:36:06 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/22 10:36:06 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/22 10:36:06 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/22 10:36:06 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/22 10:36:06 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/22 10:36:06 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/22 10:36:06 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/22 10:36:06 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/22 10:36:06 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:06 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:06 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:06 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:06 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/22 10:36:06 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/22 10:36:06 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/22 10:36:07 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/22 10:36:07 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:07 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:07 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan9B8RecordWriteCensus (1.31s)
plan9_b8_census_test.go:372: record-write census rejected 15 site(s); update the explicit inventory only after reviewing the source:
changed site internal/provision/reconcile.go:Reconcile: source differs from the reviewed digest
changed site internal/provision/sidecar_reconcile.go:ReconcileSidecars: source differs from the reviewed digest
changed site internal/release/store.go:DirStore.Put: source differs from the reviewed digest
changed site internal/release/store.go:DirStore.PutFrom: source differs from the reviewed digest
changed site internal/release/store.go:S3Store.Put: source differs from the reviewed digest
changed site internal/release/store.go:S3Store.PutFrom: source differs from the reviewed digest
unlisted site internal/provision/recordview.go:RecordView.readApp [key:SidecarManifestKey]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview.go:RecordView.readScope [lit:instances,lit:sidecars]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview.go:RecordView.walk [lit:instances,lit:sidecars]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview.go:walkComplete [lit:instances,lit:sidecars]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview_store.go:ViewSnapshot.list [lit:instances,lit:sidecars]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview_store.go:ViewSnapshot.lookup [lit:instances,lit:sidecars]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview_store.go:snapStore.Put [def:Put,sel:Put]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview_store.go:snapStore.PutFrom [def:PutFrom,sel:PutObjectFrom]: review it, classify it, and update its explicit inventory entry
unlisted site internal/provision/recordview_store.go:uncachedStore.Put [def:Put]: review it, classify it, and update its explicit inventory entry
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io channel stable -> edge
2026/10/22 10:36:09 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/22 10:36:09 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/22 10:36:09 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/22 10:36:09 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io hibernate false -> true
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged acme.go.tito.io status-named false -> true
2026/10/22 10:36:09 provision: converged acme.go.tito.io status-named true -> false
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/22 10:36:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/22 10:36:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/22 10:36:09 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/22 10:36:09 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/22 10:36:09 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/22 10:36:09 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/22 10:36:09 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/22 10:36:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:09 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/22 10:36:09 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:10 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/22 10:36:10 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:10 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/22 10:36:10 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/22 10:36:10 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/22 10:36:10 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:10 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/22 10:36:10 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:10 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/22 10:36:10 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/22 10:36:10 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:10 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/22 10:36:10 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:10 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/22 10:36:10 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: converged acme.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/22 10:36:10 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/22 10:36:10 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/22 10:36:10 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/22 10:36:10 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/22 10:36:10 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/22 10:36:10 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/22 10:36:10 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/22 10:36:10 provision: dry-run: would converge owned.go.tito.io seed
2026/10/22 10:36:10 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/22 10:36:10 provision: converged owned.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/22 10:36:10 provision: converged owned.go.tito.io hibernate false -> true
2026/10/22 10:36:10 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/22 10:36:10 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: removed retired static route site.go.tito.io (record gone)
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/22 10:36:10 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/22 10:36:10 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:10 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/22 10:36:10 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/22 10:36:10 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/22 10:36:10 provision: converged acme.go.tito.io channel stable -> edge
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/22 10:36:10 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/22 10:36:10 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/22 10:36:10 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/22 10:36:10 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/22 10:36:10 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/22 10:36:10 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/22 10:36:10 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/22 10:36:10 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/22 10:36:10 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/22 10:36:10 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/22 10:36:11 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/22 10:36:11 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/22 10:36:11 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/22 10:36:11 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:11 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:11 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:11 provision: not removing acme.go.tito.io: the record view lacks its record but the bucket has it; the view catches up at its next refresh or rebuild
2026/10/22 10:36:11 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/22 10:36:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:12 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/22 10:36:12 provision: not removing bac/tito's rows: the record view has its trash marker but the bucket does not (<nil>)
2026/10/22 10:36:12 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/22 10:36:12 provision: not removing sidecar bot.shop.bac.sidecar.internal: the record view lacks its record but the bucket has it; the view catches up at its next refresh or rebuild
2026/10/22 10:36:12 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/22 10:36:12 provision: a.shop.example.com: swapped for import sd_yrjaj6ubwdcypv4bvb4lkhak7u (the original route is journaled)
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_yrjaj6ubwdcypv4bvb4lkhak7u committed: its gate is cleared and its journal row dropped
2026/10/22 10:36:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_uh2h2oml3ifqg24ues6vpfczui rolled back to its original route
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_uh2h2oml3ifqg24ues6vpfczui: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_uh2h2oml3ifqg24ues6vpfczui is cleaned up and its journal row retired
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_qcie2a5jazcrxxdh4azirkby44 rolled back to its original route
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_qcie2a5jazcrxxdh4azirkby44: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_qcie2a5jazcrxxdh4azirkby44 is cleaned up and its journal row retired
2026/10/22 10:36:12 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_szvdqjzblvpxye2crpppfri7dq
2026/10/22 10:36:12 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/22 10:36:12 provision: dry-run: would converge a.shop.example.com seed source
2026/10/22 10:36:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:12 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/22 10:36:12 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/22 10:36:12 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_5rxo75evfe3h7oqt5pvbfrgzmy cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_ov37scff4lx4sa7mobayciabyq cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_ms2yxs7ksrbrdoq7gr5rgjvknm is journaled, but this box cannot act on imports; its row is left alone
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_ojgjqja7b42blv74hxf3ooliy4 is journaled, but this box cannot act on imports; its row is left alone
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_umzm3oli7llviuv3hofkhqtxku is journaled, but this box cannot act on imports; its row is left alone
2026/10/22 10:36:12 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/22 10:36:12 provision: b.shop.example.com: import sd_3w56d7npzw3ix2une2b4pwzgvq left alone this pass: the import journal could not be read
2026/10/22 10:36:12 provision: c.shop.example.com: import sd_z67axgbsyyei2josexbb55i6vq left alone this pass: the import journal could not be read
2026/10/22 10:36:12 provision: a.shop.example.com: import sd_psnjkrfkpqzjizh5r7inttqb24 committed: its gate is cleared and its journal row dropped
2026/10/22 10:36:12 provision: a.shop.example.com: not converging import sd_fd2pea7teu4e4pfyyw3x2j4uea: it is swapped but not committed, and the row on this box has no import gate
2026/10/22 10:36:12 provision: a.shop.example.com: not converging import sd_evsgaxbgg4lpvc65lraf6sgwiq: it is swapped but not committed, and the row on this box has no import gate
2026/10/22 10:36:12 provision: a.shop.example.com: not converging import sd_disrlu2qakbfxoizeju54a7vcq: it is placed but not committed, and the row on this box has no import gate
2026/10/22 10:36:12 provision: a.shop.example.com: not converging import sd_blttcviqardl7kvinaw6mv5xjm: it is placed but not committed, and the row on this box has no import gate
2026/10/22 10:36:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:12 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/22 10:36:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_u44apgsolrhfvimfm77mnzkpra committed: its gate is cleared and its journal row dropped
2026/10/22 10:36:13 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:13 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:13 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/22 10:36:13 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_5vs3f5jj5eo2cevxipf6f3cavy: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_5vs3f5jj5eo2cevxipf6f3cavy rolled back (no box had swapped it)
2026/10/22 10:36:13 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/22 10:36:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:13 provision: removed instance route acme.go.tito.io (its seed sd_pnpwuxziinustqreh2gpgvdvma was aborted); the next pass rolls it back
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_pnpwuxziinustqreh2gpgvdvma: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_pnpwuxziinustqreh2gpgvdvma rolled back (no box had swapped it)
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_l6ql6reipt6jsqrd7jiwfrxwgi has a row this pass may not remove; leaving the seed aborted
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_doip5zueg2iyytgvekypcvzhxe rolled back (no box had swapped it)
2026/10/22 10:36:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:13 provision: removed instance route acme.go.tito.io (its seed sd_gnhiv7bzbr4btky4s32oi27bne was aborted); the next pass rolls it back
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_gnhiv7bzbr4btky4s32oi27bne has no route on this box: its local data is removed
2026/10/22 10:36:13 provision: acme.go.tito.io: aborted seed sd_gnhiv7bzbr4btky4s32oi27bne rolled back (no box had swapped it)
2026/10/22 10:36:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_h655dy4c5uhqismo4czrv73zny (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_h655dy4c5uhqismo4czrv73zny rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_h655dy4c5uhqismo4czrv73zny is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_5ql6oyn63kkdudzh742ihlfiay (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_5ql6oyn63kkdudzh742ihlfiay rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_5ql6oyn63kkdudzh742ihlfiay is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_hm2vxsgwxr323mfwdvcfliazq4 (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_hm2vxsgwxr323mfwdvcfliazq4 rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_hm2vxsgwxr323mfwdvcfliazq4: the host's replica is left alone: its record no longer names this seed
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_hm2vxsgwxr323mfwdvcfliazq4 is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_c5tbdbu42z4xibxujqcv4bjqby (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c5tbdbu42z4xibxujqcv4bjqby rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c5tbdbu42z4xibxujqcv4bjqby rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c5tbdbu42z4xibxujqcv4bjqby is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_wtstrep3qvyfcsvou7ovdkzuju (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_wtstrep3qvyfcsvou7ovdkzuju rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_4tbrp6a5xwrxn2vcl5t4cnxyp4 (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_4tbrp6a5xwrxn2vcl5t4cnxyp4 is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_33h5encv4greco2qalo23tihke (the original route is journaled)
2026/10/22 10:36:13 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_33h5encv4greco2qalo23tihke left alone this pass: its record could not be read
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_xqipq2xy7xap4b7rp6bfg3ywue (the original route is journaled)
2026/10/22 10:36:13 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_xqipq2xy7xap4b7rp6bfg3ywue left alone this pass: its record could not be read
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_yz7wnf5rsqrolq2nteoxalbtsq (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_yz7wnf5rsqrolq2nteoxalbtsq: close the import gate: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_yz7wnf5rsqrolq2nteoxalbtsq rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_yz7wnf5rsqrolq2nteoxalbtsq is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_lslawlgmq2cc7okhzz6k7pnpci (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_lslawlgmq2cc7okhzz6k7pnpci: close the import gate: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_lslawlgmq2cc7okhzz6k7pnpci rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_lslawlgmq2cc7okhzz6k7pnpci is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_omwhp6zmp4kzdgaxvaafcdvfsm (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_omwhp6zmp4kzdgaxvaafcdvfsm rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_omwhp6zmp4kzdgaxvaafcdvfsm rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_omwhp6zmp4kzdgaxvaafcdvfsm is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_u2etapjalk446sagmpsp6gqyqu (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_u2etapjalk446sagmpsp6gqyqu rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_u2etapjalk446sagmpsp6gqyqu is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_zvyohhyazwcv22elwlw75ic5tm (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_zvyohhyazwcv22elwlw75ic5tm: retire its instance key: injected failure at retire-key
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_zvyohhyazwcv22elwlw75ic5tm rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_zvyohhyazwcv22elwlw75ic5tm is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_lwfcales2oupmimsysi4j63bo4 (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_lwfcales2oupmimsysi4j63bo4 rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_lwfcales2oupmimsysi4j63bo4 is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_jvqmv3vovzfovhlccm6fyn7vvi (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_jvqmv3vovzfovhlccm6fyn7vvi: remove its local data: injected failure at remove-local
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_jvqmv3vovzfovhlccm6fyn7vvi rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_jvqmv3vovzfovhlccm6fyn7vvi is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_odsdec37tuqp4txhrsu6erxpt4 (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_odsdec37tuqp4txhrsu6erxpt4 rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_odsdec37tuqp4txhrsu6erxpt4 is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_ilgk2nvzrriyh3oos5w4rmvnsm (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_ilgk2nvzrriyh3oos5w4rmvnsm: remove its local data: injected failure at remove-local-done
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_ilgk2nvzrriyh3oos5w4rmvnsm rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_ilgk2nvzrriyh3oos5w4rmvnsm is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_e7tjpfrliet7ozw45opjue24xq (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_e7tjpfrliet7ozw45opjue24xq rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_e7tjpfrliet7ozw45opjue24xq is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_2ft6buwgy5mxordzr47qzgl6ra (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_2ft6buwgy5mxordzr47qzgl6ra: restore the original route: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_2ft6buwgy5mxordzr47qzgl6ra rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_2ft6buwgy5mxordzr47qzgl6ra is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_dlvyl5nnqwbuprygv7r42mp4ou (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_dlvyl5nnqwbuprygv7r42mp4ou: restore the original route: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_dlvyl5nnqwbuprygv7r42mp4ou rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_dlvyl5nnqwbuprygv7r42mp4ou is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_paasbfb7dyxhvjoe2rzrpjimlq (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_paasbfb7dyxhvjoe2rzrpjimlq is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_pihlq5seb3x6gccu4fczeniuxy (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_pihlq5seb3x6gccu4fczeniuxy rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_pihlq5seb3x6gccu4fczeniuxy: delete its snapshot: injected failure at delete-snapshot
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_pihlq5seb3x6gccu4fczeniuxy is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_qhhypkvshcbrvtouwtr6zds63y (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_qhhypkvshcbrvtouwtr6zds63y rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_qhhypkvshcbrvtouwtr6zds63y is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_s3busrkcrjf5jthf3ijcszo76a (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_s3busrkcrjf5jthf3ijcszo76a rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_s3busrkcrjf5jthf3ijcszo76a: delete the host's replica: injected failure at delete-replica
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_s3busrkcrjf5jthf3ijcszo76a is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_n2saul7l3zp7p4gpc46tdblfzm (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_n2saul7l3zp7p4gpc46tdblfzm rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_n2saul7l3zp7p4gpc46tdblfzm is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_c67whpyknufxugjkel52ubxjay (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c67whpyknufxugjkel52ubxjay rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c67whpyknufxugjkel52ubxjay: retire its journal row: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_c67whpyknufxugjkel52ubxjay is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_wtrwfhqgmpesjhe7lgjo76nx7q (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_wtrwfhqgmpesjhe7lgjo76nx7q rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_wtrwfhqgmpesjhe7lgjo76nx7q: retire its journal row: constraint failed: injected (1811)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_wtrwfhqgmpesjhe7lgjo76nx7q is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_7cawmmq6m7343elyglsjnu6k4m (the original route is journaled)
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_7cawmmq6m7343elyglsjnu6k4m rolled back to its original route
2026/10/22 10:36:13 provision: a.shop.example.com: import sd_7cawmmq6m7343elyglsjnu6k4m is cleaned up and its journal row retired
2026/10/22 10:36:13 provision: a.shop.example.com: mark import sd_7cawmmq6m7343elyglsjnu6k4m rolled back: injected failure at cas
2026/10/22 10:36:13 provision: a.shop.example.com: aborted seed sd_7cawmmq6m7343elyglsjnu6k4m rolled back (no box had swapped it)
2026/10/22 10:36:13 provision: a.shop.example.com: swapped for import sd_2s36rumzbacl5arfobo7fgwsfe (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_2s36rumzbacl5arfobo7fgwsfe rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_2s36rumzbacl5arfobo7fgwsfe is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: aborted seed sd_2s36rumzbacl5arfobo7fgwsfe rolled back (no box had swapped it)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_l4cpkyiyjlrtp5vhi6aglteo5q (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_l4cpkyiyjlrtp5vhi6aglteo5q rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_l4cpkyiyjlrtp5vhi6aglteo5q rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_l4cpkyiyjlrtp5vhi6aglteo5q rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_l4cpkyiyjlrtp5vhi6aglteo5q is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_zg6t4skthu7ghovwac2yu32ice (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_zg6t4skthu7ghovwac2yu32ice rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_zg6t4skthu7ghovwac2yu32ice is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_p6rd7tednnvdomcp2hgumyhhpy (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_p6rd7tednnvdomcp2hgumyhhpy rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_p6rd7tednnvdomcp2hgumyhhpy is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_sajghqgkpw3cxfp4y5ov5vpoaa (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_sajghqgkpw3cxfp4y5ov5vpoaa rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_sajghqgkpw3cxfp4y5ov5vpoaa is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_jevlkvlzujr7ocnzhsvrmoaegy (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_jevlkvlzujr7ocnzhsvrmoaegy rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_jevlkvlzujr7ocnzhsvrmoaegy is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_m4o4lojv3med4iu356pahlmypu (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_m4o4lojv3med4iu356pahlmypu rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_m4o4lojv3med4iu356pahlmypu is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_s4xwf4qgtbqebb4y3pzndt5sva (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_s4xwf4qgtbqebb4y3pzndt5sva rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_s4xwf4qgtbqebb4y3pzndt5sva is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_tlctajzsad57xya6f27koqvaky (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_tlctajzsad57xya6f27koqvaky rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_tlctajzsad57xya6f27koqvaky is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_76m5tzxfkx47qovjfes5tuexqm (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_76m5tzxfkx47qovjfes5tuexqm has no route on this box: its local data is removed and no route is put back
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_76m5tzxfkx47qovjfes5tuexqm is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_lzhnltckdd7zngu4kmz6sbzgye (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_jhaejtgbxg6jfb4kcino4it67e (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_a2xacvkknhix35t6uzpgjcstzi (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_a2xacvkknhix35t6uzpgjcstzi cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_se74rzr6meaicnimc45lufdzvi (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_se74rzr6meaicnimc45lufdzvi is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/22 10:36:14 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/22 10:36:14 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_yz44kk4rwyknp7uzzakod4yz3e (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_zqajnfyfmqkzn6vilqtyziv77m (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_oplfnb34isdefotpj4o25nduiq (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_ebnskft7fpumnlbjtoxysi7qnu (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_x225wme6b2rnjhztyeb3hxwjsu (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_x225wme6b2rnjhztyeb3hxwjsu rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_iymjiem3dfgyaz3myxfjfi6gje (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_iymjiem3dfgyaz3myxfjfi6gje has no route on this box: its local data is removed and no route is put back
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_iymjiem3dfgyaz3myxfjfi6gje is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_ndtijilwjt7uflrjcicybjxr5e (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_ndtijilwjt7uflrjcicybjxr5e: remove its local data: injected failure at remove-local
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_ndtijilwjt7uflrjcicybjxr5e is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_xpqykt4b5h44am4t5xrscsqneq (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_xpqykt4b5h44am4t5xrscsqneq: a route appeared at the host, so its local database is left and the rollback waits
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_lkr5fpjudfrlot437fwzlvwnui (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_lkr5fpjudfrlot437fwzlvwnui rollback deferred: no fence generation: registry busy
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_pbduy3fvaufsncqmvez22xtaui (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_pbduy3fvaufsncqmvez22xtaui rollback deferred: no fence generation: the generation source answered 0
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_ehlw64qlztlgb4keimusptjozy (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_sfrnlwm2iexghc72fqmjw464du (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_wnico5aefzr5ee4lcsp53v55he (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/22 10:36:14 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_2khwcifnccjana4nhtwgqgeywm (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_2khwcifnccjana4nhtwgqgeywm rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_2khwcifnccjana4nhtwgqgeywm is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_leufcynn73vzgyslg4zlo7zsde (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_leufcynn73vzgyslg4zlo7zsde rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_leufcynn73vzgyslg4zlo7zsde is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_w6zgg2frdxptbg2tp7vgfu64dm (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_w6zgg2frdxptbg2tp7vgfu64dm rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_w6zgg2frdxptbg2tp7vgfu64dm is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_h2dwmrhqk3njipnoysoo5su2da (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_h2dwmrhqk3njipnoysoo5su2da: retire its instance key: the env file is a symlink
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_h2dwmrhqk3njipnoysoo5su2da rolled back to its original route
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_h2dwmrhqk3njipnoysoo5su2da is cleaned up and its journal row retired
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_7wnjceofgvjrsf5qpdknxk4e74 (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/22 10:36:14 provision: a.shop.example.com: swapped for import sd_hesyyghjozntdgejmqt6js5ute (the original route is journaled)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_avole5pdcxf6cmgv54s7obvhsy refused: there is no route at a.shop.example.com to replace
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_z72gfxi76ajpiokmnpc7bnmhky refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_cnry2xz4j47j5d7sbcco7qpcs4 refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_q44n6cc7bui32rkxc3d6mlskzi refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_gmr6pa3h4pqvdoezcsrube42ee refused: the route belongs to app other
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_7ygu2t32nnr5rayrtufncrqwtu refused: the route is filed under another account
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_dz7ld3lao4odzjt2tumbomkqy4 refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_ezwdwo4c42zwfpowgjfgcbuaum refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_hlifwjvqb5lrykhu54dpdcbfpa refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_7t6j3ygm4pju3sym62otpa4j5e refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_y3w76sratevoslg6n4iheql2ii refused: the route is not an address route this import can take over (socket)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_matslo2zp5mx4l4um65magawre refused: the route is not an address route this import can take over (no address)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_szgvkpm3lfhjhysz4zik66qrxe refused: the route is not an address route this import can take over (provisioned)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_hdrrginttn252flesgnjd22pwa refused: the route is not an address route this import can take over (kind static)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_fugizp2hokhsqwzhmktkosuynq refused: the route is not an address route this import can take over (backing)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_2lm776dchwipumzv5u5457yhke refused: the route is not an address route this import can take over (tunnel)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_nmbhbxdgw4garecminhimt47qm refused: the route is not an address route this import can take over (caps)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_4r5gctm5esp6ybdvodjzgkhmqy refused: the route is not an address route this import can take over (platform)
2026/10/22 10:36:14 provision: a.shop.example.com: import sd_rmwso3wmll2tp2mvvihhrdxu6i refused: the route is not an address route this import can take over (canary)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_udrsglborgkud4dafzji7sqd4i refused: the route is not an address route this import can take over (kind sidecar)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_d6em2xtsopqesrc2377kdgzuey refused: the route is not an address route this import can take over (region)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_sv3gecykd3ljvn5udlmcw5zyoy refused: the route is not an address route this import can take over (fleet_box)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_m56ndvwxkrekcjl5fklzobbsoq refused: the route is not an address route this import can take over (hibernate)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_snkabegwd5gv5m3xbua4kjkwoi refused: the route is not an address route this import can take over (claim_owner)
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_grqco2g5u7inqefycxjwdhfmwy refused: this box does not run instances nested
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_atpa6thnamtf4t5rapm3tgz6ei refused: this box does not run instances nested
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_mzf7qdoa5jtlfewtk27ai4k3qy refused: claim control is on for this box
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_3r2j2enefeoqiqu6iqult37u5e (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_coicwgeyjym7myfmllpc6dlii4 (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/22 10:36:15 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_ewh3zqhspnxdinglmwm2352cjy: its verify hash is not a sha256
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_v7sywoxedstwttorw7frps2g6i: its verify hash is not a sha256
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_5bj4bv6dopzmr336apjwxy4kuq: its verify hash is not a sha256
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/22 10:36:15 provision: A.shop.example.com/..: not swapping for import sd_7acibdufbxujncgrhwriopq2je: its host or seed id is malformed (seeds: not a host)
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_eeikthuern4fvjr3jyhjyo3mnq: its record does not describe a nested, hibernating instance
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_ubprobrxd7rtx26jhcbwqn445m: its record does not describe a nested, hibernating instance
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_hi3ouejygiku2tidi45i5otize: its record does not describe a nested, hibernating instance
2026/10/22 10:36:15 provision: a.shop.example.com: not swapping for import sd_dzhfc2ewbkbzrx2l722esedzh4: its record does not describe a nested, hibernating instance
2026/10/22 10:36:15 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_d4ioh4zunghchxihvkyiv6hjr4 (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_3dhyuiq77rjne57tqb2vgzxhey (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_4yxubcdc6plqkr5zzaisuofpfi (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/22 10:36:15 provision: dry-run: would consider swapping a.shop.example.com for import sd_envcgo43qkuldttw5a3yo3hln4
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_dpfzxse3oeeyhrlrgirh3ydj7u rolled back to its original route
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_dpfzxse3oeeyhrlrgirh3ydj7u: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_dpfzxse3oeeyhrlrgirh3ydj7u is cleaned up and its journal row retired
2026/10/22 10:36:15 provision: a.shop.example.com: import sd_nnlae5fud6xbsmdcdlpwcuhl2u left alone: the import journal holds seed sd_dfrnvsc2xbmuxhx5cw47cbcl5e at swapped
2026/10/22 10:36:15 provision: a.shop.example.com: not minting import sd_nk67wh3hstui6w6ep5ubb5riv4: it is swapped but not committed, and only its swap makes its row
2026/10/22 10:36:15 provision: a.shop.example.com: not minting import sd_w3bxtfpnije2bgcpluuctnrcuq: it is placed but not committed, and only its swap makes its row
2026/10/22 10:36:15 provision: a.shop.example.com: swapped for import sd_r4jaq3hthuwqsorptcl7pc2nwy (the original route is journaled)
2026/10/22 10:36:15 provision: a.shop.example.com: not minting import sd_r4jaq3hthuwqsorptcl7pc2nwy: it is swapped but not committed, and only its swap makes its row
2026/10/22 10:36:15 provision: a.shop.example.com: not minting import sd_r4jaq3hthuwqsorptcl7pc2nwy: it is placed but not committed, and only its swap makes its row
2026/10/22 10:36:15 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/22 10:36:15 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/22 10:36:15 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/22 10:36:15 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/22 10:36:15 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/22 10:36:15 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/22 10:36:15 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/22 10:36:15 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/22 10:36:15 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/22 10:36:15 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/22 10:36:15 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/22 10:36:15 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/22 10:36:15 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/22 10:36:15 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/22 10:36:15 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/22 10:36:15 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/22 10:36:15 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/22 10:36:15 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/22 10:36:15 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/22 10:36:15 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/22 10:36:15 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/22 10:36:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:15 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/22 10:36:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/22 10:36:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/22 10:36:15 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/22 10:36:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/22 10:36:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/22 10:36:15 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/22 10:36:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/22 10:36:15 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/22 10:36:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/22 10:36:15 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:15 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/22 10:36:15 provision: an app's records could not be listed: they are skipped
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/22 10:36:15 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/22 10:36:15 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): the walk records no generations
2026/10/22 10:36:15 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/22 10:36:15 provision: record view: change index unusable (get refused): walking
2026/10/22 10:36:15 provision: record view: change index unusable (get refused): the walk records no generations
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/22 10:36:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/22 10:36:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/22 10:36:15 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/22 10:36:15 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/22 10:36:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (15.96s)
plan14_root_reads_test.go:1124: root read census: 60 discrepancies
changed root read internal/appconfig/syncexec.go:SyncExecWithCallbacks; review class, task, sites and digest
changed root read internal/host/activation_command_linux.go:runSystemClaims; review class, task, sites and digest
changed root read internal/host/agent.go:configGenLoop; review class, task, sites and digest
changed root read internal/host/agent.go:fastConvergeLoop; review class, task, sites and digest
changed root read internal/host/agent.go:fastTick; review class, task, sites and digest
changed root read internal/host/agent.go:genPollPass; review class, task, sites and digest
changed root read internal/host/agent.go:runAgent; review class, task, sites and digest
changed root read internal/host/steering_loop.go:steeringPass; review class, task, sites and digest
changed root read internal/provision/reconcile.go:Reconcile; review class, task, sites and digest
changed root read internal/provision/reconcile.go:holdExisting; review class, task, sites and digest
changed root read internal/provision/sidecar_reconcile.go:ReconcileSidecars; review class, task, sites and digest
changed root read internal/release/adopt.go:PassFilter; review class, task, sites and digest
changed root read internal/telemetry/telemetry.go:Resolve; review class, task, sites and digest
changed root source envelope .
changed root source envelope cmd/console
changed root source envelope internal/appconfig
changed root source envelope internal/edge
changed root source envelope internal/host
changed root source envelope internal/hostops
changed root source envelope internal/ops
changed root source envelope internal/provision
changed root source envelope internal/release
changed root source envelope internal/telemetry
changed root source envelope internal/vet
stale root read internal/host/buildstate.go:buildStates.refreshAll
stale root read internal/host/telemetry_pass.go:telemetrySwitches
stale root read internal/release/adopt.go:type passConfig
stale root read internal/release/conditional_delete.go:DirStore.DeleteIfMatch
stale root read internal/release/store.go:DirStore.Put
stale root read internal/release/store.go:DirStore.PutFrom
unclassified root read internal/appconfig/syncexec.go:syncOptions.kept whole:Route@84
unclassified root read internal/host/agent.go:fastDeps.canaryPointerGate whole:Route@486
unclassified root read internal/host/agent.go:fastDeps.canarySettled registry:Host@244,whole:Route@137,whole:Route@190
unclassified root read internal/host/buildstate.go:buildStates.refreshKept registry:Host@549,whole:Route@442,whole:Route@472,whole:Route@517
unclassified root read internal/host/changeindex.go:movedRoutes registry:App@297,whole:Route@292
unclassified root read internal/host/telemetry_pass.go:telemetryReads.read file:Read@396
unclassified root read internal/host/telemetry_pass.go:telemetrySwitchesWith registry:App@459,registry:App@547,registry:App@842,registry:Kind@436,whole:Route@422,whole:Route@521
unclassified root read internal/provision/recordview_store.go:confirmRecordGone registry:Account@195,registry:App@207,registry:Host@215,registry:Host@428,registry:Host@646
unclassified root read internal/provision/recordview_store.go:confirmSidecarGone registry:Account@210,registry:App@222,registry:Host@451,registry:Host@498,registry:Host@700,registry:Slug@230
unclassified root read internal/provision/recordview_store.go:snapStore.Open file:Open@20
unclassified root read internal/release/adopt.go:passConfig.gatePointers registry:App@305,whole:Route@300,whole:Route@326,whole:Route@420,whole:Route@442,whole:Route@479,whole:Route@497,whole:Route@548,whole:Route@567,whole:Route@578,whole:Route@750,whole:Route@955,whole:Route@979
unclassified root read internal/release/conditional_delete.go:DirStore.deleteIfMatchRaw file:ReadFile@244
unclassified root read internal/release/store.go:DirStore.ListETags file:WalkDir@145
unclassified root read internal/release/store.go:DirStore.putFromRaw file:Stat@462
unclassified root read internal/release/store.go:DirStore.putRaw file:Stat@443
unclassified root read internal/release/store.go:S3Store.GetIfChanged file:ReadAll@511
changed legacy declaration internal/appconfig/syncexec.go:SyncExecWithCallbacks
changed legacy declaration internal/host/agent.go:configGenLoop
changed legacy declaration internal/host/agent.go:fastTick
changed legacy declaration internal/host/agent.go:genPollPass
changed legacy declaration internal/host/steering_loop.go:steeringPass
changed legacy declaration internal/host/telemetry_pass.go:telemetrySwitches
changed legacy declaration internal/provision/reconcile.go:Reconcile
changed legacy declaration internal/provision/reconcile.go:holdExisting
changed legacy declaration internal/provision/sidecar_reconcile.go:ReconcileSidecars
changed legacy declaration internal/release/conditional_delete.go:DirStore.DeleteIfMatch
changed legacy declaration internal/release/store.go:DirStore.Put
changed legacy declaration internal/release/store.go:DirStore.PutFrom
changed legacy declaration internal/telemetry/telemetry.go:Resolve
changed reclassified declaration internal/host/agent.go:fastConvergeLoop
FAIL
FAIL github.com/carlosframework/platform/internal/provision 26.196s
ok github.com/carlosframework/platform/internal/publicrelease 0.011s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 6.900s
ok github.com/carlosframework/platform/internal/relay 7.220s
ok github.com/carlosframework/platform/internal/release 28.781s
ok github.com/carlosframework/platform/internal/rollout 4.655s
ok github.com/carlosframework/platform/internal/rolloutwords 0.010s
ok github.com/carlosframework/platform/internal/rootaws 0.020s
ok github.com/carlosframework/platform/internal/rootparticipation 0.062s
ok github.com/carlosframework/platform/internal/rootstate 0.044s
ok github.com/carlosframework/platform/internal/runs 0.605s
ok github.com/carlosframework/platform/internal/s3compat 0.012s
ok github.com/carlosframework/platform/internal/safefile 0.007s
ok github.com/carlosframework/platform/internal/schedule 3.388s
ok github.com/carlosframework/platform/internal/seeds 0.013s
ok github.com/carlosframework/platform/internal/sim/hostlife 15.199s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.711s
ok github.com/carlosframework/platform/internal/status 0.113s
ok github.com/carlosframework/platform/internal/steering 0.011s
ok github.com/carlosframework/platform/internal/stopevidence 0.490s
ok github.com/carlosframework/platform/internal/svcrole 0.005s
ok github.com/carlosframework/platform/internal/telemetry 0.008s
ok github.com/carlosframework/platform/internal/termfence 0.007s
ok github.com/carlosframework/platform/internal/tunnelclient 0.013s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.006s
ok github.com/carlosframework/platform/internal/usage 0.015s
ok github.com/carlosframework/platform/internal/version 0.008s
ok github.com/carlosframework/platform/internal/vet 1.883s
FAIL
=== internal_console@0 (82 s)
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: backfilled account manifest for bab (ops)
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:35:56 console: could not notify paul@keymail.dev of a CLI token issue: Post "http://127.0.0.1:40059/api/app/send": dial tcp 127.0.0.1:40059: connect: connection refused
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:56 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: api instance move bab/titogo/acme.go.tito.io: storage binding: read timeout
2026/10/22 10:35:57 console: api instance move bab/titogo/acme.go.tito.io: manifest: read timeout
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: could not notify roller@keymail.dev of a CLI token issue: Post "http://127.0.0.1:39561/api/app/send": EOF
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: api system status: canary pointer: system pointer does not verify: system pointer router/canary: signature does not verify
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:57 console: app region bab/tmp-docs: (default)
2026/10/22 10:35:57 console: backfilled account manifest for bab (ops)
2026/10/22 10:35:57 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:35:58 console: api updates bab/km policy: enable refused, not every box applies update settings: ie-1: runs a binary that does not apply update settings
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:35:58 console: api updates bab/km policy: enable refused, the box inventory could not be read: edges records: AccessDenied: s3://carlos-prod-secret-bucket/ledgers request id ABC123
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: backfilled account manifest for bab (ops)
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: backfilled account manifest for bab (ops)
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: store destroy bab/kass: objstore: empty bucket carlos-bab-kass: backend unreachable
2026/10/22 10:35:58 console: backfilled account manifest for bab (ops)
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: account bac names owner "not-an-address", which is not a keymail address — nobody could sign in as it, so no membership was created. Fix it with: console member --add <address> bac --role owner
2026/10/22 10:35:58 console: materialized account bac (Typo) from the bucket
2026/10/22 10:35:58 console: accounts/bac/account.json names owner "not-an-address", which is not a keymail address — materializing the account without it; give it one with `console member --add <address> bac --role owner`
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:58 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: refused: no eligible box in uk
2026/10/22 10:35:59 console: claim request refused: no eligible box in uk: hostops: region has no box that can take this
2026/10/22 10:35:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:35:59 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:00 console: api relay enable: hostops: pending
2026/10/22 10:36:00 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation 03ab73179d55aa38, op 1)
2026/10/22 10:36:01 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation a8388cc5521caf02, op 1)
2026/10/22 10:36:01 console: api relay adopted: hostops: pending
2026/10/22 10:36:02 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation d01e273371ef0937, op 1)
2026/10/22 10:36:02 console: api relay disable: hostops: pending
2026/10/22 10:36:02 console: relay disabled bab/titogo by paul@keymail.dev (op 2)
2026/10/22 10:36:03 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation c36dd317a2330032, op 1)
2026/10/22 10:36:03 console: relay rotate eu-west-1: bab/titogo: hostops: pending
2026/10/22 10:36:03 console: relay rotated eu-west-1 to kid k2, re-issued 0 app(s)
2026/10/22 10:36:03 console: audit relay-rotate platform: sql: no rows in result set
2026/10/22 10:36:03 console: relay rotated eu-west-1 to kid k3, re-issued 1 app(s)
2026/10/22 10:36:03 console: audit relay-rotate platform: sql: no rows in result set
2026/10/22 10:36:04 console: mail push: bab/titogo: hostops: pending
2026/10/22 10:36:10 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation b120a40c5970df65, op 1)
2026/10/22 10:36:11 console: relay writer lease acquired (holder d3d828d05a380fc6)
2026/10/22 10:36:11 console: relay reconcile bab/titogo: hostops: pending
2026/10/22 10:36:14 console: relay writer lease acquired (holder d3d828d05a380fc6)
2026/10/22 10:36:18 console: app region bab/tmp-claim-expiry: (default)
2026/10/22 10:36:18 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:18 console: cli token revoke paul@keymail.dev: hostops: pending
2026/10/22 10:36:18 console: api store rotate bab/titogo: hostops: pending
2026/10/22 10:36:18 console: api config apply bab/titogo (default): hostops: pending
2026/10/22 10:36:18 console: expiry pass: trash bab/tmp-claim-expiry: hostops: pending
2026/10/22 10:36:18 console: scan drain: bab/titogo: hostops: pending
2026/10/22 10:36:18 console: scan pass: bab/titogo: hostops: pending
2026/10/22 10:36:19 console: app region bab/tmp-claim-ttl: (default)
2026/10/22 10:36:19 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:19 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:36:19 console: lifecycle tmp-claim-ttl: hostops: pending
2026/10/22 10:36:19 console: api scan grant bab/titogo: enable: hostops: pending
2026/10/22 10:36:19 console: ses events: logging Delivery for bab/titogo: hostops: pending
2026/10/22 10:36:19 console: ses events: recording Delivery for bab/titogo: hostops: pending
2026/10/22 10:36:20 console: api store provision bab/titogo: hostops: pending
2026/10/22 10:36:20 console: api store grant bab/titogo (bucket ""): hostops: pending
2026/10/22 10:36:20 console: api scan callback rotate bab/titogo: deliver: hostops: pending
2026/10/22 10:36:21 console: api scan request bab/titogo: hostops: pending
2026/10/22 10:36:21 console: api store rotate finish bab/titogo: hostops: pending
2026/10/22 10:36:21 console: service rotate bab/svc-a: hostops: pending
2026/10/22 10:36:21 console: service revoke bab: hostops: pending
2026/10/22 10:36:21 console: service create bab/svc-cli: hostops: pending
2026/10/22 10:36:21 console: api scan sweep bab/titogo: hostops: pending
2026/10/22 10:36:22 console: api scan revoke bab/titogo: hostops: pending
2026/10/22 10:36:22 console: api scan enqueue bab/titogo "claim-fixture.txt": hostops: pending
2026/10/22 10:36:22 console: api store create bab/titogo: hostops: pending
2026/10/22 10:36:22 console: api scan enforce bab/titogo: hostops: pending
2026/10/22 10:36:22 console: api sidecars wake bab/titogo/bot: hostops: pending
2026/10/22 10:36:23 console: claim request refused: hostops: pending
2026/10/22 10:36:23 console: api schedule run bab/titogo sync: hostops: pending
2026/10/22 10:36:23 console: api schedule rm bab/titogo sync: hostops: pending
2026/10/22 10:36:23 console: api schedule set bab/titogo sync: hostops: pending
2026/10/22 10:36:23 console: api rollback kass canary/x: hostops: pending
2026/10/22 10:36:24 console: api retention set bab/titogo: hostops: pending
2026/10/22 10:36:24 console: api restore bab/titogo: hostops: pending
2026/10/22 10:36:24 console: api restart scope bab/kass (host kass.carlosapp.dev): hostops: pending
2026/10/22 10:36:24 console: api restart bab/titogo: hostops: pending
2026/10/22 10:36:24 console: api authorize bab/kass v1: hostops: pending
2026/10/22 10:36:24 console: api ship titogo abc1234: hostops: pending
2026/10/22 10:36:25 console: api promote kass v1 -> canary/claim: hostops: pending
2026/10/22 10:36:25 console: door key mint bab/titogo: hostops: pending
2026/10/22 10:36:25 console: api instance channel bab/titogo/web.go.tito.io: hostops: pending
2026/10/22 10:36:25 console: api instance token bab/titogo: hostops: pending
2026/10/22 10:36:25 console: claim request refused: hostops: pending
2026/10/22 10:36:26 console: api features set bab/titogo: hostops: pending
2026/10/22 10:36:26 console: api email rotate bab/titogo: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email pause bab/titogo: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email enable bab/titogo: apply grant: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email domain verify bab/titogo.bab.oncarlos.com: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email domain ensure bab/titogo (titogo.bab.oncarlos.com): mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email credentials create bab/titogo: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: api email declare bab/titogo: mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:26 console: door key revoke bab/titogo b8e033c56c19fbce: hostops: pending
2026/10/22 10:36:27 console: trash bab/titogo: hostops: pending
2026/10/22 10:36:27 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:36:27 console: promote kass v1 -> canary/x: hostops: pending
2026/10/22 10:36:27 console: production kass -> true: hostops: pending
2026/10/22 10:36:27 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:27 console: pipeline settings save bab/titogo: set: hostops: pending
2026/10/22 10:36:27 console: app region bab/tmp-claim-keep: (default)
2026/10/22 10:36:27 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:27 console: reject pending change ffe85dc5bef760b91ca31e5832e4b524: hostops: pending
2026/10/22 10:36:27 console: confirm pending change: hostops: pending
2026/10/22 10:36:27 console: keep bab/tmp-claim-keep: hostops: pending
2026/10/22 10:36:28 console: api updates install bab/titogo/acme.go.tito.io: hostops: pending
2026/10/22 10:36:28 console: browser retry bab/titogo/web.go.tito.io: hostops: pending
2026/10/22 10:36:28 console: api updates bab/titogo/acme.go.tito.io: hostops: pending
2026/10/22 10:36:28 console: browser restart bab/kass/kass.carlosapp.dev: hostops: pending
2026/10/22 10:36:28 console: fast-track kass stable: hostops: pending
2026/10/22 10:36:28 console: gate write bab/titogo/acme.go.tito.io: hostops: pending
2026/10/22 10:36:28 console: errors level bab/kass: hostops: pending
2026/10/22 10:36:29 console: api updates bab/titogo/acme.go.tito.io: hostops: pending
2026/10/22 10:36:29 console: door keys re-sign bab/titogo: hostops: pending
2026/10/22 10:36:29 console: api version-target set bab/titogo: hostops: pending
2026/10/22 10:36:29 console: api updates policy bab/titogo: hostops: pending
2026/10/22 10:36:29 console: api telemetry bab/titogo: hostops: pending
2026/10/22 10:36:29 console: api instances enable bab/titogo: hostops: pending
2026/10/22 10:36:30 console: api swap bab/titogo: hostops: pending
2026/10/22 10:36:30 console: api sidecars enable bab/titogo: hostops: pending
2026/10/22 10:36:30 console: api email push key bab/titogo: hostops: pending
2026/10/22 10:36:30 console: api pipeline put bab/titogo: set: hostops: pending
2026/10/22 10:36:30 console: api domains paths bab/titogo ti.to: hostops: pending
2026/10/22 10:36:30 console: claim request refused: hostops: pending
2026/10/22 10:36:31 console: api trash bab/titogo: hostops: pending
2026/10/22 10:36:31 console: api config sync bab/titogo: hostops: pending
2026/10/22 10:36:31 console: api resign bab/kass canary/x: hostops: pending
2026/10/22 10:36:31 console: restore bab/titogo: hostops: pending
2026/10/22 10:36:32 console: app region bab/claim-cli-new: (default)
2026/10/22 10:36:32 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:32 console: control-plane set bab/titogo: hostops: pending
2026/10/22 10:36:32 console: claim restore bab/titogo: recovered (restored)
2026/10/22 10:36:32 console: control-plane clear bab/titogo: hostops: pending
2026/10/22 10:36:32 console: app region bab/tmp-claim-cli: (default)
2026/10/22 10:36:32 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:32 console: retire kass canary/rc: hostops: pending
2026/10/22 10:36:33 console: canary kass v1 -> canary/rc: hostops: pending
2026/10/22 10:36:33 console: audit db-read titogo: hostops: pending
2026/10/22 10:36:33 console: analytics setting bab/kass: hostops: pending
2026/10/22 10:36:34 console: app region bab/claim-web-new: (default)
2026/10/22 10:36:34 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:34 console: audit sidecar-delete titogo: hostops: pending
2026/10/22 10:36:34 console: api instances update bab/titogo/ha.go.tito.io: hostops: pending
2026/10/22 10:36:34 console: ledger blob bab/titogo/prompts: hostops: pending
2026/10/22 10:36:34 console: control-plane clear bab/titogo: hostops: pending
2026/10/22 10:36:34 console: ledger append bab/titogo/carbon: hostops: pending
2026/10/22 10:36:34 console: api updates install bab/titogo/acme.go.tito.io: hostops: pending
2026/10/22 10:36:35 console: app region bab/tmp-claim-keep: (default)
2026/10/22 10:36:35 console: backfilled account manifest for bab (ops)
2026/10/22 10:36:35 console: ledger publish bab/titogo: hostops: pending
2026/10/22 10:36:35 console: lifecycle tmp-claim-keep: hostops: pending
2026/10/22 10:36:35 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:36:36 console: service revoke bab: hostops: pending
2026/10/22 10:36:36 console: service create bab/svc-web: hostops: pending
2026/10/22 10:36:36 console: service rotate bab/svc-a: hostops: pending
2026/10/22 10:36:36 console: store destroy bab/titogo: hostops: pending
2026/10/22 10:36:36 console: swap policy bab/kass: hostops: pending
2026/10/22 10:36:36 console: api instance retry bab/titogo/web.go.tito.io: hostops: pending
2026/10/22 10:36:37 console: rollback kass canary/x: hostops: pending
2026/10/22 10:36:37 console: api email push off bab/titogo: hostops: pending
2026/10/22 10:36:37 console: api email domain remove bab/titogo (titogo.bab.oncarlos.com): mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:37 console: audit instance-delete titogo: hostops: pending
2026/10/22 10:36:37 console: api email credentials revoke bab/titogo (laptop): mail: lifecycle resource operation pending
hostops: pending
2026/10/22 10:36:38 console: browser restart bab/kass/: hostops: pending
2026/10/22 10:36:48 console: geoip published ed972738 (GeoIP2-City, manual)
2026/10/22 10:36:48 console: audit geoip off platform: sql: no rows in result set
2026/10/22 10:36:48 console: relay enabled bab/titogo in eu-west-1 by paul@keymail.dev (incarnation 45c1c7ee7e98c16b, op 1)
2026/10/22 10:36:49 console: geoip published ed972738 (GeoIP2-City, manual)
2026/10/22 10:36:50 console: geoip published ed972738 (GeoIP2-City, manual)
2026/10/22 10:36:50 console: audit geoip publish platform: sql: no rows in result set
2026/10/22 10:36:57 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:36:58 console: audit store-decline : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:36:58 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:36:58 console: audit store-approve : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:36:58 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:36:58 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:36:59 console: audit seeds-floor-publish platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-activation-cancel platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-rollback platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit pointer-digests-on platform: sql: no rows in result set
2026/10/22 10:36:59 console: audit system-layout platform: sql: no rows in result set
2026/10/22 10:36:59 console: object scanning enabled deployment-wide by paul@keymail.dev
2026/10/22 10:37:00 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-detach-box pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:00 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:01 console: audit fleet-tag-box pi: sql: no rows in result set
2026/10/22 10:37:01 console: audit fleet-rotate-box pi: sql: no rows in result set
2026/10/22 10:37:01 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:37:01 console: audit store-decline : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:37:02 console: audit analytics-setting account: sql: no rows in result set
2026/10/22 10:37:03 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:37:04 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:37:04 console: audit system-update platform: sql: no rows in result set
2026/10/22 10:37:05 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
2026/10/22 10:37:05 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-rotate-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-add-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit fleet-tag-box pi: sql: no rows in result set
2026/10/22 10:37:07 console: audit errors-level account: sql: no rows in result set
2026/10/22 10:37:08 console: audit fleet-detach-box pi: sql: no rows in result set
2026/10/22 10:37:08 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:10 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:10 console: audit fleet-create pi: sql: no rows in result set
2026/10/22 10:37:10 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:10 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:10 console: account telemetry bab: dirstore: invalid key "apps/bab//telemetry.json"
2026/10/22 10:37:10 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:11 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:12 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:12 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:12 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:12 console: audit store-apply : constraint failed: claim: app write not permitted (1811)
--- FAIL: TestClaimConsoleMutationInvariant (72.95s)
claim_invariant_test.go:360: app-scoped writer bucket/GetIfChanged is reached by no door's open run and has no reason in claimWriterReasons
claim_invariant_test.go:360: app-scoped writer bucket/ListETags is reached by no door's open run and has no reason in claimWriterReasons
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:12 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:12 console: could not notify alice@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:37:12 console: could not notify paul@keymail.dev of a CLI token issue: keymail: grant revoked or invalid
2026/10/22 10:37:12 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:13 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: materialized account bbw (Blog) from the bucket
2026/10/22 10:37:14 console: materialized account mdbbndjqx (Kass) from the bucket
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: moves: acme.go.tito.io: move mv_373oxhx46baky7xwqxtq3xr4cm is still preparing after 6h0m0s; aborted
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:14 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: app region bab/solo: (default)
2026/10/22 10:37:15 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:15 console: ALERT [carlos] SQL effect old on bab/demo needs the restore break-glass path
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: could not notify alice@keymail.dev of a CLI token issue: keymail: lookup alice@keymail.dev: Get "http://127.0.0.1:34803/api/federation/lookup?addr=alice%40keymail.dev&from=console%40127.0.0.1%3A37369": dial tcp 127.0.0.1:34803: connect: connection reset by peer
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 provision: skipping record apps/bab/shop/instances/other.example.com.json: unreadable: slow down
2026/10/22 10:37:16 console: run sweep: some records could not be read; their runs wait for a later pass, and no orphan run is deleted on this one
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: scan queue rebuild: acct-bad: transient listing failure
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: scan queue rebuild: acct-bad/shop: transient listing failure
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000000 txid.db=0000000000000001
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000001 maxTXID=0000000000000001 size=720
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000001
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000001 txid.db=0000000000000002
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000002 maxTXID=0000000000000002 size=370
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000002
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000002 txid.db=0000000000000003
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000003 maxTXID=0000000000000003 size=454
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000003
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000003 txid.db=0000000000000004
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000004 maxTXID=0000000000000004 size=537
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000004 txid.db=0000000000000004
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000004 txid.db=0000000000000005
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000005 maxTXID=0000000000000005 size=621
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000005 txid.db=0000000000000005
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000005 txid.db=0000000000000006
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000006 maxTXID=0000000000000006 size=704
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000006 txid.db=0000000000000006
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000006 txid.db=0000000000000007
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000007 maxTXID=0000000000000007 size=788
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000007 txid.db=0000000000000007
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000007 txid.db=0000000000000008
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000008 maxTXID=0000000000000008 size=871
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000008 txid.db=0000000000000008
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000008 txid.db=0000000000000009
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000009 maxTXID=0000000000000009 size=955
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000009 txid.db=0000000000000009
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000009 txid.db=000000000000000a
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000a maxTXID=000000000000000a size=1038
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000a txid.db=000000000000000a
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000a txid.db=000000000000000b
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000b maxTXID=000000000000000b size=1124
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000b txid.db=000000000000000b
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000b txid.db=000000000000000c
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000c maxTXID=000000000000000c size=1208
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000c txid.db=000000000000000c
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000c txid.db=000000000000000d
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000d maxTXID=000000000000000d size=1294
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000d txid.db=000000000000000d
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000d txid.db=000000000000000e
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000e maxTXID=000000000000000e size=1387
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000e txid.db=000000000000000e
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000e txid.db=000000000000000f
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=000000000000000f maxTXID=000000000000000f size=1481
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000f txid.db=000000000000000f
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=000000000000000f txid.db=0000000000000010
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000010 maxTXID=0000000000000010 size=1569
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000010 txid.db=0000000000000010
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000010 txid.db=0000000000000011
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000011 maxTXID=0000000000000011 size=1661
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000011 txid.db=0000000000000011
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000011 txid.db=0000000000000012
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000012 maxTXID=0000000000000012 size=1755
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000012 txid.db=0000000000000012
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000012 txid.db=0000000000000013
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000013 maxTXID=0000000000000013 size=1843
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000013 txid.db=0000000000000013
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000013 txid.db=0000000000000014
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000014 maxTXID=0000000000000014 size=1935
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000014 txid.db=0000000000000014
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000014 txid.db=0000000000000015
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000015 maxTXID=0000000000000015 size=2031
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000015 txid.db=0000000000000015
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000015 txid.db=0000000000000016
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000016 maxTXID=0000000000000016 size=2119
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000016 txid.db=0000000000000016
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000016 txid.db=0000000000000017
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000017 maxTXID=0000000000000017 size=2216
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000017 txid.db=0000000000000017
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000017 txid.db=0000000000000018
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000018 maxTXID=0000000000000018 size=2309
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000018 txid.db=0000000000000018
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000018 txid.db=0000000000000019
2026/10/22 10:37:16 INFO ltx file uploaded system=store db=source.db replica=file level=0 minTXID=0000000000000019 maxTXID=0000000000000019 size=2403
2026/10/22 10:37:16 INFO replica sync system=store db=source.db replica=file txid.replica=0000000000000019 txid.db=0000000000000019
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 edges: box "kass-2" has an edges file under control/edges/ but no status snapshot at all, so no region outside the home is automatically eligible until it reports; said once
2026/10/22 10:37:16 console: seeds readiness: box "kass-2" in region "the home region" is unconfirmed: stale heartbeat, generated 2026-10-22T09:37:16Z
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: place bab/shop: read instances: records that do not read: [broken.example.com]
2026/10/22 10:37:16 console: place bab/shop: read instances: provision: read record apps/bab/shop/instances/plain.example.com.json: injected read failure
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: seeds pass: abandoned seed sd_sm55ezvo53mlupfjif52raynki of bab/shop/canary.shop.example.com (reserving since 2026-10-22T10:27:16Z)
2026/10/22 10:37:16 console: seeds pass: deleted seeds/canary.shop.example.com/sd_sm55ezvo53mlupfjif52raynki/snapshot.db
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:16 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: ses events: refused topic "arn:aws:sns:eu-west-1:9:someone-else" — this deployment expects "arn:aws:sns:eu-west-1:1:carlos-mail"
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: materialized account bac (Testco) from the bucket
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: store decision bab: no keymail grant for paul@keymail.dev — not notified (they must sign in to the console once)
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: account telemetry bab: dirstore: invalid key "apps/bab//telemetry.json"
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 console: app region bab/tmp-plain: ie (default)
2026/10/22 10:37:17 console: backfilled account manifest for bab (ops)
2026/10/22 10:37:17 console: CARLOS_SECRETS_PUBKEY is unset, so no region outside the home can be chosen automatically
2026/10/22 10:37:17 status: fleet: skipping status/boxes/zz-1.json: corrupt snapshot: invalid character 'n' looking for beginning of object key string
FAIL
FAIL github.com/carlosframework/platform/internal/console 81.143s
FAIL
=== internal_console@1 (39 s)
ok github.com/carlosframework/platform/internal/console 37.425s
=== internal_console@2 (44 s)
ok github.com/carlosframework/platform/internal/console 42.997s
=== internal_console@3 (39 s)
ok github.com/carlosframework/platform/internal/console 37.704s
=== internal_console@4 (39 s)
ok github.com/carlosframework/platform/internal/console 37.970s
=== internal_console@5 (37 s)
ok github.com/carlosframework/platform/internal/console 35.879s
=== internal_console@6 (38 s)
ok github.com/carlosframework/platform/internal/console 36.915s
=== internal_console@7 (45 s)
ok github.com/carlosframework/platform/internal/console 43.631s
=== root@0 (33 s)
ok github.com/carlosframework/platform 31.981s
=== root@1 (47 s)
ok github.com/carlosframework/platform 46.172s
=== root@2 (48 s)
ok github.com/carlosframework/platform 46.599s
FAILED: rest internal_console@0
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 1d266f687..2385e2cc8 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -163,6 +163,7 @@ import ( |
| "path" |
| "path/filepath" |
| "runtime" |
| + "slices" |
| "sort" |
| "strings" |
| "sync" |
| @@ -1681,6 +1682,13 @@ type genCache struct { |
| // tick it appeared. |
| exec, unit string |
| shapeSeen bool |
| + |
| + // gate skips the read of an app whose scope has not moved since its last |
| + // successful read, younger than the backstop (S3 change-index design §5). |
| + // nil, or a gate over a nil or unusable watch, reads every app every |
| + // tick, as before the index. now is the gate's clock (nil: time.Now). |
| + gate *release.ReadGate |
| + now func() time.Time |
| } |
| func newGenCache() *genCache { return &genCache{etags: map[[2]string]string{}} } |
| @@ -1739,6 +1747,10 @@ func newGenCache() *genCache { return &genCache{etags: map[[2]string]string{}} } |
| // for the caller to aggregate into ONE log line per pass, never one per |
| // app per tick. Clearing the entry instead would turn a recovered blip |
| // into a bogus first sight or a bogus change. |
| +// - With the change index armed (cache.gate), an app already in the cache |
| +// is read only when its scope moved, its last read failed, or that read |
| +// is the backstop's age (S3 change-index design §5). A skipped app keeps |
| +// its entry and reports no movement. |
| // |
| // The cache is mutated in place — it lives across ticks in the loop, which |
| // is the whole point (there is nowhere else to remember what "changed" |
| @@ -1774,10 +1786,29 @@ func genPollPass(ctx context.Context, routes []registry.Route, store release.Sto |
| apps = append(apps, k) |
| } |
| } |
| + now := time.Now |
| + if cache.now != nil { |
| + now = cache.now |
| + } |
| + at := now() |
| live := make(map[[2]string]bool, len(apps)) |
| for _, k := range apps { |
| live[k] = true |
| - _, etag, err := store.Get(ctx, appconfig.GenKey(k[0], k[1])) |
| + key := appconfig.GenKey(k[0], k[1]) |
| + // The gate (spec §5): an app already in the cache is read again only |
| + // when its scope moved, its last read failed, or that read is the |
| + // backstop's age. An app not in the cache is always read: its first |
| + // sight is what a later change is compared against. The generation |
| + // is taken before the read, so a write landing during it stays due. |
| + scope, gated := release.ChangeScope(key) |
| + gen, due := cache.gate.Due(scope, at) |
| + if _, seen := cache.etags[k]; seen && gated && !due { |
| + continue |
| + } |
| + _, etag, err := store.Get(ctx, key) |
| + if gated { |
| + cache.gate.Mark(scope, gen, at, err == nil || errors.Is(err, os.ErrNotExist)) |
| + } |
| switch { |
| case errors.Is(err, os.ErrNotExist): |
| etag = genAbsent |
| @@ -2020,8 +2051,10 @@ func configGenLoop(reg *registry.Registry, svc *activator.Service, dd string, wr |
| return |
| } |
| // One cache for the life of the process: what "changed" means is |
| - // exactly "differs from the last tick that could read it". |
| + // exactly "differs from the last tick that could read it". Its gate is |
| + // built here, after RunAgent armed the change index. |
| cache := newGenCache() |
| + cache.gate = newChangeGate() |
| log.Printf("agent: config generation loop armed (3s + jitter, exec-config and unit-backed apps)") |
| for { |
| time.Sleep(3*time.Second + rand.N(time.Second)) |
| @@ -2139,6 +2172,14 @@ type fastDeps struct { |
| fullDue *atomic.Bool // the backstop's request for a full pass; nil = none |
| indexActed map[string]uint64 // nil until the first good observation seeds it |
| indexBroken bool // last Poll failed and was logged; reset on success |
| + |
| + // canaryGate gates the sweep's canary pointer reads (spec §5): a canary |
| + // route stays a member of every tick, but its group's pointer is read |
| + // only when the app's scope moved, the last read failed or is the |
| + // backstop's age, or a route of the group is not live on the head that |
| + // read returned (canaryHeads). nil: every pointer is read, as before. |
| + canaryGate *release.ReadGate |
| + canaryHeads map[[3]string]string // (account, app, channel) -> the head its last good read returned |
| } |
| // restartFailure is one host's cross-pass memo of a version its restart |
| @@ -2677,8 +2718,12 @@ func fastTick(ctx context.Context, d *fastDeps, now time.Time) { |
| for _, h := range members { |
| member[h] = true |
| } |
| + opts := d.passOpts |
| + if d.canaryGate != nil { |
| + opts = append(slices.Clip(opts), release.WithPointerGate(d.canaryPointerGate(now))) |
| + } |
| if err := release.PassFilter(ctx, d.reg, d.store, d.pub, d.optDir, d.restart, |
| - func(rt registry.Route) bool { return member[rt.Host] }, d.passOpts...); err != nil { |
| + func(rt registry.Route) bool { return member[rt.Host] }, opts...); err != nil { |
| log.Printf("agent: fast converge sweep: %v", err) |
| } |
| for _, h := range members { |
| @@ -2707,6 +2752,58 @@ func fastTick(ctx context.Context, d *fastDeps, now time.Time) { |
| } |
| } |
| +// canaryPointerGate is the sweep's release.PointerGate at now. Only canary |
| +// groups are gated; every other group is read as before. A canary group is |
| +// skipped when its app scope has not moved since its last good read, that |
| +// read is younger than the backstop, and every route of the group is live on |
| +// the head it returned: then a read could only repeat what the box already |
| +// runs. Anything unsettled (an unstaged route, a failed restart rolled back, |
| +// a proof wait, a build the update settings chose over the head) keeps |
| +// today's read on every tick. |
| +func (d *fastDeps) canaryPointerGate(now time.Time) release.PointerGate { |
| + return func(acct, app, channel string, routes []registry.Route) (bool, func(*release.ChannelPointer, error)) { |
| + if !strings.HasPrefix(channel, "canary/") { |
| + return false, nil |
| + } |
| + scope, gated := release.ChangeScope(release.ChannelKey(acct, app, channel)) |
| + if !gated { |
| + return false, nil |
| + } |
| + k := [3]string{acct, app, channel} |
| + gen, due := d.canaryGate.Due(scope, now) |
| + if !due && d.canarySettled(k, routes) { |
| + return true, nil |
| + } |
| + return false, func(ptr *release.ChannelPointer, err error) { |
| + if d.canaryHeads == nil { |
| + d.canaryHeads = map[[3]string]string{} |
| + } |
| + if err != nil || ptr == nil { |
| + delete(d.canaryHeads, k) |
| + d.canaryGate.Mark(scope, gen, now, false) |
| + return |
| + } |
| + d.canaryHeads[k] = ptr.Version |
| + d.canaryGate.Mark(scope, gen, now, true) |
| + } |
| + } |
| +} |
| + |
| +// canarySettled reports whether every route of group k is live on the head |
| +// the group's last good read returned. |
| +func (d *fastDeps) canarySettled(k [3]string, routes []registry.Route) bool { |
| + head, ok := d.canaryHeads[k] |
| + if !ok || head == "" || len(routes) == 0 { |
| + return false |
| + } |
| + for _, rt := range routes { |
| + if v, err := release.LiveVersion(d.optDir, rt.Host); err != nil || v != head { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| // runFull runs the full pass set's fullPass, taking any backstop request |
| // first: a full pass that starts after the backstop's rebuild satisfies it. |
| func (d *fastDeps) runFull(ctx context.Context) { |
| @@ -2910,6 +3007,9 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| d.watch = watch |
| d.scopedPass = legs.scoped |
| d.fullDue = &backstopFullDue |
| + // The sweep's canary pointer reads are gated on the same watch |
| + // (spec §5). Built here, after RunAgent armed the index. |
| + d.canaryGate = newChangeGate() |
| } |
| // The restart leg: this process's own children bounced here, everything |
| // unit-shaped relayed to a root pass. Deliberately NOT inside fullPass |
diff --git a/internal/host/changeindex.go b/internal/host/changeindex.go| index 996dc96c8..1e768acec 100644 |
| --- a/internal/host/changeindex.go |
| +++ b/internal/host/changeindex.go |
| @@ -74,6 +74,18 @@ func armChangeIndex(ctx context.Context, getenv func(string) string, open func(c |
| return v |
| } |
| +// newChangeGate is a read gate over the armed watch, built by a loop when it |
| +// starts: RunAgent arms the index before it starts any loop, so the watch a |
| +// gate is built over is the one the process keeps. Disarmed, the watch is nil |
| +// and every read is due, as before the index (spec §5). |
| +func newChangeGate() *release.ReadGate { return newChangeGateOver(armedChangeWatch.Load()) } |
| + |
| +// newChangeGateOver is the one place the gated reads' maximum age is set: a |
| +// successful read stands in for the next one for at most the backstop. |
| +func newChangeGateOver(w *release.ChangeWatch) *release.ReadGate { |
| + return release.NewReadGate(w, release.ChangeIndexBackstop) |
| +} |
| + |
| // viewUse remembers whether the last viewStore call used the view, so the |
| // switch is logged once per change of state, never per tick. |
| var viewUse struct { |
| @@ -230,8 +242,9 @@ func (l *passLegs) full(ctx context.Context) { l.run(ctx, "serial pass stages", |
| // same legs, with build states, adopt and config sync restricted to the |
| // hosted routes whose app scope moved, and without the box-wide alias leg, |
| // so changing one app re-reads that app only. Reconcile and the moves pass |
| -// run whole, over the record view. Telemetry still resolves every hosted |
| -// app (its sink takes a whole map); it is gated separately. |
| +// run whole, over the record view. Telemetry resolves every hosted app (its |
| +// sink takes a whole map), but its reads are gated on their own scopes |
| +// (telemetryReads), so only the moved apps' files are read. |
| func (l *passLegs) scoped(ctx context.Context, moved []string) { |
| l.run(ctx, "scoped pass stages", movedRoutes(moved)) |
| } |
diff --git a/internal/host/changeindex_test.go b/internal/host/changeindex_test.go| index c3b2063b4..b27dd528a 100644 |
| --- a/internal/host/changeindex_test.go |
| +++ b/internal/host/changeindex_test.go |
| @@ -18,6 +18,7 @@ import ( |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/telemetry" |
| ) |
| // ciCountingStore is a DirStore with the change index on that records every |
| @@ -568,6 +569,11 @@ func TestScopedPassTouchesOnlyMovedApp(t *testing.T) { |
| armView(t, w.live) |
| armedMovesWorker.Store(newMovesWorker(moveCfg{})) |
| t.Cleanup(func() { armedMovesWorker.Store(nil) }) |
| + // A sink, so the telemetry leg runs (telemetryPass returns early without |
| + // one): its reads are gated too, so the scoped pass reads only shop's. |
| + var pushed []map[string]bool |
| + setAnalyticsSink(func(on map[string]bool) { pushed = append(pushed, on) }) |
| + t.Cleanup(func() { analyticsSink.Store(nil) }) |
| raw, _, err := appconfig.GenerateKey() |
| if err != nil { |
| t.Fatal(err) |
| @@ -586,7 +592,8 @@ func TestScopedPassTouchesOnlyMovedApp(t *testing.T) { |
| w.live.reset() |
| legs.full(context.Background()) |
| // The fixture exercises every box-wide leg on the full pass. |
| - for _, want := range []string{"inst/cafe.example.com/", "domains/", "control/edges"} { |
| + for _, want := range []string{"inst/cafe.example.com/", "domains/", "control/edges", |
| + telemetry.AccountKey(w.acct), telemetry.AppKey(w.acct, "cafe")} { |
| if !slices.ContainsFunc(w.live.got(), func(c string) bool { _, k, _ := strings.Cut(c, " "); return strings.HasPrefix(k, want) }) { |
| t.Fatalf("fixture: the full pass read nothing under %s: %v", want, w.live.got()) |
| } |
| @@ -607,6 +614,9 @@ func TestScopedPassTouchesOnlyMovedApp(t *testing.T) { |
| w.live.reset() |
| legs.scoped(context.Background(), []string{w.scope("shop")}) |
| + if len(pushed) != 2 || len(pushed[1]) != 0 { |
| + t.Fatalf("switch maps pushed %v, want one per pass, both all off", pushed) |
| + } |
| shop := "apps/" + w.acct + "/shop/" |
| var shopReads int |
| for _, c := range w.live.got() { |
diff --git a/internal/host/readgate_test.go b/internal/host/readgate_test.go| new file mode 100644 |
| index 000000000..13c7b2b7b |
| --- /dev/null |
| +++ b/internal/host/readgate_test.go |
| @@ -0,0 +1,464 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "runtime" |
| + "slices" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/telemetry" |
| +) |
| + |
| +// The box's GET loops read only on change or at the backstop age (S3 |
| +// change-index design §5, codex 9): config-gen.json, canary channel |
| +// pointers and telemetry.json. |
| + |
| +// gateClock is a fake clock a test moves by hand. |
| +type gateClock struct{ t time.Time } |
| + |
| +func (c *gateClock) now() time.Time { return c.t } |
| +func (c *gateClock) add(d time.Duration) { c.t = c.t.Add(d) } |
| +func newGateClock() *gateClock { return &gateClock{t: time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)} } |
| +func pollWatch(t *testing.T, w *release.ChangeWatch) { |
| + t.Helper() |
| + if _, err := w.Poll(context.Background()); err != nil { |
| + t.Fatalf("poll the change index: %v", err) |
| + } |
| +} |
| + |
| +// noIndexStore hides every optional extension of the store it wraps, so a |
| +// watch over it is unusable: the fleet store door's shape (spec §3.4). |
| +type noIndexStore struct{ release.Store } |
| + |
| +// flakyGetStore fails Get of one key while fail is set. |
| +type flakyGetStore struct { |
| + *ciCountingStore |
| + key string |
| + fail atomic.Bool |
| +} |
| + |
| +func (s *flakyGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if key == s.key && s.fail.Load() { |
| + s.note("Get", key) |
| + return nil, "", errors.New("503 slow down") |
| + } |
| + return s.ciCountingStore.Get(ctx, key) |
| +} |
| + |
| +// genGateKit is an indexed counting store with config generations for the |
| +// given apps under account bab, a polled watch over it, and a gated cache on |
| +// a fake clock. |
| +func genGateKit(t *testing.T, apps ...string) (*ciCountingStore, *release.ChangeWatch, *genCache, *gateClock, []registry.Route) { |
| + t.Helper() |
| + cs := newCICountingStore(t) |
| + var routes []registry.Route |
| + for _, app := range apps { |
| + touch(t, context.Background(), cs, "bab", app) |
| + routes = append(routes, execRoute(app+".example.com", "bab", app)) |
| + } |
| + w := release.NewChangeWatch(cs) |
| + pollWatch(t, w) |
| + clock := newGateClock() |
| + cache := newGenCache() |
| + cache.gate = release.NewReadGate(w, release.ChangeIndexBackstop) |
| + cache.now = clock.now |
| + cs.reset() |
| + return cs, w, cache, clock, routes |
| +} |
| + |
| +func genGets(cs *ciCountingStore, app string) int { |
| + return cs.count("Get", appconfig.GenKey("bab", app)) |
| +} |
| + |
| +func TestConfigGenSkipsUnmovedApp(t *testing.T) { |
| + t.Parallel() |
| + cs, w, cache, clock, routes := genGateKit(t, "shop") |
| + ctx := context.Background() |
| + genPollPass(ctx, routes, cs, cache) |
| + clock.add(3 * time.Second) |
| + pollWatch(t, w) |
| + if exec, unit, errs := genPollPass(ctx, routes, cs, cache); exec || unit || len(errs) != 0 { |
| + t.Fatalf("an unmoved app reported exec %v unit %v errs %v", exec, unit, errs) |
| + } |
| + if n := genGets(cs, "shop"); n != 1 { |
| + t.Fatalf("two ticks with no bump read config-gen.json %d time(s), want 1 (the first): %v", n, cs.got()) |
| + } |
| +} |
| + |
| +func TestConfigGenReadsMovedApp(t *testing.T) { |
| + t.Parallel() |
| + cs, w, cache, clock, routes := genGateKit(t, "shop", "cafe") |
| + ctx := context.Background() |
| + genPollPass(ctx, routes, cs, cache) |
| + touch(t, ctx, cs, "bab", "shop") |
| + cs.reset() |
| + clock.add(3 * time.Second) |
| + pollWatch(t, w) |
| + exec, _, errs := genPollPass(ctx, routes, cs, cache) |
| + if !exec || len(errs) != 0 { |
| + t.Fatalf("a moved app's config: exec %v errs %v, want a sync", exec, errs) |
| + } |
| + if genGets(cs, "shop") != 1 || genGets(cs, "cafe") != 0 { |
| + t.Fatalf("after shop's bump the tick read %v, want shop's config-gen.json only", cs.got()) |
| + } |
| +} |
| + |
| +func TestConfigGenReadsAtMaxAge(t *testing.T) { |
| + t.Parallel() |
| + cs, w, cache, clock, routes := genGateKit(t, "shop") |
| + ctx := context.Background() |
| + genPollPass(ctx, routes, cs, cache) |
| + clock.add(release.ChangeIndexBackstop - time.Second) |
| + pollWatch(t, w) |
| + genPollPass(ctx, routes, cs, cache) |
| + if n := genGets(cs, "shop"); n != 1 { |
| + t.Fatalf("a read younger than the backstop was repeated: %d reads", n) |
| + } |
| + clock.add(time.Second) |
| + genPollPass(ctx, routes, cs, cache) |
| + if n := genGets(cs, "shop"); n != 2 { |
| + t.Fatalf("a read the backstop's age old was not repeated: %d reads, want 2", n) |
| + } |
| +} |
| + |
| +func TestConfigGenRetriesAfterError(t *testing.T) { |
| + t.Parallel() |
| + cs, w, cache, clock, routes := genGateKit(t, "shop") |
| + ctx := context.Background() |
| + fs := &flakyGetStore{ciCountingStore: cs, key: appconfig.GenKey("bab", "shop")} |
| + genPollPass(ctx, routes, fs, cache) |
| + touch(t, ctx, cs, "bab", "shop") |
| + pollWatch(t, w) |
| + cs.reset() |
| + fs.fail.Store(true) |
| + clock.add(3 * time.Second) |
| + if exec, _, errs := genPollPass(ctx, routes, fs, cache); exec || len(errs) != 1 { |
| + t.Fatalf("the failed read: exec %v errs %v, want one error and no sync", exec, errs) |
| + } |
| + // No new bump: the failed read alone makes the next tick read again, |
| + // and that read sees the change. |
| + fs.fail.Store(false) |
| + clock.add(3 * time.Second) |
| + pollWatch(t, w) |
| + if exec, _, errs := genPollPass(ctx, routes, fs, cache); !exec || len(errs) != 0 { |
| + t.Fatalf("the tick after a failed read: exec %v errs %v, want the change synced", exec, errs) |
| + } |
| + if n := genGets(cs, "shop"); n != 2 { |
| + t.Fatalf("config-gen.json read %d time(s) after the bump, want 2 (the failure, the retry)", n) |
| + } |
| +} |
| + |
| +// ---- canary pointers ------------------------------------------------------ |
| + |
| +// canaryKit is a box with one canary route (fastapp on canary/dev, account |
| +// bab) whose head is v1, over an indexed counting store. |
| +type canaryKit struct { |
| + env *fastFixture |
| + ds *release.DirStore |
| + cs *ciCountingStore |
| + d *fastDeps |
| + clock *gateClock |
| +} |
| + |
| +const canaryHost = "canary.example.com" |
| + |
| +func newCanaryKit(t *testing.T, usable bool) *canaryKit { |
| + t.Helper() |
| + env := newFastFixture(t) |
| + cs := newCICountingStore(t) |
| + k := &canaryKit{env: env, ds: cs.DirStore, cs: cs, clock: newGateClock()} |
| + k.ship(t, "v1") |
| + env.addRoute(t, canaryHost, "canary/dev") |
| + var store release.Store = cs |
| + if !usable { |
| + store = noIndexStore{cs} |
| + } |
| + env.store = store |
| + k.d = env.deps(nil) |
| + k.d.watch = release.NewChangeWatch(store) |
| + k.d.canaryGate = release.NewReadGate(k.d.watch, release.ChangeIndexBackstop) |
| + cs.reset() |
| + return k |
| +} |
| + |
| +func (k *canaryKit) ship(t *testing.T, ver string) { |
| + t.Helper() |
| + ctx := context.Background() |
| + plat := runtime.GOOS + "-" + runtime.GOARCH |
| + m := release.Manifest{App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| + if err := release.Ship(ctx, k.ds, "bab", m, map[string][]byte{plat: []byte("build " + ver)}); err != nil { |
| + t.Fatalf("Ship %s: %v", ver, err) |
| + } |
| + if err := release.Promote(ctx, k.ds, k.env.priv, "bab", "fastapp", "canary/dev", ver); err != nil { |
| + t.Fatalf("Promote %s: %v", ver, err) |
| + } |
| +} |
| + |
| +// tick runs one fastTick at the kit's clock, checks the canary is still a |
| +// member, and returns how many times the pointer has been read so far. |
| +func (k *canaryKit) tick(t *testing.T, d time.Duration) int { |
| + t.Helper() |
| + k.clock.add(d) |
| + if m := k.d.fastMembers(k.clock.now()); !slices.Contains(m, canaryHost) { |
| + t.Fatalf("the canary route left the sweep: members %v", m) |
| + } |
| + fastTick(context.Background(), k.d, k.clock.now()) |
| + return k.cs.count("Get", release.ChannelKey("bab", "fastapp", "canary/dev")) |
| +} |
| + |
| +func (k *canaryKit) live(t *testing.T) string { |
| + t.Helper() |
| + v, err := release.LiveVersion(k.env.optDir, canaryHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return v |
| +} |
| + |
| +func TestCanaryPointerGated(t *testing.T) { |
| + t.Parallel() |
| + k := newCanaryKit(t, true) |
| + if n := k.tick(t, 0); n != 1 || k.live(t) != "v1" { |
| + t.Fatalf("first tick: %d pointer reads, live %q; want 1 and v1", n, k.live(t)) |
| + } |
| + for i := range 3 { |
| + if n := k.tick(t, 2*time.Second); n != 1 { |
| + t.Fatalf("quiet tick %d read the pointer again (%d reads)", i, n) |
| + } |
| + } |
| + // A promote bumps the app's scope: the next tick reads and adopts. |
| + k.ship(t, "v2") |
| + if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v2" { |
| + t.Fatalf("after a promote: %d pointer reads, live %q; want 2 and v2", n, k.live(t)) |
| + } |
| + if n := k.tick(t, 2*time.Second); n != 2 { |
| + t.Fatalf("the tick after the adopt read the pointer again (%d reads)", n) |
| + } |
| + // The age bound: a pointer read the backstop's age ago is read again. |
| + if n := k.tick(t, release.ChangeIndexBackstop); n != 3 { |
| + t.Fatalf("at the backstop age: %d pointer reads, want 3", n) |
| + } |
| +} |
| + |
| +// A canary group whose routes are not live on the head its last read |
| +// returned (here the live link was moved off it, as a rolled-back restart |
| +// leaves it) keeps today's read on every tick until it is. |
| +func TestCanaryPointerReadWhileUnsettled(t *testing.T) { |
| + t.Parallel() |
| + k := newCanaryKit(t, true) |
| + k.tick(t, 0) |
| + k.env.stageLive(t, canaryHost, "v0") |
| + if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v1" { |
| + t.Fatalf("an unsettled group: %d pointer reads, live %q; want 2 and v1", n, k.live(t)) |
| + } |
| + if n := k.tick(t, 2*time.Second); n != 2 { |
| + t.Fatalf("the group settled again but its pointer was read (%d reads)", n) |
| + } |
| +} |
| + |
| +// A stable route keeps today's path: the gate never skips its pointer. |
| +func TestCanaryGateLeavesStableRoutes(t *testing.T) { |
| + t.Parallel() |
| + k := newCanaryKit(t, true) |
| + skip, read := k.d.canaryPointerGate(k.clock.now())("bab", "fastapp", "stable", nil) |
| + if skip || read != nil { |
| + t.Fatalf("the canary gate took a stable group: skip %v read %v", skip, read != nil) |
| + } |
| +} |
| + |
| +// ---- telemetry ------------------------------------------------------------ |
| + |
| +func telemetryRoutes(t *testing.T, acct string, apps ...string) *registry.Registry { |
| + t.Helper() |
| + reg := edgesTestRegistry(t) |
| + for _, app := range apps { |
| + if err := reg.Upsert(registry.Route{Host: app + ".example.com", Kind: "instance", Account: acct, App: app, Status: "ready"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + return reg |
| +} |
| + |
| +func TestTelemetryAccountReadOncePerPass(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + cs := newCICountingStore(t) |
| + reg := telemetryRoutes(t, "bab", "shop", "cafe", "deli") |
| + if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| + st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + cs.reset() |
| + on, err := telemetrySwitchesWith(ctx, cs, reg, newTelemetryReads(nil), newGateClock().now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| + t.Fatalf("three apps in one account read its telemetry.json %d time(s), want 1: %v", n, cs.got()) |
| + } |
| + if !on["bab/shop"] || !on["bab/cafe"] || !on["bab/deli"] { |
| + t.Fatalf("switches %v, want every app on from the account", on) |
| + } |
| +} |
| + |
| +func TestTelemetryGatedWithAgeBound(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + cs := newCICountingStore(t) |
| + reg := telemetryRoutes(t, "bab", "shop", "cafe") |
| + if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| + st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + w := release.NewChangeWatch(cs) |
| + pollWatch(t, w) |
| + r := newTelemetryReads(w) |
| + clock := newGateClock() |
| + pass := func() map[string]bool { |
| + t.Helper() |
| + on, err := telemetrySwitchesWith(ctx, cs, reg, r, clock.now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return on |
| + } |
| + cs.reset() |
| + if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { |
| + t.Fatalf("first pass switches %v", on) |
| + } |
| + if n := cs.count("Get", ""); n != 3 { |
| + t.Fatalf("the first pass made %d reads, want 3 (account, shop, cafe): %v", n, cs.got()) |
| + } |
| + clock.add(time.Minute) |
| + pollWatch(t, w) |
| + cs.reset() |
| + if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { |
| + t.Fatalf("a pass that skipped its reads lost the last good values: %v", on) |
| + } |
| + if n := cs.count("Get", ""); n != 0 { |
| + t.Fatalf("a quiet pass read %v", cs.got()) |
| + } |
| + // cafe's own setting moves: only cafe's file is read. |
| + if err := telemetry.Update(ctx, cs, telemetry.AppKey("bab", "cafe"), func(st *telemetry.Settings) { |
| + st.Analytics = &telemetry.Analytics{Count: telemetry.CountOff} |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + clock.add(time.Minute) |
| + pollWatch(t, w) |
| + cs.reset() |
| + if on := pass(); !on["bab/shop"] || on["bab/cafe"] { |
| + t.Fatalf("after cafe's change switches %v, want shop on, cafe off", on) |
| + } |
| + if got := cs.got(); !slices.Equal(got, []string{"Get " + telemetry.AppKey("bab", "cafe")}) { |
| + t.Fatalf("after cafe's change the pass read %v, want cafe's telemetry.json only", got) |
| + } |
| + // The age bound: every file is read again at the backstop's age. |
| + cs.reset() |
| + clock.add(release.ChangeIndexBackstop) |
| + pass() |
| + if n := cs.count("Get", ""); n != 3 { |
| + t.Fatalf("at the backstop age the pass made %d reads, want 3: %v", n, cs.got()) |
| + } |
| +} |
| + |
| +// A failed read keeps the last good value and is retried on the next pass. |
| +func TestTelemetryKeepsLastGoodOnError(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + cs := newCICountingStore(t) |
| + reg := telemetryRoutes(t, "bab", "shop") |
| + if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| + st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + fs := &flakyGetStore{ciCountingStore: cs, key: telemetry.AccountKey("bab")} |
| + w := release.NewChangeWatch(cs) |
| + pollWatch(t, w) |
| + r := newTelemetryReads(w) |
| + clock := newGateClock() |
| + if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { |
| + t.Fatalf("first pass %v", on) |
| + } |
| + clock.add(release.ChangeIndexBackstop) // due by age |
| + fs.fail.Store(true) |
| + cs.reset() |
| + if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { |
| + t.Fatalf("a failed account read dropped the last good value: %v", on) |
| + } |
| + if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| + t.Fatalf("the due pass read the account %d time(s), want 1", n) |
| + } |
| + fs.fail.Store(false) |
| + cs.reset() |
| + clock.add(time.Second) |
| + telemetrySwitchesWith(ctx, fs, reg, r, clock.now()) |
| + if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| + t.Fatalf("the pass after a failed read read the account %d time(s), want 1 (a retry)", n) |
| + } |
| +} |
| + |
| +// ---- an unusable watch ---------------------------------------------------- |
| + |
| +// Review Focus 5: a watch that cannot vouch (here, a store with no |
| +// conditional read, the fleet door's shape) makes every gated read due on |
| +// every tick. The gates are the production ones, built from the armed watch. |
| +func TestGatesDueWhenWatchUnusable(t *testing.T) { |
| + cs := newCICountingStore(t) |
| + plain := noIndexStore{cs} |
| + w := release.NewChangeWatch(plain) |
| + if _, err := w.Poll(context.Background()); err == nil { |
| + t.Fatal("fixture: a watch over a store with no conditional read polled cleanly") |
| + } |
| + armedChangeWatch.Store(w) |
| + t.Cleanup(func() { armedChangeWatch.Store(nil) }) |
| + ctx := context.Background() |
| + clock := newGateClock() |
| + |
| + // config-gen.json |
| + touch(t, ctx, cs, "bab", "shop") |
| + cache := newGenCache() |
| + cache.gate = newChangeGate() |
| + cache.now = clock.now |
| + routes := []registry.Route{execRoute("shop.example.com", "bab", "shop")} |
| + cs.reset() |
| + for range 3 { |
| + clock.add(3 * time.Second) |
| + genPollPass(ctx, routes, plain, cache) |
| + } |
| + if n := genGets(cs, "shop"); n != 3 { |
| + t.Fatalf("config-gen.json read %d time(s) in three ticks, want 3", n) |
| + } |
| + |
| + // telemetry.json, through the production state |
| + reg := telemetryRoutes(t, "bab", "shop") |
| + cs.reset() |
| + for range 3 { |
| + clock.add(time.Minute) |
| + if _, err := telemetrySwitchesWith(ctx, plain, reg, currentTelemetryReads(), clock.now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if a, b := cs.count("Get", telemetry.AccountKey("bab")), cs.count("Get", telemetry.AppKey("bab", "shop")); a != 3 || b != 3 { |
| + t.Fatalf("telemetry reads in three passes: account %d app %d, want 3 each", a, b) |
| + } |
| + |
| + // canary pointers |
| + k := newCanaryKit(t, false) |
| + k.d.canaryGate = newChangeGate() |
| + k.d.watch = nil // fastTick's own poll would fail the same way; the gate reads the armed watch |
| + for i := range 3 { |
| + if n := k.tick(t, 2*time.Second); n != i+1 { |
| + t.Fatalf("tick %d: %d pointer reads, want %d", i, n, i+1) |
| + } |
| + } |
| +} |
diff --git a/internal/host/telemetry_pass.go b/internal/host/telemetry_pass.go| index 1891c8d19..25ce64b03 100644 |
| --- a/internal/host/telemetry_pass.go |
| +++ b/internal/host/telemetry_pass.go |
| @@ -8,7 +8,9 @@ package host |
| import ( |
| "context" |
| "log" |
| + "sync" |
| "sync/atomic" |
| + "time" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -21,15 +23,72 @@ var analyticsSink atomic.Pointer[func(map[string]bool)] |
| // SetAnalyticsOn), the way setUpstreamSource publishes the prober table. |
| func setAnalyticsSink(fn func(map[string]bool)) { analyticsSink.Store(&fn) } |
| +// telemetryReads is what the switch pass remembers between passes while the |
| +// change index is armed (S3 change-index design §5): a gate over the app and |
| +// account scopes, and the last good settings object per key, so a read the |
| +// gate skips, or one that fails, reuses the last good value. A failed read |
| +// marks its scope failed, so the next pass reads it again. |
| +// |
| +// Disarmed, each pass gets a fresh one over a nil watch: every read is due, |
| +// nothing is remembered, and a failed read leaves its apps off, as before. |
| +type telemetryReads struct { |
| + // mu is held across a whole pass, reads included: the fast converge loop |
| + // and the instances loop both run the pass and take turns here. Nothing |
| + // else is locked under it. |
| + mu sync.Mutex |
| + w *release.ChangeWatch |
| + gate *release.ReadGate |
| + last map[string]telemetry.Settings // telemetry.AccountKey / AppKey -> last good object |
| +} |
| + |
| +func newTelemetryReads(w *release.ChangeWatch) *telemetryReads { |
| + return &telemetryReads{w: w, gate: newChangeGateOver(w), last: map[string]telemetry.Settings{}} |
| +} |
| + |
| +// telemetryGate is the process's telemetryReads for the armed watch. It is |
| +// rebuilt whenever the armed watch differs from the one it was built over, |
| +// so a pass that ran before arming never pins a nil watch. |
| +var telemetryGate struct { |
| + mu sync.Mutex |
| + r *telemetryReads |
| +} |
| + |
| +// currentTelemetryReads is the state a pass uses: telemetryGate's while a |
| +// watch is armed, a fresh one otherwise. |
| +func currentTelemetryReads() *telemetryReads { |
| + w := armedChangeWatch.Load() |
| + if w == nil { |
| + return newTelemetryReads(nil) |
| + } |
| + telemetryGate.mu.Lock() |
| + defer telemetryGate.mu.Unlock() |
| + if telemetryGate.r == nil || telemetryGate.r.w != w { |
| + telemetryGate.r = newTelemetryReads(w) |
| + } |
| + return telemetryGate.r |
| +} |
| + |
| // telemetrySwitches resolves the count switch for each distinct |
| // (account, app) among the box's routes. Status routes carry no app and |
| -// are skipped. One Resolve (two Gets) per pair; a pair that fails to |
| -// resolve is left OFF and logged — off is the safe side of this switch. |
| +// are skipped. |
| func telemetrySwitches(ctx context.Context, s release.Store, reg *registry.Registry) (map[string]bool, error) { |
| + return telemetrySwitchesWith(ctx, s, reg, currentTelemetryReads(), time.Now()) |
| +} |
| + |
| +// telemetrySwitchesWith is telemetrySwitches over r at now. Each account's |
| +// telemetry.json is read at most once per pass, however many of its apps the |
| +// box serves, and each app's at most once; a read is skipped when r's gate |
| +// says its scope has not moved and r holds a last good value. A pair with no |
| +// good value at either level is left OFF and logged: off is the safe side of |
| +// this switch. |
| +func telemetrySwitchesWith(ctx context.Context, s release.Store, reg *registry.Registry, r *telemetryReads, now time.Time) (map[string]bool, error) { |
| routes, err := reg.List() |
| if err != nil { |
| return nil, err |
| } |
| + r.mu.Lock() |
| + defer r.mu.Unlock() |
| + pass := map[string]error{} // keys this pass has settled, and the error when there is no good value |
| on := map[string]bool{} |
| seen := map[string]bool{} |
| for _, rt := range routes { |
| @@ -42,18 +101,63 @@ func telemetrySwitches(ctx context.Context, s release.Store, reg *registry.Regis |
| continue |
| } |
| seen[key] = true |
| - res, err := telemetry.Resolve(ctx, s, sqid, rt.App) |
| + acct, err := r.read(ctx, s, telemetry.AccountKey(sqid), now, pass) |
| if err != nil { |
| log.Printf("agent: telemetry resolve %s: %v (analytics stays off)", key, err) |
| continue |
| } |
| - if res.Count == telemetry.CountEdge { |
| + app, err := r.read(ctx, s, telemetry.AppKey(sqid, rt.App), now, pass) |
| + if err != nil { |
| + log.Printf("agent: telemetry resolve %s: %v (analytics stays off)", key, err) |
| + continue |
| + } |
| + if telemetry.Fold(acct, app).Count == telemetry.CountEdge { |
| on[key] = true |
| } |
| } |
| + // Objects no route needs any more are forgotten, so a returning app is |
| + // read afresh. |
| + for k := range r.last { |
| + if _, used := pass[k]; !used { |
| + delete(r.last, k) |
| + } |
| + } |
| return on, nil |
| } |
| +// read is key's settings for this pass: settled once per pass in pass. The |
| +// last good value stands in when the gate says the key's scope has not moved; |
| +// otherwise the object is read. A failed read keeps the last good value, logs |
| +// it and marks the scope failed; with no last good value it returns the |
| +// error. |
| +func (r *telemetryReads) read(ctx context.Context, s release.Store, key string, now time.Time, pass map[string]error) (telemetry.Settings, error) { |
| + if err, done := pass[key]; done { |
| + return r.last[key], err |
| + } |
| + scope, gated := release.ChangeScope(key) |
| + gen, due := r.gate.Due(scope, now) |
| + if prev, ok := r.last[key]; ok && gated && !due { |
| + pass[key] = nil |
| + return prev, nil |
| + } |
| + st, _, err := telemetry.Read(ctx, s, key) |
| + if gated { |
| + r.gate.Mark(scope, gen, now, err == nil) |
| + } |
| + if err == nil { |
| + r.last[key] = st |
| + pass[key] = nil |
| + return st, nil |
| + } |
| + if prev, ok := r.last[key]; ok { |
| + log.Printf("agent: telemetry read %s: %v (the last good value stands; retried next pass)", key, err) |
| + pass[key] = nil |
| + return prev, nil |
| + } |
| + pass[key] = err |
| + return telemetry.Settings{}, err |
| +} |
| + |
| // telemetryPass resolves and pushes; errors are one log line each. |
| func telemetryPass(ctx context.Context, s release.Store, reg *registry.Registry) { |
| sink := analyticsSink.Load() |
diff --git a/internal/release/adopt.go b/internal/release/adopt.go| index 2867b8797..d3b5c56b3 100644 |
| --- a/internal/release/adopt.go |
| +++ b/internal/release/adopt.go |
| @@ -139,6 +139,62 @@ type passConfig struct { |
| // tags is "<acct>/<app>" -> release version -> its newest semver tag, read |
| // once per pass for apps with a route under a major setting. |
| tags map[string]map[string]string |
| + |
| + // pointerGate is WithPointerGate's gate; nil (every caller but the |
| + // agent's fast sweep) reads every group's pointer, as before. |
| + pointerGate PointerGate |
| +} |
| + |
| +// PointerGate lets a caller skip a channel pointer read that cannot have |
| +// anything new (S3 change-index design §5). PassFilter asks it once per |
| +// (account, app, channel) group, with the group's routes this pass would |
| +// adopt. skip true leaves the whole group out of this pass: no pointer read |
| +// and no adoption, exactly as if extra had refused its routes. Otherwise |
| +// read, when non-nil, is called once with the outcome of the group's read: |
| +// the verified pointer, or the read's error or the ratchet's refusal. |
| +type PointerGate func(acct, app, channel string, routes []registry.Route) (skip bool, read func(ptr *ChannelPointer, err error)) |
| + |
| +// WithPointerGate arms g for this pass. The agent's fast sweep passes one |
| +// that skips settled canary groups whose app scope has not moved. |
| +func WithPointerGate(g PointerGate) PassOption { |
| + return func(c *passConfig) { c.pointerGate = g } |
| +} |
| + |
| +// gatePointers asks cfg.pointerGate about every group adopts admits, and |
| +// returns adopts narrowed to the groups it did not skip, with each kept |
| +// group's read callback. |
| +func (cfg *passConfig) gatePointers(routes []registry.Route, adopts func(registry.Route) bool) (func(registry.Route) bool, map[[3]string]func(*ChannelPointer, error)) { |
| + if cfg.pointerGate == nil { |
| + return adopts, nil |
| + } |
| + groupOf := func(rt registry.Route) [3]string { return [3]string{RouteAccount(rt), rt.App, routeChannel(rt)} } |
| + groups := map[[3]string][]registry.Route{} |
| + var order [][3]string |
| + for _, rt := range routes { |
| + if !adopts(rt) { |
| + continue |
| + } |
| + k := groupOf(rt) |
| + if _, ok := groups[k]; !ok { |
| + order = append(order, k) |
| + } |
| + groups[k] = append(groups[k], rt) |
| + } |
| + reads := map[[3]string]func(*ChannelPointer, error){} |
| + skipped := map[[3]string]bool{} |
| + for _, k := range order { |
| + skip, read := cfg.pointerGate(k[0], k[1], k[2], groups[k]) |
| + switch { |
| + case skip: |
| + skipped[k] = true |
| + case read != nil: |
| + reads[k] = read |
| + } |
| + } |
| + if len(skipped) == 0 { |
| + return adopts, reads |
| + } |
| + return func(rt registry.Route) bool { return adopts(rt) && !skipped[groupOf(rt)] }, reads |
| } |
| // WithRunningVersion retains root's verified running build through a repoint. |
| @@ -584,6 +640,9 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| adopts := func(rt registry.Route) bool { |
| return (Adoptable(rt) || SidecarAdoptable(rt)) && (extra == nil || extra(rt)) |
| } |
| + // WithPointerGate: a skipped group leaves the pass before anything is |
| + // selected for it. |
| + adopts, pointerRead := cfg.gatePointers(routes, adopts) |
| selected := make(map[string]hostwrite.Work, len(routes)) |
| cfg.historySelected = make(map[string]historySelection) |
| for _, rt := range routes { |
| @@ -688,7 +747,11 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| continue |
| } |
| ptr, err := Channel(ctx, s, pub, k.acct, k.app, k.channel) |
| + read := pointerRead[[3]string{k.acct, k.app, k.channel}] |
| if err != nil { |
| + if read != nil { |
| + read(nil, err) |
| + } |
| pointers[k] = nil // resolved-and-failed: don't retry per host |
| if errors.Is(err, os.ErrNotExist) { |
| // SKIP, never abort: one app with no pointer must not stop |
| @@ -732,10 +795,16 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| // Any error from Admit refuses the pointer, ErrRatchetUnpersisted |
| // included, not only ErrPointerWithoutDigests. |
| if rerr := cfg.ratchet.Admit(k.acct, ptr); rerr != nil { |
| + if read != nil { |
| + read(nil, rerr) |
| + } |
| pointers[k] = nil // refused: the group keeps what it has |
| fmt.Printf(" %s/%s/%s: channel pointer refused: %v\n", k.acct, k.app, k.channel, rerr) |
| continue |
| } |
| + if read != nil { |
| + read(ptr, nil) |
| + } |
| pointers[k] = ptr |
| } |
diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go| index 1ffd286c8..85cbcf96f 100644 |
| --- a/internal/release/adopt_test.go |
| +++ b/internal/release/adopt_test.go |
| @@ -9,6 +9,7 @@ import ( |
| "errors" |
| "os" |
| "path/filepath" |
| + "slices" |
| "strconv" |
| "strings" |
| "testing" |
| @@ -581,6 +582,67 @@ func TestPassFilterOnlyTouchesRoutesExtraAccepts(t *testing.T) { |
| } |
| } |
| +// WithPointerGate (S3 change-index design §5): a group the gate skips is out |
| +// of the pass, its pointer unread; every other group is read as before and |
| +// its outcome reported once. |
| +func TestPassFilterPointerGate(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + for _, ch := range []string{"canary/x", "stable"} { |
| + if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { |
| + t.Fatalf("Promote %s: %v", ch, err) |
| + } |
| + } |
| + addRoute(t, reg, "canary.example.com", "myapp", "canary/x", false) |
| + addRoute(t, reg, "stable.example.com", "myapp", "stable", false) |
| + addRoute(t, reg, "beta.example.com", "myapp", "beta", false) // nothing promoted |
| + |
| + asked := map[string][]string{} |
| + read := map[string]string{} |
| + gate := func(acct, app, channel string, routes []registry.Route) (bool, func(*ChannelPointer, error)) { |
| + for _, rt := range routes { |
| + asked[channel] = append(asked[channel], rt.Host) |
| + } |
| + if channel == "canary/x" { |
| + return true, nil |
| + } |
| + return false, func(ptr *ChannelPointer, err error) { |
| + switch { |
| + case err != nil: |
| + read[channel] += "error;" |
| + default: |
| + read[channel] += ptr.Version + ";" |
| + } |
| + } |
| + } |
| + if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, nil, WithPointerGate(gate)); err != nil { |
| + t.Fatalf("PassFilter: %v", err) |
| + } |
| + want := map[string][]string{"canary/x": {"canary.example.com"}, "stable": {"stable.example.com"}, "beta": {"beta.example.com"}} |
| + if len(asked) != len(want) { |
| + t.Fatalf("the gate was asked about %v, want %v", asked, want) |
| + } |
| + for ch, hosts := range want { |
| + if !slices.Equal(asked[ch], hosts) { |
| + t.Fatalf("the gate was asked about %s with %v, want %v", ch, asked[ch], hosts) |
| + } |
| + } |
| + if v, _ := LiveVersion(opt, "canary.example.com"); v != "" { |
| + t.Errorf("a skipped group adopted: canary.example.com live = %q", v) |
| + } |
| + if v, _ := LiveVersion(opt, "stable.example.com"); v != "v1" { |
| + t.Errorf("stable.example.com live = %q, want v1", v) |
| + } |
| + if read["stable"] != "v1;" || read["beta"] != "error;" || read["canary/x"] != "" { |
| + t.Errorf("read outcomes %v, want stable v1 once, beta's error once, nothing for the skipped group", read) |
| + } |
| +} |
| + |
| // writeBackdatedPointer signs and writes a channel pointer with an arbitrary |
| // PromotedAt, bypassing Promote/signPointer's own "stamp now" behavior — the |
| // only way to get an EXPIRED canary pointer into a store under test, since |
diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go| index c723e06f5..33f7a5e0f 100644 |
| --- a/internal/telemetry/telemetry.go |
| +++ b/internal/telemetry/telemetry.go |
| @@ -107,18 +107,25 @@ type Resolved struct { |
| // Resolve reads both levels and folds them over the defaults. A store |
| // error fails the resolve; an absent object at either level is fine. |
| func Resolve(ctx context.Context, s release.Store, sqid, app string) (Resolved, error) { |
| - r := Resolved{LogLevel: LogLevelError, LogLevelFrom: "default", Count: CountOff, CountFrom: "default", Door: DoorOn, DoorFrom: "default"} |
| if s == nil { |
| - return r, nil |
| + return Fold(Settings{}, Settings{}), nil |
| } |
| acct, _, err := Read(ctx, s, AccountKey(sqid)) |
| if err != nil { |
| - return r, err |
| + return Fold(Settings{}, Settings{}), err |
| } |
| appSt, _, err := Read(ctx, s, AppKey(sqid, app)) |
| if err != nil { |
| - return r, err |
| + return Fold(Settings{}, Settings{}), err |
| } |
| + return Fold(acct, appSt), nil |
| +} |
| + |
| +// Fold is Resolve over settings already read: the account level, then the |
| +// app level, over the built-in defaults. A reader that caches the two |
| +// objects (the box's switch pass) folds them here. |
| +func Fold(acct, appSt Settings) Resolved { |
| + r := Resolved{LogLevel: LogLevelError, LogLevelFrom: "default", Count: CountOff, CountFrom: "default", Door: DoorOn, DoorFrom: "default"} |
| for _, lvl := range []struct { |
| from string |
| st Settings |
| @@ -135,5 +142,5 @@ func Resolve(ctx context.Context, s release.Store, sqid, app string) (Resolved, |
| } |
| } |
| } |
| - return r, nil |
| + return r |
| } |