amadan
harness: infra container, MinIO + audit sink + dnsmasq, CAS probe (step 3)
MinIO RELEASE.2025-09-07 ignores If-Match on DeleteObject: stale-ETag deletes succeed where S3 answers 412. Recorded in RUNLOG with the measured S3 column.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+374
−0
harness/RUNLOG.md+78 −0harness/infra/casprobe.py+129 −0harness/infra/infra-setup.sh+167 −0
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 306.699s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.738s
ok github.com/carlosframework/platform/cmd/moveprobe 92.098s
ok github.com/carlosframework/platform/cmd/zddprobe 0.707s
ok github.com/carlosframework/platform/internal/activator 251.371s
ok github.com/carlosframework/platform/internal/agentsock 0.016s
ok github.com/carlosframework/platform/internal/alerting 0.003s
ok github.com/carlosframework/platform/internal/analytics 0.019s
ok github.com/carlosframework/platform/internal/appconfig 2.434s
ok github.com/carlosframework/platform/internal/apperr 0.070s
ok github.com/carlosframework/platform/internal/applog 0.141s
ok github.com/carlosframework/platform/internal/awsinv 0.003s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.840s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.699s
ok github.com/carlosframework/platform/internal/changelog 0.318s
ok github.com/carlosframework/platform/internal/claimactivation 0.803s
ok github.com/carlosframework/platform/internal/clamd 0.055s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 403.819s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.010s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.005s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.005s
ok github.com/carlosframework/platform/internal/domains 2.335s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.005s
ok github.com/carlosframework/platform/internal/edge 44.866s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.131s
ok github.com/carlosframework/platform/internal/gate 1.657s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.016s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.153s
ok github.com/carlosframework/platform/internal/host 181.278s
ok github.com/carlosframework/platform/internal/hostactors 6.340s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.012s
ok github.com/carlosframework/platform/internal/hostbox 2.737s
ok github.com/carlosframework/platform/internal/hostclaim 0.536s
ok github.com/carlosframework/platform/internal/hostenv 0.012s
ok github.com/carlosframework/platform/internal/hostevidence 0.012s
ok github.com/carlosframework/platform/internal/hostexec 5.147s
ok github.com/carlosframework/platform/internal/hostfence 0.031s
ok github.com/carlosframework/platform/internal/hostfiles 10.606s
ok github.com/carlosframework/platform/internal/hostfloor 0.013s
ok github.com/carlosframework/platform/internal/hostguard 1.034s
ok github.com/carlosframework/platform/internal/hostops 1.604s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.012s
ok github.com/carlosframework/platform/internal/hostquarantine 0.014s
ok github.com/carlosframework/platform/internal/hostrelocate 0.446s
ok github.com/carlosframework/platform/internal/hostresolve 0.135s
ok github.com/carlosframework/platform/internal/hostwrite 0.012s
ok github.com/carlosframework/platform/internal/infraview 0.013s
ok github.com/carlosframework/platform/internal/instancectl 5.236s
ok github.com/carlosframework/platform/internal/instanceexec 2.770s
ok github.com/carlosframework/platform/internal/instancefs 0.019s
ok github.com/carlosframework/platform/internal/instancekey 0.015s
ok github.com/carlosframework/platform/internal/instanceupdates 0.018s
ok github.com/carlosframework/platform/internal/layout 1.558s
ok github.com/carlosframework/platform/internal/lbhealth 0.002s
ok github.com/carlosframework/platform/internal/lease 9.836s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.005s
ok github.com/carlosframework/platform/internal/ledger 0.013s
ok github.com/carlosframework/platform/internal/mail 0.178s
ok github.com/carlosframework/platform/internal/maintenance 0.094s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.558s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.067s
ok github.com/carlosframework/platform/internal/metrics 0.004s
ok github.com/carlosframework/platform/internal/moves 21.107s
ok github.com/carlosframework/platform/internal/moves/archive 0.830s
ok github.com/carlosframework/platform/internal/objstore 0.179s
ok github.com/carlosframework/platform/internal/ops 0.179s
ok github.com/carlosframework/platform/internal/park 0.209s
ok github.com/carlosframework/platform/internal/privatefile 0.003s
2026/10/20 00:50:31 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/20 00:50:31 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:50:31 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/20 00:50:31 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:50:31 provision: an app's records could not be listed: they are skipped
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:50:31 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:50:31 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:50:31 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:50:31 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:50:31 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:50:31 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:50:31 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:50:31 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:50:31 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:50:31 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:50:31 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:50:31 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:50:31 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:50:31 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:50:31 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:50:31 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:50:31 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:50:31 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:50:31 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:50:31 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/20 00:50:31 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/20 00:50:32 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:50:32 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/20 00:50:32 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (4.50s)
plan14_root_reads_test.go:940: root read census: 12 discrepancies
changed root read internal/hostfence/writer.go:OpenRootFenceStore; review class, task, sites and digest
changed root source envelope harness
changed root source envelope harness/infra
changed root source envelope internal/host
changed root source envelope internal/hostfence
changed root source envelope internal/publicrelease
changed root source envelope internal/s3compat
unclassified root read harness/host-setup.sh:external file:external@0
unclassified root read harness/host-user-data.sh:external file:external@0
unclassified root read harness/infra/infra-setup.sh:external file:external@0
unclassified root read internal/s3compat/harness.go:readHarnessFile file:ReadFile@189,file:Stat@55
changed reclassified declaration internal/hostfence/writer.go:OpenRootFenceStore
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:51:02 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/20 00:51:02 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/20 00:51:02 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/20 00:51:02 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io hibernate false -> true
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged acme.go.tito.io status-named false -> true
2026/10/20 00:51:02 provision: converged acme.go.tito.io status-named true -> false
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:51:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:51:02 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:51:02 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:51:02 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/20 00:51:02 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:51:02 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:51:02 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:51:02 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:51:02 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:02 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:51:02 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/20 00:51:02 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/20 00:51:02 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:51:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:51:02 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:51:02 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:51:02 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/20 00:51:03 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:51:03 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:51:03 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:51:03 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:51:03 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:51:03 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/20 00:51:03 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/20 00:51:03 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:51:03 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/20 00:51:03 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/20 00:51:03 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:51:03 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/20 00:51:03 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/20 00:51:03 provision: dry-run: would converge owned.go.tito.io seed
2026/10/20 00:51:03 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:51:03 provision: converged owned.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/20 00:51:03 provision: converged owned.go.tito.io hibernate false -> true
2026/10/20 00:51:03 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/20 00:51:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: removed retired static route site.go.tito.io (record gone)
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/20 00:51:03 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/20 00:51:03 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:03 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:51:03 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:51:03 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:51:03 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:51:03 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/20 00:51:03 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:51:03 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:51:03 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:51:03 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:51:03 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/20 00:51:03 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/20 00:51:03 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:51:03 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/20 00:51:03 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/20 00:51:03 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:51:03 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/20 00:51:03 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:51:03 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:03 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/20 00:51:04 provision: a.shop.example.com: swapped for import sd_asojrb4o6xiey53lkfsbhrqthm (the original route is journaled)
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_asojrb4o6xiey53lkfsbhrqthm committed: its gate is cleared and its journal row dropped
2026/10/20 00:51:04 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_x5kkaafad5xb6pdc5bcfe244um rolled back to its original route
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_x5kkaafad5xb6pdc5bcfe244um: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_x5kkaafad5xb6pdc5bcfe244um is cleaned up and its journal row retired
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_jr6q64vrlibxkne2codaq3pqni rolled back to its original route
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_jr6q64vrlibxkne2codaq3pqni: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_jr6q64vrlibxkne2codaq3pqni is cleaned up and its journal row retired
2026/10/20 00:51:04 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_s5h6nt7upckhzatehtxqw4xgxa
2026/10/20 00:51:04 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/20 00:51:04 provision: dry-run: would converge a.shop.example.com seed source
2026/10/20 00:51:04 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:04 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/20 00:51:04 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/20 00:51:04 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_zrs5me4kqdvx32xgmsnbppxbu4 cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_fjssbli6zytpfov73kgpnpbi6a cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_bkgdink6rmldlaywri42jqim3e is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_yk75zvizm2alokuohj525xxiqq is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_gebpykudc3g42sqq6h53gsbvny is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:51:04 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/20 00:51:04 provision: b.shop.example.com: import sd_rhixekqyqm2etrt3vq4cwm24ri left alone this pass: the import journal could not be read
2026/10/20 00:51:04 provision: c.shop.example.com: import sd_7snd7rlr5d6mp2kwocp5mjis5e left alone this pass: the import journal could not be read
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_vaue633jf27lkgirim5s4sbaum committed: its gate is cleared and its journal row dropped
2026/10/20 00:51:04 provision: a.shop.example.com: not converging import sd_rcycvn2xcw4xrvpoqx5bl3dy2u: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:51:04 provision: a.shop.example.com: not converging import sd_i4pcegol23ttmbm2r6gvyuj6xq: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:51:04 provision: a.shop.example.com: not converging import sd_kxmbtims5qjpp2ezkzv34y2nui: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:51:04 provision: a.shop.example.com: not converging import sd_7m6viiye4n3rmk4su7xwpc5ydq: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:51:04 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:04 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/20 00:51:04 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:04 provision: a.shop.example.com: import sd_eqsvh6o4i4z3amvol5bfhq2hkq committed: its gate is cleared and its journal row dropped
2026/10/20 00:51:04 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:04 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:05 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/20 00:51:05 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_zrywjterajkewfg4paeyin3bqa: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_zrywjterajkewfg4paeyin3bqa rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/20 00:51:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:05 provision: removed instance route acme.go.tito.io (its seed sd_moubf7uxqjb5ddx3aos67ntjqa was aborted); the next pass rolls it back
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_moubf7uxqjb5ddx3aos67ntjqa: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_moubf7uxqjb5ddx3aos67ntjqa rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_dvlm2erhu6bgbua5vwm4ky36um has a row this pass may not remove; leaving the seed aborted
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_r2aplyz5a3joz5l4jqyyom42s4 rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:05 provision: removed instance route acme.go.tito.io (its seed sd_k7sa74m4xj7uylxfw4l7n6hiym was aborted); the next pass rolls it back
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_k7sa74m4xj7uylxfw4l7n6hiym has no route on this box: its local data is removed
2026/10/20 00:51:05 provision: acme.go.tito.io: aborted seed sd_k7sa74m4xj7uylxfw4l7n6hiym rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_3phvgzeyza2oimd3aiixmhy6je (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_3phvgzeyza2oimd3aiixmhy6je rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_3phvgzeyza2oimd3aiixmhy6je is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_4r3zsj25wnnlnu7mvtgtsb3lcq (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_4r3zsj25wnnlnu7mvtgtsb3lcq rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_4r3zsj25wnnlnu7mvtgtsb3lcq is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_tn2urfipznm46eao6624rqaysy (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_tn2urfipznm46eao6624rqaysy rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_tn2urfipznm46eao6624rqaysy: the host's replica is left alone: its record no longer names this seed
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_tn2urfipznm46eao6624rqaysy is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_koy4c25hfa4l72toicm7a7kxby (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_koy4c25hfa4l72toicm7a7kxby rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_koy4c25hfa4l72toicm7a7kxby rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_koy4c25hfa4l72toicm7a7kxby is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_gco6smgambl7qv3x2l4zaraf5e (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_gco6smgambl7qv3x2l4zaraf5e rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_nwppjrzyors4szihhaxm7fv5oi (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_nwppjrzyors4szihhaxm7fv5oi is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_2lxhsy76oevhzqixrxhxnyzmam (the original route is journaled)
2026/10/20 00:51:05 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_2lxhsy76oevhzqixrxhxnyzmam left alone this pass: its record could not be read
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_t4j4lcdk4i6dnk4biiks5qnmc4 (the original route is journaled)
2026/10/20 00:51:05 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_t4j4lcdk4i6dnk4biiks5qnmc4 left alone this pass: its record could not be read
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_v7ziwxrfzqefuinmug52c43tpq (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_v7ziwxrfzqefuinmug52c43tpq: close the import gate: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_v7ziwxrfzqefuinmug52c43tpq rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_v7ziwxrfzqefuinmug52c43tpq is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_miwegrbh6ut52uljsd7azhmqoy (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_miwegrbh6ut52uljsd7azhmqoy: close the import gate: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_miwegrbh6ut52uljsd7azhmqoy rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_miwegrbh6ut52uljsd7azhmqoy is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_p3ppcwns4orwsv4c2cvpvau5c4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_p3ppcwns4orwsv4c2cvpvau5c4 rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_p3ppcwns4orwsv4c2cvpvau5c4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_p3ppcwns4orwsv4c2cvpvau5c4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_5wpi6xyprejxcau6pmtciit3v4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_5wpi6xyprejxcau6pmtciit3v4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_5wpi6xyprejxcau6pmtciit3v4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_t4kjvheevhgipcgzjfjbymdzta (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_t4kjvheevhgipcgzjfjbymdzta: retire its instance key: injected failure at retire-key
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_t4kjvheevhgipcgzjfjbymdzta rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_t4kjvheevhgipcgzjfjbymdzta is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_35riayr6gpjdq3mgozbdn4enxm (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_35riayr6gpjdq3mgozbdn4enxm rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_35riayr6gpjdq3mgozbdn4enxm is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_7qg2oixhgcm4h3zioppjfxag5u (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_7qg2oixhgcm4h3zioppjfxag5u: remove its local data: injected failure at remove-local
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_7qg2oixhgcm4h3zioppjfxag5u rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_7qg2oixhgcm4h3zioppjfxag5u is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_iwhwuhl5wqjctfewmgdrtuj4im (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iwhwuhl5wqjctfewmgdrtuj4im rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iwhwuhl5wqjctfewmgdrtuj4im is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_dvgeww63yiakyzy6k734ho4bca (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_dvgeww63yiakyzy6k734ho4bca: remove its local data: injected failure at remove-local-done
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_dvgeww63yiakyzy6k734ho4bca rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_dvgeww63yiakyzy6k734ho4bca is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_y6lxa3yvfp6lhmjo2nxpbve7p4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_y6lxa3yvfp6lhmjo2nxpbve7p4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_y6lxa3yvfp6lhmjo2nxpbve7p4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_rnbra3aumiip4mporm3ql7k27u (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_rnbra3aumiip4mporm3ql7k27u: restore the original route: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_rnbra3aumiip4mporm3ql7k27u rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_rnbra3aumiip4mporm3ql7k27u is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_6wf4merzuj2likjr4xfyinroi4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_6wf4merzuj2likjr4xfyinroi4: restore the original route: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_6wf4merzuj2likjr4xfyinroi4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_6wf4merzuj2likjr4xfyinroi4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_vxwfky723i4nhkczgz2ejybpp4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_vxwfky723i4nhkczgz2ejybpp4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_ahnz4nh66xrx5fxaeqb6ttpkve (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ahnz4nh66xrx5fxaeqb6ttpkve rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ahnz4nh66xrx5fxaeqb6ttpkve: delete its snapshot: injected failure at delete-snapshot
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ahnz4nh66xrx5fxaeqb6ttpkve is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_rsgzl6uluknnatkhpfr3t4uxiu (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_rsgzl6uluknnatkhpfr3t4uxiu rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_rsgzl6uluknnatkhpfr3t4uxiu is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_raey726ptf6n5ecumidos4lmvy (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_raey726ptf6n5ecumidos4lmvy rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_raey726ptf6n5ecumidos4lmvy: delete the host's replica: injected failure at delete-replica
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_raey726ptf6n5ecumidos4lmvy is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_a36zfjejnxzt5au6mzco4hvptm (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_a36zfjejnxzt5au6mzco4hvptm rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_a36zfjejnxzt5au6mzco4hvptm is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_brrredbiz6pllojoxbyjd3gxde (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_brrredbiz6pllojoxbyjd3gxde rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_brrredbiz6pllojoxbyjd3gxde: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_brrredbiz6pllojoxbyjd3gxde is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_ajqppmmqdopaxxcd2pp5beghey (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ajqppmmqdopaxxcd2pp5beghey rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ajqppmmqdopaxxcd2pp5beghey: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ajqppmmqdopaxxcd2pp5beghey is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_zgbtofskvhjlfnznima3bk3sp4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_zgbtofskvhjlfnznima3bk3sp4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_zgbtofskvhjlfnznima3bk3sp4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: mark import sd_zgbtofskvhjlfnznima3bk3sp4 rolled back: injected failure at cas
2026/10/20 00:51:05 provision: a.shop.example.com: aborted seed sd_zgbtofskvhjlfnznima3bk3sp4 rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_kdw54zw43ns7soeli42srtz3ka (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_kdw54zw43ns7soeli42srtz3ka rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_kdw54zw43ns7soeli42srtz3ka is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: aborted seed sd_kdw54zw43ns7soeli42srtz3ka rolled back (no box had swapped it)
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_iryft3n3mrktyznrh4jhqvihvq (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iryft3n3mrktyznrh4jhqvihvq rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iryft3n3mrktyznrh4jhqvihvq rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iryft3n3mrktyznrh4jhqvihvq rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_iryft3n3mrktyznrh4jhqvihvq is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_y4zsn25avbf2legs6uf7rvnui4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_y4zsn25avbf2legs6uf7rvnui4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_y4zsn25avbf2legs6uf7rvnui4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_4i47swyh3ht2egta74sfyl6fg4 (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_4i47swyh3ht2egta74sfyl6fg4 rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_4i47swyh3ht2egta74sfyl6fg4 is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_uczgh3efsqdlfzgziqsukteynq (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_uczgh3efsqdlfzgziqsukteynq rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_uczgh3efsqdlfzgziqsukteynq is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_ulbudznoyizyghizaxkjur4sym (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ulbudznoyizyghizaxkjur4sym rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ulbudznoyizyghizaxkjur4sym is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_albs7vi2qqhlac3gqo6fqsru4m (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_albs7vi2qqhlac3gqo6fqsru4m rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_albs7vi2qqhlac3gqo6fqsru4m is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_m6j4nwhq3qcro66pnduerhj5wu (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_m6j4nwhq3qcro66pnduerhj5wu rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_m6j4nwhq3qcro66pnduerhj5wu is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:51:05 provision: a.shop.example.com: swapped for import sd_ee6bsd4sa7z4ijczc6nn7exnme (the original route is journaled)
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ee6bsd4sa7z4ijczc6nn7exnme rolled back to its original route
2026/10/20 00:51:05 provision: a.shop.example.com: import sd_ee6bsd4sa7z4ijczc6nn7exnme is cleaned up and its journal row retired
2026/10/20 00:51:05 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_2loso76xvscev7jovb6kxxcytm (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_2loso76xvscev7jovb6kxxcytm has no route on this box: its local data is removed and no route is put back
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_2loso76xvscev7jovb6kxxcytm is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_i34p2z2kv746fjmylxwwtn3u4i (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_xnjvvsd3gmrnl47fpmcth4utry (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_y5wgvj2c453vnrkpxxjpxzl6fq (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_y5wgvj2c453vnrkpxxjpxzl6fq cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_ekw5ft7gal7rusd557vk66ul3q (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_ekw5ft7gal7rusd557vk66ul3q is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/20 00:51:06 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/20 00:51:06 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_y2rfi7zvynyx7wemwcui5tgugq (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_nqbeexqga3ewuervxcoknjiz6a (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_bhantbhcb4g3pcaqk6udke5msu (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_35efs7ffzndequinjbeycs6jly (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_goqolakkucc6zrwtvforb6sq3m (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_goqolakkucc6zrwtvforb6sq3m rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_vsmte4vyqht4tidq7ikcxz36f4 (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_vsmte4vyqht4tidq7ikcxz36f4 has no route on this box: its local data is removed and no route is put back
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_vsmte4vyqht4tidq7ikcxz36f4 is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_53qlnc6fb5gpby5qznatn63rsi (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_53qlnc6fb5gpby5qznatn63rsi: remove its local data: injected failure at remove-local
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_53qlnc6fb5gpby5qznatn63rsi is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_tesiyks5ja7vvdnlmeguyjbmgu (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_tesiyks5ja7vvdnlmeguyjbmgu: a route appeared at the host, so its local database is left and the rollback waits
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_b2kmi5i3j3xt3x6qhmhfnscy4e (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_b2kmi5i3j3xt3x6qhmhfnscy4e rollback deferred: no fence generation: registry busy
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_ltbtcy77bv4wbndwjo6lz6bi5q (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_ltbtcy77bv4wbndwjo6lz6bi5q rollback deferred: no fence generation: the generation source answered 0
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_a7qtu4tgrsi42vy5aidobkyqly (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_mbgvkjyej6svbslty22sxjndsy (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_qothiw74dh3stjfq4waxr2g24u (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/20 00:51:06 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_3vflkbstn5ldyajucobd72xgfm (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_3vflkbstn5ldyajucobd72xgfm rolled back to its original route
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_3vflkbstn5ldyajucobd72xgfm is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_4z6o5kmrj3nwedgfgjc4aucehy (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_4z6o5kmrj3nwedgfgjc4aucehy rolled back to its original route
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_4z6o5kmrj3nwedgfgjc4aucehy is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_onfgfw4kc6oneeegp6rruqdqyi (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_onfgfw4kc6oneeegp6rruqdqyi rolled back to its original route
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_onfgfw4kc6oneeegp6rruqdqyi is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_qmb2d4xsgzbcpoqgfsmvhbjsda (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_qmb2d4xsgzbcpoqgfsmvhbjsda: retire its instance key: the env file is a symlink
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_qmb2d4xsgzbcpoqgfsmvhbjsda rolled back to its original route
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_qmb2d4xsgzbcpoqgfsmvhbjsda is cleaned up and its journal row retired
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_n77dgp65rp2tsmh4rabnakauwe (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_4cggmjrpennoy535ccc6vruw4y (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_hu7irbcvlorwtauydpfsb4tbqa refused: there is no route at a.shop.example.com to replace
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_as37jfmf35i7c2cfuf3i4dxmtq refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_whlom42fnpiwrmaw2klwttka3u refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_yj4fnl3jqux6zjxuzxbr6cpgvy refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_px4s6ssldmktg7nlgzjh6lgdza refused: the route belongs to app other
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_5h3bc5jyddzokftjulkpu42iey refused: the route is filed under another account
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_p6gb22x4fgepy4qamrhrvjb4wa refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_fch5jzfl4oqqta3tg6uk76ev3e refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_xf42es5ulq6pklkx42hr7d3zau refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_fjrpee4b7quid3jbqyhu2tfqce refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_hvczwtxpght6425snrgiqsoome refused: the route is not an address route this import can take over (socket)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_zof7qgjq7peep6t2awdun3eyuu refused: the route is not an address route this import can take over (no address)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_buhmwi6r2mehy4qbsvxiqaaktm refused: the route is not an address route this import can take over (provisioned)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_6twkzimi3dhpgk5x2m3ehhy3pa refused: the route is not an address route this import can take over (kind static)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_f3452spqn4z6ozzotk37csrnii refused: the route is not an address route this import can take over (backing)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_fjqxdkbmkhmddmucao2lopnul4 refused: the route is not an address route this import can take over (tunnel)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_3z52gwpvllcpdmt4cqp3se2wfe refused: the route is not an address route this import can take over (caps)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_nl52rvxoyaaiws6vwa5prqzabm refused: the route is not an address route this import can take over (platform)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_fdtqkrlj7x5zafy7rvnibpzyky refused: the route is not an address route this import can take over (canary)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_da5os6ypklwnjy5cwavtygojc4 refused: the route is not an address route this import can take over (kind sidecar)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_gfc3iejkhab3roeqbzbsa7ywwy refused: the route is not an address route this import can take over (region)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_gdqt7bpe7u7bam4ajfgpqfwbgq refused: the route is not an address route this import can take over (fleet_box)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_y3klms3zpmhhekuuft7nvjnruq refused: the route is not an address route this import can take over (hibernate)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_kp64dcqhw5xu3ya7zgxxdflsbi refused: the route is not an address route this import can take over (claim_owner)
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_5xumykgg7ik7klhkyzwxgkcuw4 refused: this box does not run instances nested
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_xikyvwvbq6vvj3xmuorib6x45i refused: this box does not run instances nested
2026/10/20 00:51:06 provision: a.shop.example.com: import sd_bi4czesa7p3llml3m5tk6ya7ye refused: claim control is on for this box
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_l6dfc3yjyhot5gmu34wcgl2pwe (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: swapped for import sd_m6rwuz6a7jyfofbgkz6ec3mv5e (the original route is journaled)
2026/10/20 00:51:06 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/20 00:51:06 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_532niubty2h2wa375m77s5yu24: its verify hash is not a sha256
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_25cdvvukq5baqft2dqo7bfan2i: its verify hash is not a sha256
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_dklm2ek2keb4tsq4yvcwrwkimq: its verify hash is not a sha256
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/20 00:51:06 provision: A.shop.example.com/..: not swapping for import sd_bqaxqwzuwmc25k726s74n3v4gu: its host or seed id is malformed (seeds: not a host)
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_lwlstma753wifv6cqh5ogl34dm: its record does not describe a nested, hibernating instance
2026/10/20 00:51:06 provision: a.shop.example.com: not swapping for import sd_olzfnpplgwdfrfc6anvq3emsfu: its record does not describe a nested, hibernating instance
2026/10/20 00:51:07 provision: a.shop.example.com: not swapping for import sd_3pj7piude6eygos3s7xy5p7ggi: its record does not describe a nested, hibernating instance
2026/10/20 00:51:07 provision: a.shop.example.com: not swapping for import sd_hijyfhlu7k4kzrowpwu6aldede: its record does not describe a nested, hibernating instance
2026/10/20 00:51:07 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/20 00:51:07 provision: a.shop.example.com: swapped for import sd_3omx4novziymiizd37ydl2h2cy (the original route is journaled)
2026/10/20 00:51:07 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/20 00:51:07 provision: a.shop.example.com: swapped for import sd_g3d5wtt3ysbxu5palmi4xbt5qu (the original route is journaled)
2026/10/20 00:51:07 provision: a.shop.example.com: swapped for import sd_ptadroy7q2uyp3t6seshdzw5jq (the original route is journaled)
2026/10/20 00:51:07 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/20 00:51:07 provision: dry-run: would consider swapping a.shop.example.com for import sd_f4rc74svoe7ulowavqpmdooaj4
2026/10/20 00:51:07 provision: a.shop.example.com: import sd_y52wc2sl3xjmxileius4zrqxam rolled back to its original route
2026/10/20 00:51:07 provision: a.shop.example.com: import sd_y52wc2sl3xjmxileius4zrqxam: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:51:07 provision: a.shop.example.com: import sd_y52wc2sl3xjmxileius4zrqxam is cleaned up and its journal row retired
2026/10/20 00:51:07 provision: a.shop.example.com: import sd_uiougllggcoo4xsx6pn3um6iba left alone: the import journal holds seed sd_l26hyhd6tw6dknpvm3rcgs645y at swapped
2026/10/20 00:51:07 provision: a.shop.example.com: not minting import sd_ctxrwnhxbkiweg2uu4n6ywcuci: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:51:07 provision: a.shop.example.com: not minting import sd_2ackb6gskh34j4o6memadpsa54: it is placed but not committed, and only its swap makes its row
2026/10/20 00:51:07 provision: a.shop.example.com: swapped for import sd_dsxkzm257cxultvis6f7jh6diu (the original route is journaled)
2026/10/20 00:51:07 provision: a.shop.example.com: not minting import sd_dsxkzm257cxultvis6f7jh6diu: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:51:07 provision: a.shop.example.com: not minting import sd_dsxkzm257cxultvis6f7jh6diu: it is placed but not committed, and only its swap makes its row
2026/10/20 00:51:07 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:51:07 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/20 00:51:07 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/20 00:51:07 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/20 00:51:07 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/20 00:51:07 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/20 00:51:07 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:51:07 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/20 00:51:07 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/20 00:51:07 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:51:07 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:51:07 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/20 00:51:07 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/20 00:51:07 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:51:07 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/20 00:51:07 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:51:07 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:51:07 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/20 00:51:07 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:51:07 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:51:07 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:07 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/20 00:51:07 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:07 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:07 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:51:07 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:51:07 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:51:07 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:51:07 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:51:07 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:51:07 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL github.com/carlosframework/platform/internal/provision 36.492s
ok github.com/carlosframework/platform/internal/publicrelease 0.009s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.555s
ok github.com/carlosframework/platform/internal/relay 7.235s
ok github.com/carlosframework/platform/internal/release 23.823s
ok github.com/carlosframework/platform/internal/rollout 3.915s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/rootparticipation 0.057s
ok github.com/carlosframework/platform/internal/rootstate 0.026s
ok github.com/carlosframework/platform/internal/runs 0.575s
ok github.com/carlosframework/platform/internal/s3compat 0.004s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.509s
ok github.com/carlosframework/platform/internal/seeds 0.011s
ok github.com/carlosframework/platform/internal/sim/hostlife 9.692s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 3.861s
ok github.com/carlosframework/platform/internal/status 0.067s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.215s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.008s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.569s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/harness/RUNLOG.md b/harness/RUNLOG.md| index 943909b0c..880841b9d 100644 |
| --- a/harness/RUNLOG.md |
| +++ b/harness/RUNLOG.md |
| @@ -86,3 +86,81 @@ tcp deb.debian.org:443 (provisioning egress) -> CONNECTED |
| ``` |
| The scratch container never reached the host role. **Step 2: GREEN** (2026-10-20T00:45:27Z). |
| + |
| +## Step 3: infra, MinIO, the audit log, the CAS probe |
| + |
| +`python3 harness/hssm.py harness/infra/infra-setup.sh` (on host, 00:45-00:49 UTC): |
| +container `infra` (203.0.113.10, profile box, 2 GiB), MinIO `RELEASE.2025-09-07T16-13-09Z` |
| +and mc `RELEASE.2025-08-13T08-35-41Z`, the CI pins, sha256-verified against |
| +`carlos/platform .amadan/ci.d/30-minio-contract` @ 17362eaf (arm64 rows). MinIO listens on |
| +203.0.113.10:9000 (plus the container's own loopback), web console off, no host port |
| +published. Audit webhook -> `harness-audit-sink` on 127.0.0.1:9100 -> |
| +`/var/log/harness/minio-audit.jsonl`. dnsmasq on 203.0.113.10:53 (`*.s3.harness.test`, |
| +upstream 203.0.113.1); hbr0's DHCP now hands out 203.0.113.10 as the resolver. |
| +Buckets (versioning enabled on all three): `harness-deployment`, `harness-data-ie`, |
| +`harness-data-uk`. MinIO users (H2, policy `readwrite`): harness-leo, -sophie, -dennis, |
| +-signer, -fencewriter, -operator; keys in `/root/harness/minio-users.json` (host, 0600). |
| + |
| +### The conditional-write probe |
| + |
| +`harness/infra/casprobe.py` (stdlib SigV4, same bytes against both stores). S3 column is |
| +**measured**, not quoted from docs: run from DOCS-T1 with kass against the harness's own |
| +AWS bucket `carlos-harness-640012953448` (unversioned), prefix `casprobe-s3/`, 00:4x UTC. |
| +MinIO columns: run on host as `harness-operator` against a scratch unversioned bucket |
| +`harness-casprobe` (removed after) and against versioned `harness-deployment`. |
| + |
| +| Case | S3 (measured) | MinIO unversioned | MinIO versioned | Same as S3? | |
| +|---|---|---|---|---| |
| +| PUT If-None-Match:\* on a new key | 200 | 200 | 200 (+version id) | yes | |
| +| PUT If-None-Match:\* on an existing key | 412 PreconditionFailed | 412 | 412 | yes | |
| +| PUT If-Match: matching ETag, quoted (as returned) | 200 | 200 | 200 | yes | |
| +| PUT If-Match: stale ETag, quoted | 412 | 412 | 412 | yes | |
| +| PUT If-Match: matching ETag, **unquoted** | 200 | 200 | 200 | yes | |
| +| PUT If-Match: stale ETag, unquoted | 412 | 412 | 412 | yes | |
| +| PUT If-Match on a missing key | 404 NoSuchKey | 404 NoSuchKey | 404 NoSuchKey | yes | |
| +| **DELETE If-Match: stale ETag, quoted** | **412** | **204, object deleted** | **204, delete marker written** | **NO** | |
| +| DELETE If-Match: matching ETag, quoted | 204 | 204 | 204 | yes | |
| +| DELETE If-Match: matching ETag, unquoted | 204 | 204 | 204 | yes | |
| +| **DELETE If-Match: stale ETag, unquoted** | **412** | **204, deleted** | **204, delete marker** | **NO** | |
| +| **DELETE If-Match on a missing key** | **404 NoSuchKey** | **204** | **204** | **NO** (callers treat 404 as success, so same outcome) | |
| +| DELETE If-Match: \* on an existing key | 204 | 204 | 204 | yes | |
| +| versioned: PUT If-None-Match:\* when current version is a delete marker | not measured (harness AWS bucket is unversioned by the approved packet) | n/a | 200 | S3 documents: succeeds | |
| +| versioned: PUT If-Match: pre-delete ETag, current version a delete marker | not measured | n/a | 404 NoSuchKey | S3 documents: 404 | |
| +| versioned: DELETE If-Match: ETag, current version a delete marker | not measured | n/a | 204 (no new marker) | unknown | |
| + |
| +Also: MinIO's 412 to a PUT carries the object's current `ETag` header; S3's does not |
| +(harmless; no caller reads it). |
| + |
| +**Finding: MinIO RELEASE.2025-09-07 ignores If-Match on DeleteObject.** A stale-ETag |
| +conditional delete succeeds and deletes (unversioned) or writes a delete marker |
| +(versioned). Every `release.DeleteIfMatch` caller therefore runs **unconditionally** in the |
| +harness: `internal/domains/claims.go:537` (claim handover), `internal/hostops/record.go:121`, |
| +`operations.go:331`, `purge.go:131`, `purge_control.go:402,433`, `mutation_boundary.go:188`, |
| +`internal/mail/claim_gate.go:140`, plus litestream's lease release. A harness run can never |
| +exercise their `ErrCASConflict` path, and a lost-race delete that S3 would refuse succeeds. |
| +PUT-side CAS (backfill-done's If-Match write, create-only collect, markers, pointers) matches |
| +S3 row for row, quoted and unquoted. Fix belongs in P2's proxy: enforce If-Match on DELETE |
| +(HEAD + compare + DELETE under a per-key lock) so the harness answers 412 as S3 does. |
| + |
| +### Audit log |
| + |
| +46 audit entries on the probe's objects, 29 carrying `If-*` request headers; every request |
| +logged, with `requestHeader` (If-Match / If-None-Match verbatim), `api.name`, |
| +`api.statusCode`, bucket, object and `accessKey`: |
| + |
| +``` |
| + DeleteObject 204 if-match x13 |
| + DeleteObject 204 - x12 |
| + PutObject 200 if-match x4 |
| + PutObject 200 if-none-match x3 |
| + PutObject 200 - x5 |
| + PutObject 404 if-match x3 |
| + PutObject 412 if-match x4 |
| + PutObject 412 if-none-match x2 |
| +entry keys: accessKey api deploymentid event remotehost requestHeader requestHost requestID |
| + requestPath responseHeader time trigger userAgent version |
| +``` |
| + |
| +Check: conditional PUT answers 412 where S3 does (all rows); If-Match on DELETE recorded |
| +(differs, above); one audit line per request with `If-*` present. **Step 3: GREEN** with the |
| +DELETE fidelity hole recorded (2026-10-20T00:49:53Z). |
diff --git a/harness/infra/casprobe.py b/harness/infra/casprobe.py| new file mode 100644 |
| index 000000000..417a10824 |
| --- /dev/null |
| +++ b/harness/infra/casprobe.py |
| @@ -0,0 +1,129 @@ |
| +#!/usr/bin/env python3 |
| +"""Conditional-write probe: what a store does with If-Match / If-None-Match on |
| +PUT and DELETE, the semantics the claim code's CAS writers rely on. |
| + |
| + casprobe.py ENDPOINT BUCKET PREFIX [--versioned] |
| + credentials from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN |
| + ENDPOINT is path-style: http://203.0.113.10:9000 or https://s3.eu-west-1.amazonaws.com |
| + |
| +Python 3 standard library only, so the same bytes run against MinIO on the |
| +harness bridge and against real S3 (the harness's own bucket). Prints one JSON |
| +object per case: name, request, status, error code, and the ETag/version seen. |
| +Writes only under PREFIX and deletes what it wrote (every version, when |
| +versioned). |
| +""" |
| +import datetime, hashlib, hmac, json, os, sys, urllib.parse, urllib.request, urllib.error, uuid |
| +import xml.etree.ElementTree as ET |
| + |
| +REGION = os.environ.get("AWS_REGION", "eu-west-1") |
| + |
| + |
| +def sign(method, url, headers, body=b""): |
| + u = urllib.parse.urlsplit(url) |
| + now = datetime.datetime.now(datetime.timezone.utc) |
| + amzdate, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") |
| + headers = dict(headers) |
| + headers["host"] = u.netloc |
| + headers["x-amz-date"] = amzdate |
| + headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest() |
| + tok = os.environ.get("AWS_SESSION_TOKEN") |
| + if tok: |
| + headers["x-amz-security-token"] = tok |
| + signed = sorted(k.lower() for k in headers) |
| + lower = {k.lower(): str(v).strip() for k, v in headers.items()} |
| + q = urllib.parse.parse_qsl(u.query, keep_blank_values=True) |
| + cq = "&".join(f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}" for k, v in sorted(q)) |
| + creq = "\n".join([method, urllib.parse.quote(u.path, safe="/-_.~"), cq, |
| + "".join(f"{k}:{lower[k]}\n" for k in signed), ";".join(signed), lower["x-amz-content-sha256"]]) |
| + scope = f"{day}/{REGION}/s3/aws4_request" |
| + sts = "\n".join(["AWS4-HMAC-SHA256", amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()]) |
| + k = ("AWS4" + os.environ["AWS_SECRET_ACCESS_KEY"]).encode() |
| + for part in (day, REGION, "s3", "aws4_request"): |
| + k = hmac.new(k, part.encode(), hashlib.sha256).digest() |
| + sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() |
| + headers["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={os.environ['AWS_ACCESS_KEY_ID']}/{scope}, " |
| + f"SignedHeaders={';'.join(signed)}, Signature={sig}") |
| + headers.pop("host") |
| + return headers |
| + |
| + |
| +def call(method, url, headers=None, body=b""): |
| + h = sign(method, url, headers or {}, body) |
| + req = urllib.request.Request(url, data=body if method in ("PUT", "POST") else None, method=method, headers=h) |
| + try: |
| + with urllib.request.urlopen(req, timeout=20) as r: |
| + return r.status, dict(r.headers), r.read() |
| + except urllib.error.HTTPError as e: |
| + return e.code, dict(e.headers), e.read() |
| + |
| + |
| +def code(body): |
| + try: |
| + return ET.fromstring(body).findtext("Code") or "" |
| + except Exception: |
| + return "" |
| + |
| + |
| +def main(): |
| + a = sys.argv[1:] |
| + versioned = "--versioned" in a |
| + a = [x for x in a if x != "--versioned"] |
| + endpoint, bucket, prefix = a[0].rstrip("/"), a[1], a[2].strip("/") + "/" + uuid.uuid4().hex[:8] |
| + obj = lambda k: f"{endpoint}/{bucket}/{prefix}/{k}" |
| + out = [] |
| + |
| + def case(name, method, key, headers=None, body=b"", expect=None): |
| + st, h, b = call(method, obj(key), headers, body) |
| + etag = h.get("ETag") or h.get("Etag") or "" |
| + row = {"case": name, "req": f"{method} {key} " + " ".join(f"{k}: {v}" for k, v in (headers or {}).items()), |
| + "status": st, "code": code(b), "etag": etag, "version": h.get("x-amz-version-id", "")} |
| + print(json.dumps(row), flush=True) |
| + out.append(row) |
| + return st, etag |
| + |
| + st, e1 = case("PUT If-None-Match:* on a new key", "PUT", "k1", {"If-None-Match": "*"}, b"one") |
| + case("PUT If-None-Match:* on an existing key", "PUT", "k1", {"If-None-Match": "*"}, b"two") |
| + st, e2 = case("PUT If-Match: matching ETag (quoted, as returned)", "PUT", "k1", {"If-Match": e1}, b"three") |
| + case("PUT If-Match: stale ETag (quoted)", "PUT", "k1", {"If-Match": e1}, b"four") |
| + st, e3 = case("PUT If-Match: matching ETag, unquoted", "PUT", "k1", {"If-Match": e2.strip('"')}, b"five") |
| + cur = e3 if st == 200 else e2 |
| + case("PUT If-Match: stale ETag, unquoted", "PUT", "k1", {"If-Match": e1.strip('"')}, b"six") |
| + case("PUT If-Match on a missing key", "PUT", "nokey", {"If-Match": cur}, b"x") |
| + case("DELETE If-Match: stale ETag (quoted)", "DELETE", "k1", {"If-Match": e1}) |
| + case("DELETE If-Match: matching ETag (quoted)", "DELETE", "k1", {"If-Match": cur}) |
| + st, ek = case("(setup) PUT k2", "PUT", "k2", {}, b"k2") |
| + case("DELETE If-Match: matching ETag, unquoted", "DELETE", "k2", {"If-Match": ek.strip('"')}) |
| + st, ek = case("(setup) PUT k3", "PUT", "k3", {}, b"k3") |
| + case("DELETE If-Match: stale ETag, unquoted", "DELETE", "k3", {"If-Match": e1.strip('"')}) |
| + case("DELETE If-Match on a missing key", "DELETE", "nokey2", {"If-Match": cur}) |
| + case("DELETE If-Match: * on an existing key", "DELETE", "k3", {"If-Match": "*"}) |
| + if versioned: |
| + # k1 now has a delete marker as its current version (deleted above). |
| + case("versioned: PUT If-None-Match:* where the current version is a delete marker", "PUT", "k1", |
| + {"If-None-Match": "*"}, b"seven") |
| + st, e5 = case("(setup) PUT k4", "PUT", "k4", {}, b"k4") |
| + case("(setup) DELETE k4 unconditionally (adds a delete marker)", "DELETE", "k4") |
| + case("versioned: PUT If-Match: the pre-delete ETag, current version a delete marker", "PUT", "k4", |
| + {"If-Match": e5}, b"k4b") |
| + case("versioned: DELETE If-Match: ETag, current version a delete marker", "DELETE", "k4", {"If-Match": e5}) |
| + |
| + # Cleanup: every version (versioned) or every key under the prefix. |
| + lst = f"{endpoint}/{bucket}?" + ("versions&" if versioned else "list-type=2&") + "prefix=" + urllib.parse.quote(prefix + "/", safe="") |
| + st, h, b = call("GET", lst) |
| + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" |
| + n = 0 |
| + try: |
| + root = ET.fromstring(b) |
| + for tag in ("Version", "DeleteMarker", "Contents"): |
| + for el in root.iter(ns + tag): |
| + k, v = el.findtext(ns + "Key"), el.findtext(ns + "VersionId") |
| + q = "?versionId=" + urllib.parse.quote(v, safe="") if (versioned and v) else "" |
| + call("DELETE", f"{endpoint}/{bucket}/{urllib.parse.quote(k)}{q}") |
| + n += 1 |
| + except ET.ParseError: |
| + pass |
| + print(json.dumps({"cleanup": n, "list_status": st}), flush=True) |
| + |
| + |
| +if __name__ == "__main__": |
| + main() |
diff --git a/harness/infra/infra-setup.sh b/harness/infra/infra-setup.sh| new file mode 100644 |
| index 000000000..947d651e7 |
| --- /dev/null |
| +++ b/harness/infra/infra-setup.sh |
| @@ -0,0 +1,167 @@ |
| +#!/bin/bash |
| +# Step 3: the infra container (203.0.113.10): MinIO with the audit log, dnsmasq. |
| +# Run as root on the harness host, after host-setup.sh. |
| +# |
| +# MinIO and mc are the release CI's pins, verified by sha256 |
| +# (carlos/platform .amadan/ci.d/30-minio-contract @ 17362eaf, arm64 rows). |
| +# MinIO binds 203.0.113.10:9000 only; its web console is off; nothing is |
| +# published on the host. Keys live on the host under /root/harness (0700). |
| +set -euo pipefail |
| + |
| +H=/root/harness |
| +mkdir -p $H/bin && chmod 700 $H |
| +MINIO_RELEASE=RELEASE.2025-09-07T16-13-09Z |
| +MC_RELEASE=RELEASE.2025-08-13T08-35-41Z |
| +minio_sha=5c83cd2cf151717ba0243f73e1c7802ff36e272b67144bdd7f1f7d684fd6f03d |
| +mc_sha=14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c |
| + |
| +fetch() { # name url sha dest |
| + if [ -f "$4" ] && [ "$(sha256sum <"$4" | cut -d' ' -f1)" = "$3" ]; then return 0; fi |
| + curl -sfL --retry 3 -o "$4.part" "$2" |
| + got=$(sha256sum <"$4.part" | cut -d' ' -f1) |
| + [ "$got" = "$3" ] || { echo "$1 digest mismatch: want $3 got $got" >&2; rm -f "$4.part"; exit 1; } |
| + chmod 755 "$4.part" && mv "$4.part" "$4" |
| +} |
| +fetch minio "https://github.com/minio/minio/releases/download/$MINIO_RELEASE/minio.linux-arm64.$MINIO_RELEASE" $minio_sha $H/bin/minio |
| +fetch mc "https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.linux-arm64.$MC_RELEASE" $mc_sha $H/bin/mc |
| +echo "minio+mc: sha256 verified" |
| + |
| +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT |
| +put() { cat > "$T/f"; incus file push "$T/f" "infra$1" --mode "$2" -q; } |
| + |
| +if [ ! -f $H/minio-root.env ]; then |
| + umask 077 |
| + printf 'MINIO_ROOT_USER=harness-root\nMINIO_ROOT_PASSWORD=%s\n' "$(head -c 24 /dev/urandom | base64 | tr -d '/+=')" > $H/minio-root.env |
| + umask 022 |
| +fi |
| + |
| +if ! incus info infra >/dev/null 2>&1; then |
| + incus init images:debian/13/cloud infra -p box -q |
| + incus config set infra limits.memory=2GiB |
| + incus config device override infra eth0 ipv4.address=203.0.113.10 |
| + incus start infra |
| +fi |
| +for i in $(seq 1 60); do incus exec infra -- true 2>/dev/null && break; sleep 2; done |
| +incus exec infra -- cloud-init status --wait >/dev/null || true |
| +incus exec infra -- sh -c 'DEBIAN_FRONTEND=noninteractive apt-get update -q >/dev/null && apt-get install -qy dnsmasq >/dev/null' |
| + |
| +incus file push $H/bin/minio infra/usr/local/bin/minio --mode 0755 |
| +incus file push $H/bin/mc infra/usr/local/bin/mc --mode 0755 |
| +incus file push $H/minio-root.env infra/etc/default/minio-root --mode 0600 |
| + |
| +incus exec infra -- sh -c 'id minio >/dev/null 2>&1 || useradd --system --home /var/lib/minio --shell /usr/sbin/nologin minio; mkdir -p /var/lib/minio/data /var/log/harness; chown -R minio:minio /var/lib/minio /var/log/harness' |
| + |
| +put /etc/default/minio 0644 <<'EOF' |
| +MINIO_VOLUMES=/var/lib/minio/data |
| +MINIO_OPTS=--address 203.0.113.10:9000 |
| +MINIO_BROWSER=off |
| +MINIO_DOMAIN=s3.harness.test |
| +MINIO_AUDIT_WEBHOOK_ENABLE_harness=on |
| +MINIO_AUDIT_WEBHOOK_ENDPOINT_harness=http://127.0.0.1:9100/audit |
| +EOF |
| + |
| +# The audit sink: one JSON line per MinIO audit entry, loopback only. |
| +put /usr/local/bin/audit-sink 0755 <<'EOF' |
| +#!/usr/bin/env python3 |
| +# MinIO audit webhook -> /var/log/harness/minio-audit.jsonl, one entry per line. |
| +import json, http.server |
| +OUT = "/var/log/harness/minio-audit.jsonl" |
| +class H(http.server.BaseHTTPRequestHandler): |
| + def do_POST(self): |
| + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) |
| + entries = [] |
| + try: |
| + v = json.loads(body) |
| + entries = v if isinstance(v, list) else [v] |
| + except ValueError: |
| + for ln in body.splitlines(): |
| + if ln.strip(): |
| + entries.append(json.loads(ln)) |
| + with open(OUT, "a") as f: |
| + for e in entries: |
| + f.write(json.dumps(e, separators=(",", ":")) + "\n") |
| + self.send_response(200); self.end_headers() |
| + def log_message(self, *a): |
| + pass |
| +http.server.ThreadingHTTPServer(("127.0.0.1", 9100), H).serve_forever() |
| +EOF |
| + |
| +put /etc/systemd/system/harness-audit-sink.service 0644 <<'EOF' |
| +[Unit] |
| +Description=harness: MinIO audit webhook sink (127.0.0.1:9100) |
| +Before=minio.service |
| +[Service] |
| +User=minio |
| +ExecStart=/usr/bin/python3 /usr/local/bin/audit-sink |
| +Restart=always |
| +[Install] |
| +WantedBy=multi-user.target |
| +EOF |
| + |
| +put /etc/systemd/system/minio.service 0644 <<'EOF' |
| +[Unit] |
| +Description=harness: MinIO (203.0.113.10:9000) |
| +After=network-online.target harness-audit-sink.service |
| +Wants=network-online.target harness-audit-sink.service |
| +[Service] |
| +User=minio |
| +Group=minio |
| +EnvironmentFile=/etc/default/minio |
| +EnvironmentFile=/etc/default/minio-root |
| +ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES |
| +Restart=always |
| +LimitNOFILE=65536 |
| +[Install] |
| +WantedBy=multi-user.target |
| +EOF |
| +incus exec infra -- chown root:minio /etc/default/minio-root |
| +incus exec infra -- chmod 0640 /etc/default/minio-root |
| + |
| +# dnsmasq: the bridge's resolver for harness names; everything else goes to |
| +# incus's dnsmasq on 203.0.113.1. |
| +put /etc/dnsmasq.d/harness.conf 0644 <<'EOF' |
| +listen-address=203.0.113.10 |
| +bind-interfaces |
| +no-resolv |
| +server=203.0.113.1 |
| +domain-needed |
| +address=/s3.harness.test/203.0.113.10 |
| +address=/minio.harness.test/203.0.113.10 |
| +address=/infra.harness.test/203.0.113.10 |
| +EOF |
| +incus exec infra -- sh -c 'systemctl daemon-reload && systemctl enable --now harness-audit-sink minio >/dev/null 2>&1 && systemctl restart dnsmasq' |
| +incus network set hbr0 raw.dnsmasq "dhcp-option=option:dns-server,203.0.113.10" |
| + |
| +for i in $(seq 1 30); do curl -sf http://203.0.113.10:9000/minio/health/live && break; sleep 1; done |
| +. $H/minio-root.env |
| +export MC_CONFIG_DIR=$H/mc |
| +$H/bin/mc alias set harness http://203.0.113.10:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null |
| +for b in harness-deployment harness-data-ie harness-data-uk; do |
| + $H/bin/mc mb --ignore-existing harness/$b >/dev/null |
| + $H/bin/mc version enable harness/$b >/dev/null |
| +done |
| + |
| +# One MinIO user per principal (H2). Broad policy on purpose: IAM is checked |
| +# offline from the audit log, never by MinIO's policy engine. |
| +if [ ! -f $H/minio-users.json ]; then |
| + umask 077 |
| + python3 - > $H/minio-users.json <<'PY' |
| +import json, secrets |
| +users = ["leo", "sophie", "dennis", "signer", "fencewriter", "operator"] |
| +print(json.dumps({u: {"access_key": "harness-" + u, "secret_key": secrets.token_urlsafe(30)} for u in users}, indent=1)) |
| +PY |
| + umask 022 |
| +fi |
| +python3 -c 'import json; [print(v["access_key"], v["secret_key"]) for v in json.load(open("/root/harness/minio-users.json")).values()]' | |
| +while read ak sk; do |
| + $H/bin/mc admin user add harness "$ak" "$sk" >/dev/null |
| + $H/bin/mc admin policy attach harness readwrite --user "$ak" >/dev/null 2>&1 || true |
| +done |
| + |
| +echo "== infra" |
| +incus exec infra -- systemctl is-active minio harness-audit-sink dnsmasq |
| +incus exec infra -- ss -ltnp | awk 'NR==1 || /9000|9100|:53 /' |
| +$H/bin/mc ls harness |
| +for b in harness-deployment harness-data-ie harness-data-uk; do $H/bin/mc version info harness/$b; done |
| +$H/bin/mc admin user list harness |
| +echo "infra-setup: done" |