harness: Run 10 B setup (defect 6 PASS), B1 PASS ~20 s offline, B3 abort PASS, B4 stale-report PASS (flip first try after 3 min; tombstone never moved by the 15-s refresh), finding R10-1
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
14 files changed,
+1552
−0
harness/RUNLOG.md+131 −0harness/evidence/run10/b-precheck.log+58 −0harness/evidence/run10/b-setup.log+74 −0harness/evidence/run10/b-setup2.log+6 −0harness/evidence/run10/b-setup3.log+51 −0harness/evidence/run10/b1-move.log+334 −0harness/evidence/run10/b2-grow.log+0 −0harness/evidence/run10/b3-abort.log+198 −0harness/evidence/run10/b4-stale.log+374 −0harness/scenarios/cohort1/blocks/r10-grow.host.sh+62 −0harness/scenarios/cohort1/blocks/r10-move-precheck.host.sh+43 −0harness/scenarios/cohort1/blocks/r10-move-setup.host.sh+92 −0harness/scenarios/cohort1/blocks/r10-move.host.sh+111 −0harness/scenarios/cohort1/blocks/r10-report.host.sh+18 −0
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 301.444s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.802s
ok github.com/carlosframework/platform/cmd/moveprobe 93.183s
ok github.com/carlosframework/platform/cmd/zddprobe 0.700s
? github.com/carlosframework/platform/harness/apps/wakecheck [no test files]
ok github.com/carlosframework/platform/internal/activator 251.090s
ok github.com/carlosframework/platform/internal/agentsock 0.018s
ok github.com/carlosframework/platform/internal/alerting 0.003s
ok github.com/carlosframework/platform/internal/analytics 0.023s
ok github.com/carlosframework/platform/internal/appconfig 2.507s
ok github.com/carlosframework/platform/internal/apperr 0.074s
ok github.com/carlosframework/platform/internal/applog 0.141s
ok github.com/carlosframework/platform/internal/awsinv 0.003s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.845s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.696s
ok github.com/carlosframework/platform/internal/changelog 0.311s
ok github.com/carlosframework/platform/internal/claimactivation 0.768s
ok github.com/carlosframework/platform/internal/clamd 0.055s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 402.946s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.223s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.004s
ok github.com/carlosframework/platform/internal/dnsrec 0.002s
ok github.com/carlosframework/platform/internal/docsite 0.005s
ok github.com/carlosframework/platform/internal/domains 2.375s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 44.775s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.134s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.014s
ok github.com/carlosframework/platform/internal/geo 0.020s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.157s
ok github.com/carlosframework/platform/internal/host 180.202s
ok github.com/carlosframework/platform/internal/hostactors 6.433s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.013s
ok github.com/carlosframework/platform/internal/hostbox 2.745s
ok github.com/carlosframework/platform/internal/hostclaim 0.540s
ok github.com/carlosframework/platform/internal/hostenv 0.012s
ok github.com/carlosframework/platform/internal/hostevidence 0.012s
ok github.com/carlosframework/platform/internal/hostexec 4.735s
ok github.com/carlosframework/platform/internal/hostfence 0.037s
ok github.com/carlosframework/platform/internal/hostfiles 10.349s
ok github.com/carlosframework/platform/internal/hostfloor 0.014s
ok github.com/carlosframework/platform/internal/hostguard 1.033s
ok github.com/carlosframework/platform/internal/hostops 1.683s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.011s
ok github.com/carlosframework/platform/internal/hostquarantine 0.011s
ok github.com/carlosframework/platform/internal/hostrelocate 0.468s
ok github.com/carlosframework/platform/internal/hostresolve 0.145s
ok github.com/carlosframework/platform/internal/hostwrite 0.011s
ok github.com/carlosframework/platform/internal/infraview 0.015s
ok github.com/carlosframework/platform/internal/instancectl 5.217s
ok github.com/carlosframework/platform/internal/instanceexec 2.706s
ok github.com/carlosframework/platform/internal/instancefs 0.015s
ok github.com/carlosframework/platform/internal/instancekey 0.002s
ok github.com/carlosframework/platform/internal/instanceupdates 0.010s
ok github.com/carlosframework/platform/internal/layout 1.555s
ok github.com/carlosframework/platform/internal/lbhealth 0.002s
ok github.com/carlosframework/platform/internal/lease 9.790s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.005s
ok github.com/carlosframework/platform/internal/ledger 0.012s
ok github.com/carlosframework/platform/internal/mail 0.171s
ok github.com/carlosframework/platform/internal/maintenance 0.084s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.507s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.073s
ok github.com/carlosframework/platform/internal/metrics 0.004s
ok github.com/carlosframework/platform/internal/moves 21.113s
ok github.com/carlosframework/platform/internal/moves/archive 0.779s
ok github.com/carlosframework/platform/internal/objstore 0.168s
ok github.com/carlosframework/platform/internal/ops 0.173s
ok github.com/carlosframework/platform/internal/park 0.196s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
2026/10/20 21:57:01 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/20 21:57:01 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/20 21:57:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:01 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 21:57:01 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 21:57:01 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/20 21:57:01 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 21:57:01 provision: an app's records could not be listed: they are skipped
2026/10/20 21:57:01 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:01 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 21:57:01 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 21:57:01 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 21:57:01 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 21:57:01 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/20 21:57:01 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:01 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 21:57:02 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 21:57:02 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 21:57:02 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 21:57:02 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 21:57:02 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 21:57:02 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 21:57:02 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 21:57:02 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 21:57:02 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:02 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:02 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:02 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:02 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 21:57:02 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 21:57:02 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/20 21:57:02 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/20 21:57:02 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:02 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:02 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (4.67s)
plan14_root_reads_test.go:940: root read census: 142 discrepancies
changed root read internal/hostfence/writer.go:OpenRootFenceStore; review class, task, sites and digest
changed root source envelope harness
changed root source envelope harness/apps/wakecheck
changed root source envelope harness/boxes
changed root source envelope harness/boxes/shims
changed root source envelope harness/checks
changed root source envelope harness/console
changed root source envelope harness/infra
changed root source envelope harness/infra-copies/747f699/dev
changed root source envelope harness/infra-copies/a0b3f5a/dev
changed root source envelope harness/infra-copies/a0b3f5a/scripts
changed root source envelope harness/infra-copies/pr47
changed root source envelope harness/operator
changed root source envelope harness/release
changed root source envelope harness/scenarios
changed root source envelope harness/scenarios/cohort1
changed root source envelope harness/scenarios/cohort1/blocks
changed root source envelope harness/stand-ins
changed root source envelope internal/host
changed root source envelope internal/hostfence
changed root source envelope internal/publicrelease
changed root source envelope internal/s3compat
unclassified root read harness/apps/wakecheck/main.go:main file:Exec@333,file:Exec@804,file:Open@277,file:QueryRow@870
unclassified root read harness/boxes/boxes-up.sh:external file:external@0
unclassified root read harness/boxes/dmi-shim.sh:external file:external@0
unclassified root read harness/boxes/layout-probe.sh:external file:external@0
unclassified root read harness/boxes/leo-ext4-from.sh:external file:external@0
unclassified root read harness/boxes/leo-ext4.sh:external file:external@0
unclassified root read harness/boxes/leo-swap.sh:external file:external@0
unclassified root read harness/boxes/shims/systemd-zram-setup@.service:external file:external@0
unclassified root read harness/boxes/vendor-shims.sh:external file:external@0
unclassified root read harness/checks/check2.sh:external file:external@0
unclassified root read harness/checks/check5.sh:external file:external@0
unclassified root read harness/checks/check6.sh:external file:external@0
unclassified root read harness/checks/check6b.sh:external file:external@0
unclassified root read harness/checks/check7.sh:external file:external@0
unclassified root read harness/checks/check8.sh:external file:external@0
unclassified root read harness/checks/fix5.sh:external file:external@0
unclassified root read harness/checks/run6b.sh:external file:external@0
unclassified root read harness/checks/snap-base.sh:external file:external@0
unclassified root read harness/checks/snap-boot.sh:external file:external@0
unclassified root read harness/checks/ssmsmoke.sh:external file:external@0
unclassified root read harness/console/console-ext4.sh:external file:external@0
unclassified root read harness/console/console-setup.sh:external file:external@0
unclassified root read harness/console/signin.sh:external file:external@0
unclassified root read harness/console/system-update.sh:external file:external@0
unclassified root read harness/donor-user-data.sh:external file:external@0
unclassified root read harness/evidence.sh:external file:external@0
unclassified root read harness/host-setup.sh:external file:external@0
unclassified root read harness/host-user-data.sh:external file:external@0
unclassified root read harness/hrun.sh:external file:external@0
unclassified root read harness/infra-copies/747f699/dev/host-init.sh:external file:external@0
unclassified root read harness/infra-copies/a0b3f5a/dev/host-init.sh:external file:external@0
unclassified root read harness/infra-copies/a0b3f5a/scripts/bootstrap-box.sh:external file:external@0
unclassified root read harness/infra-copies/a0b3f5a/scripts/ship-app.sh:external file:external@0
unclassified root read harness/infra-copies/a0b3f5a/scripts/update-platform-binary.sh:external file:external@0
unclassified root read harness/infra-copies/pr47/probe.sh:external file:external@0
unclassified root read harness/infra-copies/pr47/remediation.sh:external file:external@0
unclassified root read harness/infra/infra-setup.sh:external file:external@0
unclassified root read harness/operator/operator-setup.sh:external file:external@0
unclassified root read harness/release/release-setup.sh:external file:external@0
unclassified root read harness/release/release-window.sh:external file:external@0
unclassified root read harness/release/step7-operator.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/50-publish-record.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/51-open-signer.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/51-roster-resume.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/51-roster.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/61-sign.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/61a-install-channel.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/62-guard-arm.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/63-run.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/63-sophie.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7-console-report.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7-evidence-key.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7-run.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7-runtime.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/74-run.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7a-leo.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/7a-run.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/acceptance.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b1.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b2-c8.leo.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b2-list.leo.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b2.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b4-after.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b4-pair.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b4-resume.sophie.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b4.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/b5.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/g4-roll.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/g5.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/g9-check.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/p9.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/place-parent.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/pw-seed.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-a1-normal.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-console-upgrade.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-grow.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-move-precheck.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-move-setup.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-move.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-prep.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r10-report.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-console-e1.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-console.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-devcheck.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-move-precheck.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-move-setup.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-move.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-normal.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-rearm.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-release.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/r9-setup.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm-roll.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm-strict.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm-strict.leo.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm-strict2.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm2.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/rearm3.host.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/reports.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/run-leo.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w-env.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w-identity.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w-open-signer.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w-sums.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w81.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w81b.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w82-f4.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w82-f5.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w82.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w82b.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w84.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/blocks/w9.box.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/box-helpers.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/signer-helpers.sh:external file:external@0
unclassified root read harness/scenarios/cohort1/ssm.sh:external file:external@0
unclassified root read harness/scenarios/sync.sh:external file:external@0
unclassified root read harness/snapshot.sh:external file:external@0
unclassified root read harness/stand-ins/certs.sh:external file:external@0
unclassified root read internal/s3compat/harness.go:readHarnessFile file:ReadFile@189,file:Stat@55
changed reclassified declaration internal/hostfence/writer.go:OpenRootFenceStore
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 21:57:14 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/20 21:57:14 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/20 21:57:14 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/20 21:57:14 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io hibernate false -> true
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged acme.go.tito.io status-named false -> true
2026/10/20 21:57:14 provision: converged acme.go.tito.io status-named true -> false
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 21:57:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 21:57:14 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 21:57:14 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 21:57:14 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/20 21:57:14 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/20 21:57:14 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 21:57:14 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 21:57:14 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 21:57:14 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:14 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 21:57:14 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/20 21:57:14 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/20 21:57:14 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:14 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 21:57:14 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:14 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/20 21:57:14 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 21:57:15 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:15 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 21:57:15 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 21:57:15 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 21:57:15 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/20 21:57:15 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/20 21:57:15 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 21:57:15 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/20 21:57:15 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/20 21:57:15 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/20 21:57:15 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/20 21:57:15 provision: dry-run: would converge owned.go.tito.io seed
2026/10/20 21:57:15 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/20 21:57:15 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/20 21:57:15 provision: converged owned.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/20 21:57:15 provision: converged owned.go.tito.io hibernate false -> true
2026/10/20 21:57:15 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/20 21:57:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: removed retired static route site.go.tito.io (record gone)
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/20 21:57:15 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/20 21:57:15 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:15 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 21:57:15 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 21:57:15 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 21:57:15 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 21:57:15 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/20 21:57:15 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 21:57:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 21:57:15 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 21:57:15 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 21:57:15 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/20 21:57:15 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/20 21:57:15 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 21:57:15 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/20 21:57:15 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/20 21:57:15 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 21:57:15 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/20 21:57:15 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 21:57:15 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:15 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/20 21:57:16 provision: a.shop.example.com: swapped for import sd_ae5jf56xzuvi2avvfe7njklkhi (the original route is journaled)
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_ae5jf56xzuvi2avvfe7njklkhi committed: its gate is cleared and its journal row dropped
2026/10/20 21:57:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_u6bqt26p23ly4422d4nrrv2aji rolled back to its original route
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_u6bqt26p23ly4422d4nrrv2aji: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_u6bqt26p23ly4422d4nrrv2aji is cleaned up and its journal row retired
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_rf5dujc4coso3zzwwrawqxsbfy rolled back to its original route
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_rf5dujc4coso3zzwwrawqxsbfy: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_rf5dujc4coso3zzwwrawqxsbfy is cleaned up and its journal row retired
2026/10/20 21:57:16 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_kfwu7zxcoqrk56prtib2mgbavu
2026/10/20 21:57:16 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/20 21:57:16 provision: dry-run: would converge a.shop.example.com seed source
2026/10/20 21:57:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:16 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/20 21:57:16 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/20 21:57:16 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_2i5fvbv26ecmhjjjplwxcv442i cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_zmcdkqv6w46jsbf6ujv476frxq cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_tjusihynzjtya6jifcdes6exvm is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_ikl4e4rrzbdxfd2biavw5okdsa is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_d5mvtg4pbavwzijitki763tjji is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 21:57:16 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/20 21:57:16 provision: b.shop.example.com: import sd_kuzfk2syjos3hvrpkvvkjmqapy left alone this pass: the import journal could not be read
2026/10/20 21:57:16 provision: c.shop.example.com: import sd_wf6balqlqk7ymygsgrbxlkzcs4 left alone this pass: the import journal could not be read
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_kydp3mlhxyy67v7obhnxutth6m committed: its gate is cleared and its journal row dropped
2026/10/20 21:57:16 provision: a.shop.example.com: not converging import sd_fvzu5zthp2sz2oh2d4facem5vm: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 21:57:16 provision: a.shop.example.com: not converging import sd_m5l46opetsxzcqfatl332flvhy: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 21:57:16 provision: a.shop.example.com: not converging import sd_7y5ypxrb6h4t7kimyy6ytppcau: it is placed but not committed, and the row on this box has no import gate
2026/10/20 21:57:16 provision: a.shop.example.com: not converging import sd_ylmbbtiiwad3jyou4lr4ltcrq4: it is placed but not committed, and the row on this box has no import gate
2026/10/20 21:57:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:16 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/20 21:57:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:16 provision: a.shop.example.com: import sd_mlxmsxhkmbc5ffsqtazeluvlmi committed: its gate is cleared and its journal row dropped
2026/10/20 21:57:16 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:16 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:16 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:17 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/20 21:57:17 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_rk3qfrfzeflifbdzhwu2f3xrta: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_rk3qfrfzeflifbdzhwu2f3xrta rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/20 21:57:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:17 provision: removed instance route acme.go.tito.io (its seed sd_2254v274ttsu3xyradcbuy52rm was aborted); the next pass rolls it back
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_2254v274ttsu3xyradcbuy52rm: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_2254v274ttsu3xyradcbuy52rm rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_6rpkbdict6ftkfn4nb4wvlw7za has a row this pass may not remove; leaving the seed aborted
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_722grpaxbjn2eqsb2xzsqmhms4 rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:17 provision: removed instance route acme.go.tito.io (its seed sd_vbttk5rcfc73wqkfz74nrqixfa was aborted); the next pass rolls it back
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_vbttk5rcfc73wqkfz74nrqixfa has no route on this box: its local data is removed
2026/10/20 21:57:17 provision: acme.go.tito.io: aborted seed sd_vbttk5rcfc73wqkfz74nrqixfa rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_cv54dyvp4uqlxmxpbpy4em3owi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_cv54dyvp4uqlxmxpbpy4em3owi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_cv54dyvp4uqlxmxpbpy4em3owi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_nyqokjuyijy6pbkq6jla2l5lpi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_nyqokjuyijy6pbkq6jla2l5lpi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_nyqokjuyijy6pbkq6jla2l5lpi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_w5y2j3fmin7vgmjgwmh7sqrjxy (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_w5y2j3fmin7vgmjgwmh7sqrjxy rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_w5y2j3fmin7vgmjgwmh7sqrjxy: the host's replica is left alone: its record no longer names this seed
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_w5y2j3fmin7vgmjgwmh7sqrjxy is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_55aljd6fm5vqolpjrhq4tgkmwi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_55aljd6fm5vqolpjrhq4tgkmwi rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_55aljd6fm5vqolpjrhq4tgkmwi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_55aljd6fm5vqolpjrhq4tgkmwi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_dtlvyxskpkwnuq56srit4ow3ci (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_dtlvyxskpkwnuq56srit4ow3ci rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_ld6uy6kgaw2lob2pn2gitsowp4 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ld6uy6kgaw2lob2pn2gitsowp4 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_vlsxbl363ympzdtqdt4h7zqypm (the original route is journaled)
2026/10/20 21:57:17 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_vlsxbl363ympzdtqdt4h7zqypm left alone this pass: its record could not be read
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_5l2hc2ohoslgueghrkkuz2ay5y (the original route is journaled)
2026/10/20 21:57:17 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_5l2hc2ohoslgueghrkkuz2ay5y left alone this pass: its record could not be read
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_bobiptgwakf35dlsoqf5o3tmda (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_bobiptgwakf35dlsoqf5o3tmda: close the import gate: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_bobiptgwakf35dlsoqf5o3tmda rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_bobiptgwakf35dlsoqf5o3tmda is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_r555ydf3rocjhozqfaoiegchtm (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_r555ydf3rocjhozqfaoiegchtm: close the import gate: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_r555ydf3rocjhozqfaoiegchtm rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_r555ydf3rocjhozqfaoiegchtm is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_hbcoea77b7qyfs7eg7u2463doy (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_hbcoea77b7qyfs7eg7u2463doy rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_hbcoea77b7qyfs7eg7u2463doy rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_hbcoea77b7qyfs7eg7u2463doy is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_o4qsf7l7iry2kry2ktq2e7ybxi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_o4qsf7l7iry2kry2ktq2e7ybxi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_o4qsf7l7iry2kry2ktq2e7ybxi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_yt2guitmqfi7irh2a7tcijwrza (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_yt2guitmqfi7irh2a7tcijwrza: retire its instance key: injected failure at retire-key
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_yt2guitmqfi7irh2a7tcijwrza rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_yt2guitmqfi7irh2a7tcijwrza is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_vvy4sys53vadorqcr65ayv5mh4 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_vvy4sys53vadorqcr65ayv5mh4 rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_vvy4sys53vadorqcr65ayv5mh4 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_66e2mcadretxwps6hf6lst45fq (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_66e2mcadretxwps6hf6lst45fq: remove its local data: injected failure at remove-local
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_66e2mcadretxwps6hf6lst45fq rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_66e2mcadretxwps6hf6lst45fq is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_pasxk476ufkzh3tmghlziirbs4 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_pasxk476ufkzh3tmghlziirbs4 rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_pasxk476ufkzh3tmghlziirbs4 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_e26ahexbbaazefguphvki6h3xe (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_e26ahexbbaazefguphvki6h3xe: remove its local data: injected failure at remove-local-done
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_e26ahexbbaazefguphvki6h3xe rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_e26ahexbbaazefguphvki6h3xe is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_6vvliqlcj2fwv7pagxauhdrvxu (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_6vvliqlcj2fwv7pagxauhdrvxu rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_6vvliqlcj2fwv7pagxauhdrvxu is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_cgfgy37bezpso7kjdjsq3qrhde (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_cgfgy37bezpso7kjdjsq3qrhde: restore the original route: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_cgfgy37bezpso7kjdjsq3qrhde rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_cgfgy37bezpso7kjdjsq3qrhde is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_brkodx4fmafrqrad3nxlwj45li (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_brkodx4fmafrqrad3nxlwj45li: restore the original route: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_brkodx4fmafrqrad3nxlwj45li rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_brkodx4fmafrqrad3nxlwj45li is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_gvzrowy6unbheuxeslmzxot74m (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gvzrowy6unbheuxeslmzxot74m is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_gas2rpaenvhvnkcuwa37zr75pq (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gas2rpaenvhvnkcuwa37zr75pq rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gas2rpaenvhvnkcuwa37zr75pq: delete its snapshot: injected failure at delete-snapshot
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gas2rpaenvhvnkcuwa37zr75pq is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_shhmmd2uea5y3rrmyi6l6tcugi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_shhmmd2uea5y3rrmyi6l6tcugi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_shhmmd2uea5y3rrmyi6l6tcugi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_k6se7ppgsioqzti3l7ztwb2h2m (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_k6se7ppgsioqzti3l7ztwb2h2m rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_k6se7ppgsioqzti3l7ztwb2h2m: delete the host's replica: injected failure at delete-replica
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_k6se7ppgsioqzti3l7ztwb2h2m is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_slayldjqnzjfwe5da3gbwmpbiq (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_slayldjqnzjfwe5da3gbwmpbiq rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_slayldjqnzjfwe5da3gbwmpbiq is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_b2o7jeo22uuh5o4bp2cpd7tasa (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_b2o7jeo22uuh5o4bp2cpd7tasa rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_b2o7jeo22uuh5o4bp2cpd7tasa: retire its journal row: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_b2o7jeo22uuh5o4bp2cpd7tasa is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_ypmtcf5larnaaba3u775esmdoi (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ypmtcf5larnaaba3u775esmdoi rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ypmtcf5larnaaba3u775esmdoi: retire its journal row: constraint failed: injected (1811)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ypmtcf5larnaaba3u775esmdoi is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_ese4bwpjiv5lkz5xuyemcfadia (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ese4bwpjiv5lkz5xuyemcfadia rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ese4bwpjiv5lkz5xuyemcfadia is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: mark import sd_ese4bwpjiv5lkz5xuyemcfadia rolled back: injected failure at cas
2026/10/20 21:57:17 provision: a.shop.example.com: aborted seed sd_ese4bwpjiv5lkz5xuyemcfadia rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_gaw7n4hnqlm6wsmc7st3cnxqle (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gaw7n4hnqlm6wsmc7st3cnxqle rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_gaw7n4hnqlm6wsmc7st3cnxqle is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: aborted seed sd_gaw7n4hnqlm6wsmc7st3cnxqle rolled back (no box had swapped it)
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_ykw2xyql57s24ny3zhvrdovtta (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ykw2xyql57s24ny3zhvrdovtta rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ykw2xyql57s24ny3zhvrdovtta rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ykw2xyql57s24ny3zhvrdovtta rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_ykw2xyql57s24ny3zhvrdovtta is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_qtbo6gwlxbndcvuokkhoggqzb4 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_qtbo6gwlxbndcvuokkhoggqzb4 rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_qtbo6gwlxbndcvuokkhoggqzb4 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_iymt3mgpspsegaebbiigqaampe (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_iymt3mgpspsegaebbiigqaampe rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_iymt3mgpspsegaebbiigqaampe is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_rv6zfhsp2pklcbysuakljk3rq4 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_rv6zfhsp2pklcbysuakljk3rq4 rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_rv6zfhsp2pklcbysuakljk3rq4 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_5kqt75p6jwgmus2njofpw5swky (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_5kqt75p6jwgmus2njofpw5swky rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_5kqt75p6jwgmus2njofpw5swky is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_fm242rajmvl5frcmszt5umybba (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_fm242rajmvl5frcmszt5umybba rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_fm242rajmvl5frcmszt5umybba is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_xd2gyguuyt4pmt66djcdrfipgm (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_xd2gyguuyt4pmt66djcdrfipgm rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_xd2gyguuyt4pmt66djcdrfipgm is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 21:57:17 provision: a.shop.example.com: swapped for import sd_serqtvmzt2jtvr6paattwtbv44 (the original route is journaled)
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_serqtvmzt2jtvr6paattwtbv44 rolled back to its original route
2026/10/20 21:57:17 provision: a.shop.example.com: import sd_serqtvmzt2jtvr6paattwtbv44 is cleaned up and its journal row retired
2026/10/20 21:57:17 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_umfbeif65asqtpciwmiuvizcku (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_umfbeif65asqtpciwmiuvizcku has no route on this box: its local data is removed and no route is put back
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_umfbeif65asqtpciwmiuvizcku is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_im6wbksc6ulcvbzvv3r4frelaa (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_q7rwqsexav3tybgs6sfb3doyl4 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_hkmuaycmsd3xbwaqh3zlnugkhu (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_hkmuaycmsd3xbwaqh3zlnugkhu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_3tksdl2htk4paemaoqwsa7mm5i (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_3tksdl2htk4paemaoqwsa7mm5i is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/20 21:57:18 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/20 21:57:18 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_55m2rowzozkhcqpikng4z3uwja (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_4o43awjbmhxpwahdr4ygcoxc7a (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_rljphe23mogmcmdyrknrkz3qti (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_b6zp67565v4regnruzho3tk64q (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_mwgtrjaahuqljqufsjvqayq74u (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_mwgtrjaahuqljqufsjvqayq74u rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_as7oauvsmm4oupzhlewl4pao7y (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_as7oauvsmm4oupzhlewl4pao7y has no route on this box: its local data is removed and no route is put back
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_as7oauvsmm4oupzhlewl4pao7y is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_w6sp3bnrjopa7ybutwm57rrht4 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_w6sp3bnrjopa7ybutwm57rrht4: remove its local data: injected failure at remove-local
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_w6sp3bnrjopa7ybutwm57rrht4 is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_nnfthpsspimrckon2pna2k7j74 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_nnfthpsspimrckon2pna2k7j74: a route appeared at the host, so its local database is left and the rollback waits
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_lkh3mlmxeu4yfut5s3hceg2rja (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_lkh3mlmxeu4yfut5s3hceg2rja rollback deferred: no fence generation: registry busy
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_zdes2w5vyzqa5cwp3fykh43nc4 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_zdes2w5vyzqa5cwp3fykh43nc4 rollback deferred: no fence generation: the generation source answered 0
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_dvyh6xcwsuoaa4zmo72kdobani (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_z5yyecgogkays7akj22gm7xjyi (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_yrgmf3f72zw7ymo7m4alyg5qqi (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/20 21:57:18 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_ozphvb7mz36jw3o6n3d5lhgfy4 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_ozphvb7mz36jw3o6n3d5lhgfy4 rolled back to its original route
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_ozphvb7mz36jw3o6n3d5lhgfy4 is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_3umpeneln2uwzcezkeyrpb54gu (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_3umpeneln2uwzcezkeyrpb54gu rolled back to its original route
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_3umpeneln2uwzcezkeyrpb54gu is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_xlxcd3sygh3hmky7ikirlsy5c4 (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_xlxcd3sygh3hmky7ikirlsy5c4 rolled back to its original route
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_xlxcd3sygh3hmky7ikirlsy5c4 is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_nhfpcjuwhgvqoz3adoi2auzeie (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_nhfpcjuwhgvqoz3adoi2auzeie: retire its instance key: the env file is a symlink
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_nhfpcjuwhgvqoz3adoi2auzeie rolled back to its original route
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_nhfpcjuwhgvqoz3adoi2auzeie is cleaned up and its journal row retired
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_pbv5zxwts435j5fkg24zeiz2yy (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_af3awdkuhoem7zupwu5j5bhdwa (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_vwp5jk5mpy56lhlllfd73v3g7i refused: there is no route at a.shop.example.com to replace
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_zyngtacujd3ltwqkdlrddrhzee refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_5afcjg3ntuxvphryijp3ooznry refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_txyxqtmsdoknx4dwsbnwybzyxu refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_ktsvqe6fwb53bephhsjqoyewkm refused: the route belongs to app other
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_53xd756miqf7gs6pdmynjzr52e refused: the route is filed under another account
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_uevfq7ld5tz2md5ulas44blqwq refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_zawipeio3ioqupdkm7kzhjqkne refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_2ngqhabq5tbmvt2fbshcwopf6u refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_v7txl73c5mqdj5x4ofd2qp73pu refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_mdup2sm2w7fpnyyqpt5zwwh3we refused: the route is not an address route this import can take over (socket)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_fhxupsuabz6obds6eadaqr37oe refused: the route is not an address route this import can take over (no address)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_w6aeh6ikrzwlw3eiwyjlm4y2fu refused: the route is not an address route this import can take over (provisioned)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_dpgd2yjxqjdditvajtpp34bg74 refused: the route is not an address route this import can take over (kind static)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_syw7g5ia4u7sghst2s35cnjh6y refused: the route is not an address route this import can take over (backing)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_grqbdvz7jrgg52gv6pw2o667om refused: the route is not an address route this import can take over (tunnel)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_6zbwa3jnm7a6gb3cuhy7nt34de refused: the route is not an address route this import can take over (caps)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_euuxxmm4taqjblqpmwoh6tp4xa refused: the route is not an address route this import can take over (platform)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_4bzf4p74yjhvc7niueiw2x46ty refused: the route is not an address route this import can take over (canary)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_e2f6jisdqocpddrnxfn5lla3u4 refused: the route is not an address route this import can take over (kind sidecar)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_rkodqtkwml3fzxxb7azl3afimi refused: the route is not an address route this import can take over (region)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_godmdju62jk4jz2enktbv243pe refused: the route is not an address route this import can take over (fleet_box)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_jx3yshnc5emfqlfelfvlu42ekq refused: the route is not an address route this import can take over (hibernate)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_a4icwztmk5j5uck5hmw2bkeege refused: the route is not an address route this import can take over (claim_owner)
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_v3cvear74sxg76scrcn45567ri refused: this box does not run instances nested
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_2uqk6e6d5n6lauvtdcwymerg5u refused: this box does not run instances nested
2026/10/20 21:57:18 provision: a.shop.example.com: import sd_hj2sb53wspacisltwmhy7xccom refused: claim control is on for this box
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_cgmt75ucnmw3k72bnyhrcixqny (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: swapped for import sd_4fohwsqqde43wfflrord2anu3m (the original route is journaled)
2026/10/20 21:57:18 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/20 21:57:18 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_euoof3tmeiuhdbeevlksllco54: its verify hash is not a sha256
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_6ukztxbz33rk5ow3zkszwv7bbu: its verify hash is not a sha256
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_a3ijigtuoqyh642cnczr5pe2ey: its verify hash is not a sha256
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/20 21:57:18 provision: A.shop.example.com/..: not swapping for import sd_fcgxtkaewotqkiullmnfjksztm: its host or seed id is malformed (seeds: not a host)
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_hpr6nqdwlhbxh6n6uw7jinxyny: its record does not describe a nested, hibernating instance
2026/10/20 21:57:18 provision: a.shop.example.com: not swapping for import sd_222a6rjydhozid77mx5r2uiyqy: its record does not describe a nested, hibernating instance
2026/10/20 21:57:19 provision: a.shop.example.com: not swapping for import sd_dx4ps6gjgmp2fk45oxmy7mwbbu: its record does not describe a nested, hibernating instance
2026/10/20 21:57:19 provision: a.shop.example.com: not swapping for import sd_7ynyl7ev7lrpw2xfkvhuyxndyy: its record does not describe a nested, hibernating instance
2026/10/20 21:57:19 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/20 21:57:19 provision: a.shop.example.com: swapped for import sd_gzv3laduzif4dvdhtp45pfn4b4 (the original route is journaled)
2026/10/20 21:57:19 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/20 21:57:19 provision: a.shop.example.com: swapped for import sd_j6hka2wqxn6bbsfggdfzai6u6i (the original route is journaled)
2026/10/20 21:57:19 provision: a.shop.example.com: swapped for import sd_i3sny5ijdeeydka2on2vzitdpy (the original route is journaled)
2026/10/20 21:57:19 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/20 21:57:19 provision: dry-run: would consider swapping a.shop.example.com for import sd_ne5vhomyitanv3kgyfzncyzcri
2026/10/20 21:57:19 provision: a.shop.example.com: import sd_52kos6qjhllg7a7dpj42xizd74 rolled back to its original route
2026/10/20 21:57:19 provision: a.shop.example.com: import sd_52kos6qjhllg7a7dpj42xizd74: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 21:57:19 provision: a.shop.example.com: import sd_52kos6qjhllg7a7dpj42xizd74 is cleaned up and its journal row retired
2026/10/20 21:57:19 provision: a.shop.example.com: import sd_25zszoof5gtcrloq2vhofvwfvu left alone: the import journal holds seed sd_4qr4mdsirz5h3d4txsjm2vnsr4 at swapped
2026/10/20 21:57:19 provision: a.shop.example.com: not minting import sd_77vghodns7nurtjgzoib2iioeq: it is swapped but not committed, and only its swap makes its row
2026/10/20 21:57:19 provision: a.shop.example.com: not minting import sd_ln6jcqwxndfgoxobywdfrxtcre: it is placed but not committed, and only its swap makes its row
2026/10/20 21:57:19 provision: a.shop.example.com: swapped for import sd_ojet2z2bfxf5ukk6oiyzj23mii (the original route is journaled)
2026/10/20 21:57:19 provision: a.shop.example.com: not minting import sd_ojet2z2bfxf5ukk6oiyzj23mii: it is swapped but not committed, and only its swap makes its row
2026/10/20 21:57:19 provision: a.shop.example.com: not minting import sd_ojet2z2bfxf5ukk6oiyzj23mii: it is placed but not committed, and only its swap makes its row
2026/10/20 21:57:19 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 21:57:19 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/20 21:57:19 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/20 21:57:19 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/20 21:57:19 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/20 21:57:19 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/20 21:57:19 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 21:57:19 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/20 21:57:19 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/20 21:57:19 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 21:57:19 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/20 21:57:19 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/20 21:57:19 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/20 21:57:19 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/20 21:57:19 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 21:57:19 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/20 21:57:19 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 21:57:19 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 21:57:19 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/20 21:57:19 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 21:57:19 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 21:57:19 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:19 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/20 21:57:19 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:19 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:19 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 21:57:19 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 21:57:19 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 21:57:19 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 21:57:19 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 21:57:19 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 21:57:19 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL github.com/carlosframework/platform/internal/provision 17.730s
ok github.com/carlosframework/platform/internal/publicrelease 0.008s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.409s
ok github.com/carlosframework/platform/internal/relay 7.215s
ok github.com/carlosframework/platform/internal/release 24.024s
ok github.com/carlosframework/platform/internal/rollout 3.635s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.006s
ok github.com/carlosframework/platform/internal/rootparticipation 0.057s
ok github.com/carlosframework/platform/internal/rootstate 0.026s
ok github.com/carlosframework/platform/internal/runs 0.574s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.485s
ok github.com/carlosframework/platform/internal/seeds 0.010s
ok github.com/carlosframework/platform/internal/sim/hostlife 8.492s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.063s
ok github.com/carlosframework/platform/internal/status 0.059s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.236s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.003s
ok github.com/carlosframework/platform/internal/termfence 0.003s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.002s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.004s
ok github.com/carlosframework/platform/internal/version 0.001s
ok github.com/carlosframework/platform/internal/vet 1.540s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/harness/RUNLOG.md b/harness/RUNLOG.md| index fbd61252f..e88f217ab 100644 |
| --- a/harness/RUNLOG.md |
| +++ b/harness/RUNLOG.md |
| @@ -2058,3 +2058,134 @@ candidate's `--upgrade --from <running sha>` is the supported road: R9-4's block |
| `install-console --identity identity.v0.28.6.json --source /opt/carlos/versions/<CV>/console ... --upgrade --from |
| <sha256 of the running console> --dry-run`, the same without `--dry-run`, `systemctl restart console.service`, |
| `console-report`. Not snapshotted (the logs carry it: `evidence/run10/a2-*.log`). |
| + |
| +## (B) The offline move on the candidate, from `post-candidate` |
| + |
| +Restored `post-candidate` (21:28-21:29Z). Every door below is the candidate CLI (harness v0.28.8) as the operator |
| +against sophie's console, `blocks/r10-move.host.sh <app> <instance> move|abort|stale`: the doors back to back with |
| +nothing between them but the door itself; a background sampler (every 5 s) records the source lease object |
| +`harness-data-ie instances/<host>/lock.json` (owner, generation, `expires_at`) and sophie's `/data/.instances/<host>/` |
| +(defect 8); a door that refuses with a wait (MV-O178 "not current", "Waiting for", exit 4/5) is retried every 5 s |
| +and every try is logged (MV-O178 visibility). |
| + |
| +### Setup: PASS, and defect 6 is fixed (21:29-21:33Z) |
| + |
| +`blocks/r10-move-setup.host.sh` (Run 9's setup for Run 10's apps): H17 secrets key (`sha256:6d7b3e95...`, the same |
| +key as Run 9, from the host's tmpfs), H13b test CA in each box's trust store, H13 certificates, H18 store binding as |
| +settings (MinIO bucket and user per app). Apps in `bab`, moveprobe @ 54ae7fb9 (`6d331a2f...`, `cmd/moveprobe` |
| +unchanged at e0986eab) as `mp1` on edge, secret, settings, `schedule set every-minute --every 1m --path /tick`: |
| +- **`shop`, two instances of ONE app**: `shop-small` (B1) and `shop-large` (B2), moved one at a time (the |
| + coordinator: flagship's real app holds several instances under one app). |
| +- `rollbackapp` (B3), `staleapp` (B4). |
| +The console's upload limit stopped the first run at staleapp's ship (`too many uploads - try again in 59s`); the |
| +block resumed with `RESUME=staleapp` (a harness pacing note, not a defect: four ships inside a minute). |
| + |
| +**Defect 6 check: PASS.** Every instance's first wake on sophie found `<host>.config.env` already in its new nested |
| +directory and answered `/ready` 200 at the first wake (2nd try, ~35 s from the first request: the reconcile pass, |
| +then the wake). Run 9's rollbackapp ran 12 min without it. |
| + |
| +Pre-checks (`blocks/r10-move-precheck.host.sh`, M9's commands, 21:36:42Z): PASS. All three boxes `carlos v0.28.8`, |
| +no `CARLOS_HOST_CLAIM` value, no `CARLOS_MOVES_PHASE1`, `User=carlos`, `Delegate=yes`, `carlos-stage.path` active; |
| +guard armed on leo and sophie; console exe `9887a9a4...`, `CARLOS_HOST_CLAIM=` empty; every edges file carries |
| +`offline-move-1` and the same `secrets_key_fp`; no `control/hosts/<host>.json`, no backfill; one cert each; all four |
| +hosts `nested` on ada-letterpress; staging lane listable from sophie's credentials; each nested dir 2.8-2.9 MB. |
| + |
| +### B1. shop-small, sophie -> dennis, doors back to back: PASS, offline about 20 s (TARGET under a minute: met) |
| + |
| +`r10-move.host.sh shop shop-small move 600` (a once-timer `once-1` armed through the app on sophie at 21:37:08Z, due |
| +21:47:08Z, after the move). |
| + |
| +| step | at | took | tries | result | |
| +|---|---|---|---|---| |
| +| start | 21:37:08.936Z | 4.2 s | 1 | `Every box in ie has stopped ...` exit 0 | |
| +| seal | 21:37:18.238Z | 4.1 s | 1 | `Copied and checked.` sophie: checkpointed (position 31, replica 31), `sealed at TXID 31 (33 staged objects)` | |
| +| flip | 21:37:22.385Z | 4.1 s | 1 | `Stopped in ie. Starting in uk. ...` exit 0 | |
| +| open | 21:37:26.521Z | 6.1 s | 1 | `... is served from uk.` exit 0 | |
| +| first request on dennis | 21:37:37.820Z | 1.5 s | | `/ready` 200 at 21:37:39.303Z | |
| +| finish | 21:37:40.048Z | 215.7 s | 1 | `Moved. ... It was offline for 18 seconds.` at 21:41:15.762Z (the CLI waits for sophie's cleanup) | |
| + |
| +**Offline time.** Measured start to the first 200 on dennis: 30.4 s, of which 10.2 s was the harness's own retry |
| +helper re-running two doors that had already succeeded (start and open print "Waiting for ..." as progress; the |
| +helper took that for a wait; fixed in the block before B3/B4). Net of that: **about 20 s** (doors 18.5 s, first |
| +request 1.5 s); the console's own figure is **17.6 s** (`offline_s 17.55`). **No door needed a retry (no MV-O178).** |
| + |
| +Checks: **PASS on every item.** |
| +- **Rows:** dennis `/report`: `bg` home 1-586 then uk 587-2413; `worker` home 1-293 then uk 294-1207; contiguous. |
| +- **Key digest:** `<host>.env` `e92a0004...` and instance key sha256 `87a15ceb...` on dennis = sophie's before; |
| + `platform_key_sha256 87a15ceb...`, `file_key_sha256 c50a3417...` unchanged; `instance.key` `7737438f...`. |
| +- **config.env BEFORE the first wake on dennis (defect 6): PASS.** `/data/shop-small.dev.harness.test.config.env` |
| + written 21:37:25.072Z (at flip), seven names, before the first request at 21:37:37.8Z. |
| +- **Files:** `instance.key` same digest. **launch.\* files:** none on dennis while the row is flat (`launch.run.json` |
| + is the nested layout's; Run 9 saw it appear at nesting). Dennis defers nesting (`stopped for the move in 56m24s |
| + unless it sleeps first`: M3 unchanged). |
| +- **Timers:** `every-minute` and `once-1` carried (dennis rows, same ids); ticks resumed on dennis (21:38:00Z |
| + onward); **`once-1` fired on dennis at its due time** (`once/uk n=1`, `fired once-1 due 1792532828 count 1`). |
| +- **Defect 8: PASS.** Sophie's `/data/.instances/shop-small.dev.harness.test/` became empty at open (sampler 21:37:35Z) |
| + and stayed empty through finish and after (21:52Z: `total 0`), although sophie's row stayed `moving` with both timer |
| + rows until finish. No `<host>.env` re-created. |
| +- **Source parked:** `/data/.offline-moves/shop-small.dev.harness.test/mv_vdogns62yqtqgb2ms62zd7p2wq/{sealer,source}`; |
| + sophie row and timers gone at finish (`removed retired instance route`, then `tombstone lifted` 21:41:15Z). |
| +- **Tombstone lifted at finish:** data-ie `lock.json` `{"owner":"moved:mv_vdogns...","expires_at":"1970-01-01T00:00:00Z"}`; |
| + data-uk lease `dennis:1359`; record `region uk`, `move null`, `last_move outcome done offline_s 17.55`. |
| +- **Tombstone expiry during the move:** see B4 (the cleaner trace). In B1 it was re-written to now+7 days at the seal |
| + (21:37:20.6Z), at flip (21:37:26.2Z), at open (21:37:31.7Z) and once after open (21:37:41.4Z), i.e. on each |
| + moves pass a door's serial bump started, then **unchanged for 3 min 34 s** (21:37:41Z to the lift at 21:41:15Z) |
| + while the 15-s report refresh ran. So the refresh never extends it; the ordinary pass does, once per pass, as |
| + `offlineSealerRefresh`'s comment says (`internal/host/movesoffline_seal.go`, e0986eab). |
| +- **Backhaul right after open: 404** through sophie's edge and leo's edge at 21:37:39Z (7 s after open), 200 at |
| + 21:41:57Z. Measured properly in B4 and B2 (finding R10-1 below). |
| + |
| +### B3. Abort after flip, on rollbackapp: PASS, back on sophie, nothing lost (21:42:23-21:42:52Z) |
| + |
| +| step | at | took | result | |
| +|---|---|---|---| |
| +| start | 21:42:23.645Z | 2.2 s | exit 0 | |
| +| seal | 21:42:25.816Z | 6.2 s | `Copied and checked.` | |
| +| flip | 21:42:31.975Z | 6.2 s | `Stopped in ie. Starting in uk.` | |
| +| abort | 21:42:38.142Z | 8.2 s | `Moving ... back to ie.` / `Every box has put ... back in ie.` exit 3 (M8); one needless helper retry (5.1 s; fixed after B3) | |
| +| finish | 21:42:51.403Z | 0.1 s | **`rollbackapp.dev.harness.test is back in ie and serving. Nothing was lost.`** exit 3 | |
| +| first request on sophie | 21:42:51.511Z | 0.6 s | `/ready` 200 at 21:42:52.063Z | |
| + |
| +Offline 28.4 s measured, **about 23 s net** of the helper's retry; console `offline_s 27.7`. After: sophie `/report` |
| +`bg` 1-1160 (count 1160, contiguous, past the 1067-1074 read before start), `worker` 1-581; `platform_key_sha256 |
| +e75e790b...`, `file_key_sha256 b9c9f5ce...`, `<host>.env` `c98f88b4...`, `instance.key` `1def5a96...` unchanged; a new |
| +boot and store object. Sophie row `ready` (nested, never purged); data-ie lease `sophie:1389` generation 2; no data-uk |
| +lease; dennis has no row and parked what it extracted under `.offline-moves/.../target`; record back in home, |
| +`last_move outcome aborted`. |
| + |
| +### B4. The deliberate stale-report case on staleapp: PASS, flip accepted first time after 3 min (21:43:53-21:52:28Z) |
| + |
| +| step | at | took | tries | result | |
| +|---|---|---|---|---| |
| +| start | 21:43:53.471Z | 2.2 s | 1 | exit 0 | |
| +| seal | 21:43:55.663Z | 8.1 s | 1 | sealed at TXID 64 (66 staged objects) | |
| +| **pause** | 21:44:03.813Z | 180 s | | `status` after it: sealed, both sources `stopped` | |
| +| **flip** | 21:47:03.893Z | 2.1 s | **1** | `Stopped in ie. Starting in uk. ...` exit 0: **no MV-O178, no "still runs a process"** | |
| +| open | 21:47:06.035Z | 4.2 s | 1 | exit 0 | |
| +| first request on dennis | 21:47:10.324Z | 3.1 s | | one 404 at 21:47:10.4Z, 200 at 21:47:13.437Z | |
| +| finish | 21:47:31.902Z | 296.3 s | 1 | `Moved. ... offline for 193 seconds.` | |
| + |
| +Offline 200.0 s (180 s of it the deliberate pause; console 192.6 s). Rows: `bg` home 1-1181 then uk 1182-1860, |
| +contiguous; keys and files unchanged; config.env on dennis at 21:47:04.640Z (flip), before the first request; sophie's |
| +`.instances/staleapp...` empty from open on (defect 8 PASS); sophie's timer row was pruned 4 s after finish |
| +(`schedule: pruned 1 timer row(s)` 21:52:30Z). |
| + |
| +**Tombstone expiry (the defect 7 fix round): PASS.** Sampler, data-ie `lock.json` `expires_at` (owner |
| +`moved:mv_rwidfnpau7xetmyhq4ooe3ogtu` throughout): |
| +``` |
| +21:44:00.8Z seal writes it -> 2026-10-27T21:44:00.83Z |
| +21:44:06.9Z the pass after the seal -> 2026-10-27T21:44:06.94Z |
| + (2 min 50 s of the pause with the 15-s refresh running: unchanged) |
| +21:46:56.6Z the timed moves pass -> 2026-10-27T21:46:56.64Z |
| +21:47:06.4Z flip's pass; 21:47:12.2Z open's pass; 21:47:32.9Z the pass after open |
| + (4 min 40 s to finish: unchanged) |
| +21:52:27Z lifted at finish -> 1970-01-01T00:00:00Z |
| +``` |
| +Never moved by the 15-s refresh; only an ordinary pass re-asserts it (once a pass, to now+7 days). |
| + |
| +**Finding R10-1 (behaviour, not a correctness defect): after open, the source box's edge answers 404 for the moved |
| +host until finish's cleanup is under way.** B4: through sophie's edge, 200 only 17.9 s after the first 200 on dennis |
| +(17 tries at 1 s); through leo's edge, 200 at once by then. B1: 404 through both at 7 s after open. Sophie keeps the |
| +row `moving` until finish removes it, and its edge serves 404 for a held row rather than backhauling it. On dev and |
| +flagship the record's DNS points where? If any client still reaches an IE edge for the host after open, it gets 404 |
| +for up to the time finish takes (4-5 min). Measured with answer codes in B2. |
diff --git a/harness/evidence/run10/b-precheck.log b/harness/evidence/run10/b-precheck.log| new file mode 100644 |
| index 000000000..e4f5a3c1c |
| --- /dev/null |
| +++ b/harness/evidence/run10/b-precheck.log |
| @@ -0,0 +1,58 @@ |
| +[hssm] i-06a1ec144aa5226fb b2d464bc-dbc4-438d-a77f-a5072347298f Success exit=0 |
| +== P0.2 switches, R3, delegation, release (21:36:42) |
| +leo: host.env CARLOS_HOST_CLAIM=0 CARLOS_MOVES_PHASE1=0 CARLOS_REGION= CARLOS_POOLS=cohort1 secrets_key=1 |
| + edge env: 0 claim/phase1 vars set |
| + User=carlos Delegate=yes ActiveState=active |
| + carlos-stage.path active / enabled |
| + carlos v0.28.8; guard policy: armed |
| +sophie: host.env CARLOS_HOST_CLAIM=0 CARLOS_MOVES_PHASE1=0 CARLOS_REGION= CARLOS_POOLS= secrets_key=1 |
| + edge env: 0 claim/phase1 vars set |
| + User=carlos Delegate=yes ActiveState=active |
| + carlos-stage.path active / enabled |
| + carlos v0.28.8; guard policy: armed |
| +dennis: host.env CARLOS_HOST_CLAIM=0 CARLOS_MOVES_PHASE1=0 CARLOS_REGION=uk CARLOS_POOLS= secrets_key=1 |
| + edge env: 0 claim/phase1 vars set |
| + User=carlos Delegate=yes ActiveState=active |
| + carlos-stage.path active / enabled |
| + carlos v0.28.8; guard policy: absent |
| +console: CARLOS_SECRETS_PUBKEY=<set> CARLOS_HOST_CLAIM= exe 9887a9a42b058708 |
| +ada-letterpress linux-arm64 canary v0.28.8 current |
| +kathleen-juggling linux-arm64 stable v0.28.8 current |
| +noor-marquetry linux-arm64 stable v0.28.8 current |
| +== P0.2 edges files: offline-move-1, secrets_key_fp, hosts |
| +2026-10-20 21:31:37 1104 ada-letterpress.json |
| +2026-10-20 21:31:01 549 kathleen-juggling.json |
| +2026-10-20 21:31:01 559 noor-marquetry.json |
| +{"box":"ada-letterpress","region":"eu-west-1","pools":null,"caps":["offline-move-1"],"secrets_key_fp":"sha256:6d7b3e95f3ce2e021cea11a430bca1084d9dcdc79daa8096e92604f28ee97fad","hibernation_configured":true,"serves_shared_pool":true,"hosts":["console.dev.harness.test","rollbackapp.dev.harness.test","shop-large.dev.harness.test","shop-small.dev.harness.test","staleapp.dev.harness.test","wakecheck.dev.harness.test"]} |
| +{"box":"kathleen-juggling","region":"eu-west-1","pools":null,"caps":["offline-move-1"],"secrets_key_fp":"sha256:6d7b3e95f3ce2e021cea11a430bca1084d9dcdc79daa8096e92604f28ee97fad","hibernation_configured":true,"serves_shared_pool":false,"hosts":[]} |
| +{"box":"noor-marquetry","region":"eu-west-1","pools":null,"caps":["offline-move-1"],"secrets_key_fp":"sha256:6d7b3e95f3ce2e021cea11a430bca1084d9dcdc79daa8096e92604f28ee97fad","hibernation_configured":true,"serves_shared_pool":true,"hosts":[]} |
| +== P0.3 claims and backfill absent, certificates, layout |
| +control/hosts/shop-small.dev.harness.test.json absent |
| +control/certs/shop-small: 1 |
| +control/hosts/shop-large.dev.harness.test.json absent |
| +control/certs/shop-large: 1 |
| +control/hosts/rollbackapp.dev.harness.test.json absent |
| +control/certs/rollbackapp: 1 |
| +control/hosts/staleapp.dev.harness.test.json absent |
| +control/certs/staleapp: 1 |
| +apps/bab/shop/instances/shop-large.dev.harness.test.json |
| +apps/bab/shop/instances/shop-small.dev.harness.test.json |
| +apps/bab/rollbackapp/instances/rollbackapp.dev.harness.test.json |
| +apps/bab/staleapp/instances/staleapp.dev.harness.test.json |
| +control/hosts-backfill.json absent |
| +BOX HOST LAYOUT REASON |
| +ada-letterpress (nested by default) rollbackapp.dev.harness.test nested — |
| +ada-letterpress (nested by default) shop-large.dev.harness.test nested — |
| +ada-letterpress (nested by default) shop-small.dev.harness.test nested — |
| +ada-letterpress (nested by default) staleapp.dev.harness.test nested — |
| +ada-letterpress (nested by default) wakecheck.dev.harness.test nested — |
| +leo: Oct 20 21:30:01 leo carlos[1379]: 2026/10/20 21:30:01 layout: no intent recorded -> nested by default (CARLOS_LAYOUT_DEFAULT unset or nested; record `carlos system layout revert` to opt out) |
| +sophie: Oct 20 21:30:01 sophie carlos[1389]: 2026/10/20 21:30:01 layout: no intent recorded -> nested by default (CARLOS_LAYOUT_DEFAULT unset or nested; record `carlos system layout revert` to opt out) |
| +dennis: Oct 20 21:30:01 dennis carlos[1359]: 2026/10/20 21:30:01 layout: no intent recorded -> nested by default (CARLOS_LAYOUT_DEFAULT unset or nested; record `carlos system layout revert` to opt out) |
| +== P0.4 staging lane: list harness-data-uk moves/ with sophie's box credentials (the IE box role) |
| +data-uk moves/ KeyCount: null |
| +== P0.6 sizes, sidecars, local copies |
| +sophie shop-small.dev.harness.test: 2.9M /data/instances/shop-small.dev.harness.test |
| +sophie shop-large.dev.harness.test: 2.9M /data/instances/shop-large.dev.harness.test |
| +sophie rollbackapp.dev.harness.test: 2.8M /data/instances/rollbackapp.dev.harness.test |
| +sophie staleapp.dev.harness.test: 2.8M /data/instances/staleapp.dev.harness.test |
diff --git a/harness/evidence/run10/b-setup.log b/harness/evidence/run10/b-setup.log| new file mode 100644 |
| index 000000000..e404814dd |
| --- /dev/null |
| +++ b/harness/evidence/run10/b-setup.log |
| @@ -0,0 +1,74 @@ |
| +[hssm] i-06a1ec144aa5226fb 85b4499a-9d22-4b6b-a4d2-8bea882cebac Success exit=0 |
| +sophie-console volume restored to post-candidate |
| +leo carlos v0.28.8 |
| +sophie carlos v0.28.8 |
| +dennis carlos v0.28.8 |
| +== H17 secrets key (21:29:56.295Z) |
| +== H13b test CA in each box's trust store |
| +done. |
| +done. |
| +done. |
| +leo edge active |
| +sophie edge active |
| +dennis edge active |
| +console active healthz 200 |
| +secrets pubkey fp: 6d7b3e95f3ce2e02 |
| +== H13 certificates for the new hosts |
| +published control/certs/shop-small.dev.harness.test (notAfter=Oct 23 21:30:01 2026 GMT) |
| +published control/certs/shop-large.dev.harness.test (notAfter=Oct 23 21:30:02 2026 GMT) |
| +published control/certs/rollbackapp.dev.harness.test (notAfter=Oct 23 21:30:03 2026 GMT) |
| +published control/certs/staleapp.dev.harness.test (notAfter=Oct 23 21:30:04 2026 GMT) |
| +== H18 MinIO stores (21:30:05.202Z) |
| +shop: bucket harness-store-shop user store-shop |
| +rollbackapp: bucket harness-store-rollbackapp user store-rollbackapp |
| +staleapp: bucket harness-store-staleapp user store-staleapp |
| +== apps (21:30:06.754Z) |
| +6d331a2fab4d59b9ed241cca18e99bdcf010315dae3cc573639d8caaea5f7cfe /home/operator/r10/moveprobe |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created app shop in account bab — it's yours |
| + open: http://127.0.0.1:8090/a/bab/apps/shop |
| + ship: carlos ship --account bab --app shop ... |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +shipped shop mp1 |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +promoted shop mp1 -> edge |
| +enabled instances for shop under dev.harness.test |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 3 secret(s) for shop (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 4 env var(s) for shop (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +recorded every-minute for bab/shop — every 1m → POST /tick; first fire at 2026-10-20T21:31:00Z (UTC); boxes pick it up within seconds — `carlos schedule ls --app shop` shows each instance's next run |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created instance shop-small.dev.harness.test (channel edge, Ireland · Dublin) |
| + the host comes live on the next reconcile pass on its box |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created instance shop-large.dev.harness.test (channel edge, Ireland · Dublin) |
| + the host comes live on the next reconcile pass on its box |
| +6d331a2fab4d59b9ed241cca18e99bdcf010315dae3cc573639d8caaea5f7cfe /home/operator/r10/moveprobe |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created app rollbackapp in account bab — it's yours |
| + open: http://127.0.0.1:8090/a/bab/apps/rollbackapp |
| + ship: carlos ship --account bab --app rollbackapp ... |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +shipped rollbackapp mp1 |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +promoted rollbackapp mp1 -> edge |
| +enabled instances for rollbackapp under dev.harness.test |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 3 secret(s) for rollbackapp (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 4 env var(s) for rollbackapp (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +recorded every-minute for bab/rollbackapp — every 1m → POST /tick; first fire at 2026-10-20T21:31:00Z (UTC); boxes pick it up within seconds — `carlos schedule ls --app rollbackapp` shows each instance's next run |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created instance rollbackapp.dev.harness.test (channel edge, Ireland · Dublin) |
| + the host comes live on the next reconcile pass on its box |
| +6d331a2fab4d59b9ed241cca18e99bdcf010315dae3cc573639d8caaea5f7cfe /home/operator/r10/moveprobe |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created app staleapp in account bab — it's yours |
| + open: http://127.0.0.1:8090/a/bab/apps/staleapp |
| + ship: carlos ship --account bab --app staleapp ... |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +2026/10/20 21:30:09 too many uploads — try again in 59s |
| +setup exit=1 |
diff --git a/harness/evidence/run10/b-setup2.log b/harness/evidence/run10/b-setup2.log| new file mode 100644 |
| index 000000000..3f0b052bb |
| --- /dev/null |
| +++ b/harness/evidence/run10/b-setup2.log |
| @@ -0,0 +1,6 @@ |
| +[hssm] i-06a1ec144aa5226fb 4e0d06f6-86a1-4e16-b2d7-23ba8f9905de Success exit=0 |
| +== apps (21:30:32.581Z) |
| +6d331a2fab4d59b9ed241cca18e99bdcf010315dae3cc573639d8caaea5f7cfe /home/operator/r10/moveprobe |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +2026/10/20 21:30:33 too many uploads — try again in 35s |
| +setup exit=1 |
diff --git a/harness/evidence/run10/b-setup3.log b/harness/evidence/run10/b-setup3.log| new file mode 100644 |
| index 000000000..434402d9c |
| --- /dev/null |
| +++ b/harness/evidence/run10/b-setup3.log |
| @@ -0,0 +1,51 @@ |
| +[hssm] i-06a1ec144aa5226fb cb828481-6c87-45f9-9227-6bd058c9ba6a Success exit=0 |
| +== apps (21:31:28.941Z) |
| +6d331a2fab4d59b9ed241cca18e99bdcf010315dae3cc573639d8caaea5f7cfe /home/operator/r10/moveprobe |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +shipped staleapp mp1 |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +promoted staleapp mp1 -> edge |
| +enabled instances for staleapp under dev.harness.test |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 3 secret(s) for staleapp (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +set 4 env var(s) for staleapp (default) |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +recorded every-minute for bab/staleapp — every 1m → POST /tick; first fire at 2026-10-20T21:32:00Z (UTC); boxes pick it up within seconds — `carlos schedule ls --app staleapp` shows each instance's next run |
| +using http://127.0.0.1:8090 as operator@keymail.dev |
| +created instance staleapp.dev.harness.test (channel edge, Ireland · Dublin) |
| + the host comes live on the next reconcile pass on its box |
| +== config.env before the first wake (defect 6) (21:31:30.813Z) |
| + shop-small: ls: cannot access '/data/instances/shop-small.dev.harness.test/': No such file or directory |
| + shop-large: ls: cannot access '/data/instances/shop-large.dev.harness.test/': No such file or directory |
| + rollbackapp: ls: cannot access '/data/instances/rollbackapp.dev.harness.test/': No such file or directory |
| + staleapp: ls: cannot access '/data/instances/staleapp.dev.harness.test/': No such file or directory |
| +== first wake on sophie only (21:32:11.187Z) |
| +shop-small /ready 200 after 2 tries, 35.6s |
| +launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env |
| +shop-large /ready 200 after 2 tries, 35.6s |
| +launch.run.json shop-large.dev.harness.test shop-large.dev.harness.test.config.env shop-large.dev.harness.test.db shop-large.dev.harness.test.db-shm shop-large.dev.harness.test.db-wal shop-large.dev.harness.test.env |
| +rollbackapp /ready 200 after 2 tries, 35.6s |
| +launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env |
| +staleapp /ready 200 after 2 tries, 35.6s |
| +launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env |
| +{"host":"shop-small.dev.harness.test","region":"home","rows":[{"kind":"bg","region":"home","count":275,"min_seq":1,"max_seq":275},{"kind":"tick","region":"home","count":2,"min_seq":0,"max_seq":0},{"kind":"worker","region":"home","count":138,"min_seq":1,"max_seq":138}],"file_key_sha256":"c50a3417bca34ab56b2877a592bf1d90f2dcf0647cc477a27f4b5ef65849e8f1","platform_key_sha256":"87a15ceb79258dccdfa79cca0da241a3a03d87abf931754437782a14a0f62eeb","object":"probe/744c9df2bfca1894.txt","has_secret":true,"worker_writes_failed":0} |
| +{"host":"shop-large.dev.harness.test","region":"home","rows":[{"kind":"bg","region":"home","count":204,"min_seq":1,"max_seq":204},{"kind":"tick","region":"home","count":1,"min_seq":0,"max_seq":0},{"kind":"worker","region":"home","count":102,"min_seq":1,"max_seq":102}],"file_key_sha256":"b8fc80ec801b0fdd8232810c4169e91e8754d0925746fe3da505081ad7544e1a","platform_key_sha256":"eac6dfc1cd29bf79a7187696351a67efd826454c867a05a5b29e5363be191f35","object":"probe/785fbd0f9fe7ecb6.txt","has_secret":true,"worker_writes_failed":0} |
| +{"host":"rollbackapp.dev.harness.test","region":"home","rows":[{"kind":"bg","region":"home","count":133,"min_seq":1,"max_seq":133},{"kind":"tick","region":"home","count":1,"min_seq":0,"max_seq":0},{"kind":"worker","region":"home","count":67,"min_seq":1,"max_seq":67}],"file_key_sha256":"b9c9f5cef335c6e96f0d92232f7a62e8d5eedc4123924852f59aceea1581e38d","platform_key_sha256":"e75e790b44457ea192fe0cc4b54822fff4b8ca33b0678f75064ba8872a5e1d8d","object":"probe/fa8ef4408998925b.txt","has_secret":true,"worker_writes_failed":0} |
| +{"host":"staleapp.dev.harness.test","region":"home","rows":[{"kind":"bg","region":"home","count":62,"min_seq":1,"max_seq":62},{"kind":"worker","region":"home","count":31,"min_seq":1,"max_seq":31}],"file_key_sha256":"b5327e8898a19c7934d803604a0a456092fa80b016fbd0cb2430d978ae6654e0","platform_key_sha256":"00b8ffeae6df2174dce295e15ee64186bdd744ac6c9daf3905f7f25360757cd9","object":"probe/93fb1a46cb625173.txt","has_secret":true,"worker_writes_failed":0} |
| +== routes |
| +-- leo |
| +-- sophie |
| +('console.dev.harness.test', 'console', '', '', 'canary/console', 'ready') |
| +('rollbackapp.dev.harness.test', 'rollbackapp', '', '', 'edge', 'ready') |
| +('shop-large.dev.harness.test', 'shop', '', '', 'edge', 'ready') |
| +('shop-small.dev.harness.test', 'shop', '', '', 'edge', 'ready') |
| +('staleapp.dev.harness.test', 'staleapp', '', '', 'edge', 'ready') |
| +('wakecheck.dev.harness.test', 'wakecheck', '', '', 'edge', 'ready') |
| +-- dennis |
| +BOX HOST LAYOUT REASON |
| +ada-letterpress (nested by default) rollbackapp.dev.harness.test nested — |
| +ada-letterpress (nested by default) shop-large.dev.harness.test nested — |
| +ada-letterpress (nested by default) shop-small.dev.harness.test nested — |
| +ada-letterpress (nested by default) wakecheck.dev.harness.test nested — |
| +setup exit=0 |
diff --git a/harness/evidence/run10/b1-move.log b/harness/evidence/run10/b1-move.log| new file mode 100644 |
| index 000000000..a86fbad84 |
| --- /dev/null |
| +++ b/harness/evidence/run10/b1-move.log |
| @@ -0,0 +1,334 @@ |
| +== shop/shop-small.dev.harness.test move (600) log dir /root/harness/r10-shop-small-move-213706 |
| +== before (21:37:06.745Z) |
| +{ |
| + "host": "shop-small.dev.harness.test", |
| + "region": "home", |
| + "region_from": "unset, treated as home", |
| + "mode": "normal", |
| + "boot_id": "744c9df2bfca1894", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 581, |
| + "min_seq": 1, |
| + "max_seq": 581 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 5, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 291, |
| + "min_seq": 1, |
| + "max_seq": 291 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531860, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532040, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532100, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + } |
| + ], |
| + "file_key_sha256": "c50a3417bca34ab56b2877a592bf1d90f2dcf0647cc477a27f4b5ef65849e8f1", |
| + "platform_k |
| +-- sophie |
| + env file: /data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.env e92a0004ca04d5d8 key-sha 87a15ceb79258dcc |
| + config.env /data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.config.env (21:32:13.202): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test/instance.key sha 7737438f556b9773 |
| + db /data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.db 64K |
| + instance dir: .activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env |
| + .instances dir: .activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env |
| + legacy /data/shop-small.dev.harness.test: ls: cannot access '/data/shop-small.dev.harness.test': No such file or directory |
| + row: [('ready', '', '', '/data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.db', '')] |
| + timers: [('shop-small.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532280, 1792532221, '200', 20, 0, 1792531809, 1792532280, 'fde048f3b8f8acdda54ff8b14d563c26', '', 0, '', '')] |
| +-- dennis |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/shop-small.dev.harness.test': No such file or directory |
| + legacy /data/shop-small.dev.harness.test: ls: cannot access '/data/shop-small.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + ie lease {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:47:46.479972136Z"}; uk lease absent |
| +== arm a once-timer in 600 s (21:37:08.818Z) |
| +{"at":"2026-10-20T21:47:08Z","due_at":1792532828,"name":"once-1"} |
| + |
| +== start (21:37:08.936Z) |
| + try 1 21:37:13.141Z exit=0: Moving shop-small.dev.harness.test from ie to uk. shop-small.dev.harness.test is offline from now until the move opens or is aborted. Waiting for ada-letterpress in ie to confirm it has stopped shop-small.dev.harness.test. shop-small.dev.harness.test is offline. Every box in ie has stopped shop-smal |
| + try 2 21:37:18.231Z exit=5: a move to uk is already offline-stopping |
| +start exit=5 at 21:37:18.234Z took 9.3s |
| +== seal (21:37:18.238Z) |
| + try 1 21:37:22.375Z exit=0: Copying shop-small.dev.harness.test to uk. Copied and checked. Run carlos move offline flip --host shop-small.dev.harness.test to start it in uk. |
| +seal exit=0 at 21:37:22.380Z took 4.1s |
| +== flip (21:37:22.385Z) |
| + try 1 21:37:26.513Z exit=0: Stopped in ie. Starting in uk. uk holds everything shop-small.dev.harness.test needs. Nothing is served yet. Run carlos move offline open --host shop-small.dev.harness.test to serve it from uk. |
| +flip exit=0 at 21:37:26.517Z took 4.1s |
| +== open (21:37:26.521Z) |
| + try 1 21:37:32.654Z exit=0: this move is offline-open. run carlos move offline status --host shop-small.dev.harness.test to see what it waits for Waiting for ada-letterpress to clean up shop-small.dev.harness.test. shop-small.dev.harness.test is served from uk. Run carlos move offline finish --host shop-small.dev.harness.test |
| + try 2 21:37:37.722Z exit=5: this move is offline-open. run carlos move offline status --host shop-small.dev.harness.test to see what it waits for |
| +open exit=5 at 21:37:37.726Z took 11.2s |
| +== dennis before the first request (21:37:37.730Z): config.env must be there (defect 6) |
| +-rw------- 1 carlos carlos 289 21:37:25.072199590 /data/shop-small.dev.harness.test.config.env |
| +== first request on dennis (21:37:37.820Z) |
| +dennis /ready [200 1.340423s] at 21:37:39.303Z: OFFLINE 30.4s since start |
| +{ |
| + "host": "shop-small.dev.harness.test", |
| + "region": "uk", |
| + "region_from": "CARLOS_REGION", |
| + "mode": "normal", |
| + "boot_id": "709fa32ddac28a98", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 586, |
| + "min_seq": 1, |
| + "max_seq": 586 |
| + }, |
| + { |
| + "kind": "bg", |
| + "region": "uk", |
| + "count": 1, |
| + "min_seq": 587, |
| + "max_seq": 587 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 5, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 293, |
| + "min_seq": 1, |
| + "max_seq": 293 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "uk", |
| + "count": 1, |
| + "min_seq": 294, |
| + "max_seq": 294 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531860, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532040, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532100, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + } |
| + ], |
| + "file_key_sha256": "c50a3417bca34ab56b2877a592bf1d90f2dcf0647cc477a27f4b5ef65849e8f1", |
| + "platform_key_sha256": "87a15ceb79258dccdfa79cca0da241a3a03d87abf931754437782a14a0f62e |
| +-- backhaul: sophie's edge [404 0.005592s], leo's edge [404 0.005988s] |
| +-- sophie after open |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: |
| + legacy /data/shop-small.dev.harness.test: ls: cannot access '/data/shop-small.dev.harness.test': No such file or directory |
| + row: [('moving', '', '', '/data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.db', '')] |
| + timers: [('shop-small.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532280, 1792532221, '200', 20, 0, 1792531809, 1792532280, 'fde048f3b8f8acdda54ff8b14d563c26', '', 0, '', ''), ('shop-small.dev.harness.test', 'once-1', 'once', 'at 1792532828', '/once', 1792532828, 0, '', 0, 0, 1792532228, 1792532828, 'b27b8df9f54cefe000e7b086e109074e', '', 0, '', '')] |
| + parked: /data/.offline-moves/shop-small.dev.harness.test/mv_vdogns62yqtqgb2ms62zd7p2wq/sealer |
| + parked: /data/.offline-moves/shop-small.dev.harness.test/mv_vdogns62yqtqgb2ms62zd7p2wq/source |
| +-- dennis after open |
| + env file: /data/shop-small.dev.harness.test.env e92a0004ca04d5d8 key-sha 87a15ceb79258dcc |
| + config.env /data/shop-small.dev.harness.test.config.env (21:37:25.072): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/shop-small.dev.harness.test/instance.key sha 7737438f556b9773 |
| + db /data/shop-small.dev.harness.test.db 72K |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/shop-small.dev.harness.test': No such file or directory |
| + legacy /data/shop-small.dev.harness.test: instance.key |
| + row: [('ready', '', 'uk', '/data/shop-small.dev.harness.test.db', '')] |
| + timers: [('shop-small.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532280, 1792532221, '200', 20, 0, 1792531809, 1792532280, 'fde048f3b8f8acdda54ff8b14d563c26', '', 0, '', ''), ('shop-small.dev.harness.test', 'once-1', 'once', 'at 1792532828', '/once', 1792532828, 0, '', 0, 0, 1792532228, 1792532828, 'b27b8df9f54cefe000e7b086e109074e', '', 0, '', '')] |
| +Moving shop-small.dev.harness.test from ie to uk. |
| +shop-small.dev.harness.test is served from uk. Run carlos move offline finish --host shop-small.dev.harness.test once ie has cleaned up. |
| +Item db is ok. |
| +Item db-tail is ok: every source box is at or below TXID 31. |
| +Item build-state is ok: the request record is in the build state the replica prefix carries. |
| +Item lease is ok. |
| +Item files is ok. |
| +Item env is ok. |
| +Item key-adoption is ok. |
| +Item config is ok. |
| +Item env-pin is ok. |
| +Item release is ok. |
| +Item store is unchecked. |
| +Item timers is ok. |
| +Item next-due is ok. |
| +Item replication is ok. |
| +Item row is ok. |
| +Item routing is unchecked. |
| +Item source-retire is unchecked. |
| +Item sidecar is ok: no source box keeps a sidecar file. |
| +Item restore-verify is unchecked. |
| +Item warm-cache is ok. |
| +Item receipts is unchecked. |
| +Item class is ok. |
| +ada-letterpress is sealed as the sealer. |
| +ada-letterpress is parked as the source. |
| +kathleen-juggling is parked as the source. |
| +noor-marquetry is opened as the target. |
| +ada-letterpress left /data/instances/shop-small.dev.harness.test in place: the mount view of the instance directory: it stays, empty. |
| +ada-letterpress left /data/.instances/shop-small.dev.harness.test in place: the instance directory stays, empty. |
| +ada-letterpress left /opt/carlos/live/shop-small.dev.harness.test in place: root-owned: listed for the census, not renamed. |
| +== finish (21:37:40.048Z) |
| + try 1 21:41:15.762Z exit=0: Waiting for ada-letterpress to clean up shop-small.dev.harness.test. Moved. shop-small.dev.harness.test is served from uk. It was offline for 18 seconds. |
| + try 2 21:41:20.829Z exit=5: shop-small.dev.harness.test has no offline move in flight |
| +finish exit=5 at 21:41:20.833Z took 220.8s |
| +Move mv_vdogns62yqtqgb2ms62zd7p2wq of shop-small.dev.harness.test is done. |
| +-- sophie after finish |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: |
| + legacy /data/shop-small.dev.harness.test: ls: cannot access '/data/shop-small.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + parked: /data/.offline-moves/shop-small.dev.harness.test/mv_vdogns62yqtqgb2ms62zd7p2wq/sealer |
| + parked: /data/.offline-moves/shop-small.dev.harness.test/mv_vdogns62yqtqgb2ms62zd7p2wq/source |
| +-- dennis after finish |
| + env file: /data/shop-small.dev.harness.test.env e92a0004ca04d5d8 key-sha 87a15ceb79258dcc |
| + config.env /data/shop-small.dev.harness.test.config.env (21:37:25.072): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/shop-small.dev.harness.test/instance.key sha 7737438f556b9773 |
| + db /data/shop-small.dev.harness.test.db 96K |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/shop-small.dev.harness.test': No such file or directory |
| + legacy /data/shop-small.dev.harness.test: instance.key |
| + row: [('ready', '', 'uk', '/data/shop-small.dev.harness.test.db', '')] |
| + timers: [('shop-small.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532520, 1792532461, '200', 11, 0, 1792531809, 1792532520, 'fde048f3b8f8acdda54ff8b14d563c26', '', 0, '', ''), ('shop-small.dev.harness.test', 'once-1', 'once', 'at 1792532828', '/once', 1792532828, 0, '', 0, 0, 1792532228, 1792532828, 'b27b8df9f54cefe000e7b086e109074e', '', 0, '', '')] |
| +-- leo after finish |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/shop-small.dev.harness.test': No such file or directory |
| + legacy /data/shop-small.dev.harness.test: ls: cannot access '/data/shop-small.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + ie lease {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"1970-01-01T00:00:00Z"}; uk lease {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T21:52:38.052020713Z"} |
| +{"region":"uk","move":null,"last_move":{"id":"mv_vdogns62yqtqgb2ms62zd7p2wq","outcome":"done","at":"2026-10-20T21:41:15.722885066Z","offline_s":17.553381779}} |
| +== sampler (changes only) |
| +sampler: 21:37:08.936Z ie {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:47:46.479972136Z"} uk absent sophie:.instances=[.activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env ] |
| +sampler: 21:37:15.708Z ie absent uk absent sophie:.instances=[.activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env ] |
| +sampler: 21:37:22.304Z ie {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"2026-10-27T21:37:20.616143874Z"} uk absent sophie:.instances=[.activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env ] |
| +sampler: 21:37:28.921Z ie {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"2026-10-27T21:37:26.156099833Z"} uk absent sophie:.instances=[.activator-shop-small.dev.harness.test.db-litestream launch.run.json shop-small.dev.harness.test shop-small.dev.harness.test.config.env shop-small.dev.harness.test.db shop-small.dev.harness.test.db-shm shop-small.dev.harness.test.db-wal shop-small.dev.harness.test.env ] |
| +sampler: 21:37:35.604Z ie {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"2026-10-27T21:37:31.715942866Z"} uk absent sophie:.instances=[] |
| +sampler: 21:37:42.269Z ie {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"2026-10-27T21:37:41.363998979Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T21:52:38.052020713Z"} sophie:.instances=[] |
| +sampler: 21:41:15.491Z ie {"owner":"moved:mv_vdogns62yqtqgb2ms62zd7p2wq","generation":1,"expires_at":"1970-01-01T00:00:00Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T21:52:38.052020713Z"} sophie:.instances=[] |
| +== sophie journal for shop-small.dev.harness.test since start |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 provision: held shop-small.dev.harness.test moving by mv_vdogns62yqtqgb2ms62zd7p2wq (was ready) |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 agent: shop-small.dev.harness.test: held for a move; asked root to re-judge what it staged |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 provision: shop-small.dev.harness.test: row is on the nested layout (db "/data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.db" socket "/run/carlos/instances/shop-small.dev |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001e txid.db=000000000000001e |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001e txid.db=000000000000001f |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=000000000000001f maxTXID=000000000000001f size=13279 |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001f txid.db=000000000000001f |
| +2026-10-20T21:37:09+00:00 sophie carlos[1389]: 2026/10/20 21:37:09 activator: shop-small.dev.harness.test hibernated |
| +2026-10-20T21:37:10+00:00 sophie carlos[1389]: 2026/10/20 21:37:10 layout: shop-small.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 provision: shop-small.dev.harness.test: row is on the nested layout (db "/data/instances/shop-small.dev.harness.test/shop-small.dev.harness.test.db" socket "/run/carlos/instances/shop-small.dev |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001f txid.db=000000000000001f |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001f txid.db=000000000000001f |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001f txid.db=000000000000001f |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=000000000000001f txid.db=000000000000001f |
| +2026-10-20T21:37:19+00:00 sophie carlos[1389]: 2026/10/20 21:37:19 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: checkpointed the local database (position 31, replica 31) |
| +2026-10-20T21:37:20+00:00 sophie carlos[1389]: 2026/10/20 21:37:20 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: sealed at TXID 31 (33 staged objects, files sha256:ebd8971fc0f435408ea47dd64db2879e16f29c57fac3f023fdbc651a2caad5 |
| +2026-10-20T21:37:20+00:00 sophie carlos[1389]: 2026/10/20 21:37:20 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: sealed |
| +2026-10-20T21:37:20+00:00 sophie carlos[1389]: 2026/10/20 21:37:20 layout: shop-small.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:37:25+00:00 sophie carlos[1389]: 2026/10/20 21:37:25 provision: not removing shop-small.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:37:26+00:00 sophie carlos[1389]: 2026/10/20 21:37:26 layout: shop-small.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:37:31+00:00 sophie carlos[1389]: 2026/10/20 21:37:31 provision: not removing shop-small.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:37:31+00:00 sophie carlos[1389]: 2026/10/20 21:37:31 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: wake fence lifted |
| +2026-10-20T21:37:31+00:00 sophie carlos[1389]: 2026/10/20 21:37:31 layout: shop-small.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:37:41+00:00 sophie carlos[1389]: 2026/10/20 21:37:41 provision: not removing shop-small.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:37:41+00:00 sophie carlos[1389]: 2026/10/20 21:37:41 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: waiting for reconcile to remove the row |
| +2026-10-20T21:37:41+00:00 sophie carlos[1389]: 2026/10/20 21:37:41 layout: shop-small.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:41:15+00:00 sophie carlos[1389]: 2026/10/20 21:41:15 provision: removed retired instance route shop-small.dev.harness.test (record gone) |
| +2026-10-20T21:41:15+00:00 sophie carlos[1389]: 2026/10/20 21:41:15 moves: shop-small.dev.harness.test: offline move mv_vdogns62yqtqgb2ms62zd7p2wq: tombstone lifted |
| +== dennis journal for shop-small.dev.harness.test since start |
| +2026-10-20T21:40:41+00:00 dennis carlos[1359]: 2026/10/20 21:40:41 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000033 txid.db=0000000000000033 |
| +2026-10-20T21:40:43+00:00 dennis carlos[1359]: 2026/10/20 21:40:43 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000033 txid.db=0000000000000033 |
| +2026-10-20T21:40:45+00:00 dennis carlos[1359]: 2026/10/20 21:40:45 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000033 txid.db=0000000000000033 |
| +2026-10-20T21:40:47+00:00 dennis carlos[1359]: 2026/10/20 21:40:47 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000033 txid.db=0000000000000033 |
| +2026-10-20T21:40:49+00:00 dennis carlos[1359]: 2026/10/20 21:40:49 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000033 txid.db=0000000000000034 |
| +2026-10-20T21:40:49+00:00 dennis carlos[1359]: 2026/10/20 21:40:49 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000034 maxTXID=0000000000000034 size=9360 |
| +2026-10-20T21:40:50+00:00 dennis carlos[1359]: 2026/10/20 21:40:50 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000034 |
| +2026-10-20T21:40:52+00:00 dennis carlos[1359]: 2026/10/20 21:40:52 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000034 |
| +2026-10-20T21:40:54+00:00 dennis carlos[1359]: 2026/10/20 21:40:54 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000034 |
| +2026-10-20T21:40:56+00:00 dennis carlos[1359]: 2026/10/20 21:40:56 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000034 |
| +2026-10-20T21:40:58+00:00 dennis carlos[1359]: 2026/10/20 21:40:58 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000034 |
| +2026-10-20T21:40:59+00:00 dennis carlos[1359]: 2026/10/20 21:40:59 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000034 txid.db=0000000000000036 |
| +2026-10-20T21:40:59+00:00 dennis carlos[1359]: 2026/10/20 21:40:59 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000035 maxTXID=0000000000000035 size=5050 |
| +2026-10-20T21:40:59+00:00 dennis carlos[1359]: 2026/10/20 21:40:59 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000036 maxTXID=0000000000000036 size=206 |
| +2026-10-20T21:41:01+00:00 dennis carlos[1359]: 2026/10/20 21:41:01 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000036 txid.db=0000000000000036 |
| +2026-10-20T21:41:03+00:00 dennis carlos[1359]: 2026/10/20 21:41:03 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000036 txid.db=0000000000000036 |
| +2026-10-20T21:41:05+00:00 dennis carlos[1359]: 2026/10/20 21:41:05 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000036 txid.db=0000000000000036 |
| +2026-10-20T21:41:07+00:00 dennis carlos[1359]: 2026/10/20 21:41:07 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000036 txid.db=0000000000000036 |
| +2026-10-20T21:41:09+00:00 dennis carlos[1359]: 2026/10/20 21:41:09 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000036 txid.db=0000000000000037 |
| +2026-10-20T21:41:09+00:00 dennis carlos[1359]: 2026/10/20 21:41:09 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000037 maxTXID=0000000000000037 size=12294 |
| +2026-10-20T21:41:10+00:00 dennis carlos[1359]: 2026/10/20 21:41:10 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000037 |
| +2026-10-20T21:41:12+00:00 dennis carlos[1359]: 2026/10/20 21:41:12 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000037 |
| +2026-10-20T21:41:14+00:00 dennis carlos[1359]: 2026/10/20 21:41:14 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000037 |
| +2026-10-20T21:41:16+00:00 dennis carlos[1359]: 2026/10/20 21:41:16 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000037 |
| +2026-10-20T21:41:16+00:00 dennis carlos[1359]: 2026/10/20 21:41:16 layout: deferred until asleep -> shop-small.dev.harness.test (awake (stopped for the move in 56m24s unless it sleeps first)) |
| +2026-10-20T21:41:18+00:00 dennis carlos[1359]: 2026/10/20 21:41:18 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000037 |
| +2026-10-20T21:41:19+00:00 dennis carlos[1359]: 2026/10/20 21:41:19 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000037 txid.db=0000000000000038 |
| +2026-10-20T21:41:19+00:00 dennis carlos[1359]: 2026/10/20 21:41:19 INFO ltx file uploaded system=store db=shop-small.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000038 maxTXID=0000000000000038 size=5833 |
| +2026-10-20T21:41:21+00:00 dennis carlos[1359]: 2026/10/20 21:41:21 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000038 txid.db=0000000000000038 |
| +2026-10-20T21:41:23+00:00 dennis carlos[1359]: 2026/10/20 21:41:23 INFO replica sync system=store db=shop-small.dev.harness.test.db replica=s3 txid.replica=0000000000000038 txid.db=0000000000000038 |
| +[hssm] i-06a1ec144aa5226fb 30c48b35-e995-4c14-a151-967968be5e7e Success exit=0 |
diff --git a/harness/evidence/run10/b2-grow.log b/harness/evidence/run10/b2-grow.log| new file mode 100644 |
| index 000000000..e69de29bb |
diff --git a/harness/evidence/run10/b3-abort.log b/harness/evidence/run10/b3-abort.log| new file mode 100644 |
| index 000000000..5bf6c9fd1 |
| --- /dev/null |
| +++ b/harness/evidence/run10/b3-abort.log |
| @@ -0,0 +1,198 @@ |
| +== rollbackapp/rollbackapp.dev.harness.test abort log dir /root/harness/r10-rollbackapp-abort-214221 |
| +== before (21:42:21.601Z) |
| +{ |
| + "host": "rollbackapp.dev.harness.test", |
| + "region": "home", |
| + "region_from": "unset, treated as home", |
| + "mode": "normal", |
| + "boot_id": "fa8ef4408998925b", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 1067, |
| + "min_seq": 1, |
| + "max_seq": 1067 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 9, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 534, |
| + "min_seq": 1, |
| + "max_seq": 534 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531860, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532100, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532280, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532340, |
| + "count": 1 |
| + |
| +-- sophie |
| + env file: /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.env c98f88b4eefa40cc key-sha e75e790b44457ea1 |
| + config.env /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.config.env (21:33:23.713): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test/instance.key sha 1def5a96ee11c991 |
| + db /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.db 100K |
| + instance dir: .activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env |
| + .instances dir: .activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env |
| + legacy /data/rollbackapp.dev.harness.test: ls: cannot access '/data/rollbackapp.dev.harness.test': No such file or directory |
| + row: [('ready', '', '', '/data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.db', '')] |
| + timers: [('rollbackapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532580, 1792532521, '200', 21, 0, 1792531815, 1792532580, 'f526119ec7c47ef5c4fe87bf4bfe6d5c', '', 0, '', '')] |
| +-- dennis |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/rollbackapp.dev.harness.test': No such file or directory |
| + legacy /data/rollbackapp.dev.harness.test: ls: cannot access '/data/rollbackapp.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + ie lease {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:53:57.956917163Z"}; uk lease absent |
| +== start (21:42:23.645Z) |
| + try 1 21:42:25.808Z exit=0: Moving rollbackapp.dev.harness.test from ie to uk. rollbackapp.dev.harness.test is offline from now until the move opens or is aborted. Waiting for ada-letterpress in ie to confirm it has stopped rollbackapp.dev.harness.test. rollbackapp.dev.harness.test is offline. Every box in ie has stopped rollb |
| +start exit=0 at 21:42:25.812Z took 2.2s |
| +== seal (21:42:25.816Z) |
| + try 1 21:42:31.963Z exit=0: Copying rollbackapp.dev.harness.test to uk. Copied and checked. Run carlos move offline flip --host rollbackapp.dev.harness.test to start it in uk. |
| +seal exit=0 at 21:42:31.969Z took 6.2s |
| +== flip (21:42:31.975Z) |
| + try 1 21:42:38.133Z exit=0: Stopped in ie. Starting in uk. uk holds everything rollbackapp.dev.harness.test needs. Nothing is served yet. Run carlos move offline open --host rollbackapp.dev.harness.test to serve it from uk. |
| +flip exit=0 at 21:42:38.137Z took 6.2s |
| +== abort (21:42:38.142Z) |
| + try 1 21:42:46.319Z exit=3: Moving rollbackapp.dev.harness.test back to ie. Waiting for ada-letterpress to clean up rollbackapp.dev.harness.test. Every box has put rollbackapp.dev.harness.test back in ie. Run carlos move offline finish --host rollbackapp.dev.harness.test to end the move. |
| + try 2 21:42:51.395Z exit=3: Moving rollbackapp.dev.harness.test back to ie. Every box has put rollbackapp.dev.harness.test back in ie. Run carlos move offline finish --host rollbackapp.dev.harness.test to end the move. |
| +abort exit=3 at 21:42:51.399Z took 13.3s (exit 3 = aborted, M8) |
| +== finish (21:42:51.403Z) |
| + try 1 21:42:51.502Z exit=3: rollbackapp.dev.harness.test is back in ie and serving. Nothing was lost. |
| +finish exit=3 at 21:42:51.506Z took 0.1s |
| +== source serves again (21:42:51.511Z) |
| +sophie /ready [200 0.451503s] at 21:42:52.063Z: OFFLINE 28.4s since start |
| +Move mv_nr5rvgh22i3fg7id66hwj5es4q of rollbackapp.dev.harness.test is aborted. |
| +{ |
| + "host": "rollbackapp.dev.harness.test", |
| + "region": "home", |
| + "region_from": "unset, treated as home", |
| + "mode": "normal", |
| + "boot_id": "03e12aff50acbc2a", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 1074, |
| + "min_seq": 1, |
| + "max_seq": 1074 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 9, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 538, |
| + "min_seq": 1, |
| + "max_seq": 538 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531860, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532100, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532280, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532340, |
| + "count": 1 |
| + |
| +-- sophie |
| + env file: /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.env c98f88b4eefa40cc key-sha e75e790b44457ea1 |
| + config.env /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.config.env (21:33:23.713): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test/instance.key sha 1def5a96ee11c991 |
| + db /data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.db 100K |
| + instance dir: .activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env |
| + .instances dir: .activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env |
| + legacy /data/rollbackapp.dev.harness.test: ls: cannot access '/data/rollbackapp.dev.harness.test': No such file or directory |
| + row: [('ready', '', '', '/data/instances/rollbackapp.dev.harness.test/rollbackapp.dev.harness.test.db', '')] |
| + timers: [('rollbackapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532580, 1792532521, '200', 21, 0, 1792531815, 1792532580, 'f526119ec7c47ef5c4fe87bf4bfe6d5c', '', 0, '', '')] |
| + parked: /data/.offline-moves/rollbackapp.dev.harness.test/mv_nr5rvgh22i3fg7id66hwj5es4q/sealer |
| +-- dennis |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/rollbackapp.dev.harness.test': No such file or directory |
| + legacy /data/rollbackapp.dev.harness.test: ls: cannot access '/data/rollbackapp.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + parked: /data/.offline-moves/rollbackapp.dev.harness.test/mv_nr5rvgh22i3fg7id66hwj5es4q/target |
| + ie lease {"owner":"sophie:1389","generation":2,"expires_at":"2026-10-20T21:57:51.703907301Z"}; uk lease absent |
| +{"region":null,"move":null,"last_move":{"id":"mv_nr5rvgh22i3fg7id66hwj5es4q","outcome":"aborted","at":"2026-10-20T21:42:51.465931513Z","offline_s":27.732732291}} |
| +== sampler (changes only) |
| +sampler: 21:42:23.645Z ie {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:53:57.956917163Z"} uk absent sophie:.instances=[.activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env ] |
| +sampler: 21:42:30.380Z ie {"owner":"moved:mv_nr5rvgh22i3fg7id66hwj5es4q","generation":1,"expires_at":"2026-10-27T21:42:30.741142243Z"} uk absent sophie:.instances=[.activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env ] |
| +sampler: 21:42:37.133Z ie {"owner":"moved:mv_nr5rvgh22i3fg7id66hwj5es4q","generation":1,"expires_at":"2026-10-27T21:42:36.94016916Z"} uk absent sophie:.instances=[.activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env ] |
| +sampler: 21:42:43.813Z ie {"owner":"moved:mv_nr5rvgh22i3fg7id66hwj5es4q","generation":1,"expires_at":"1970-01-01T00:00:00Z"} uk absent sophie:.instances=[.activator-rollbackapp.dev.harness.test.db-litestream launch.run.json rollbackapp.dev.harness.test rollbackapp.dev.harness.test.config.env rollbackapp.dev.harness.test.db rollbackapp.dev.harness.test.db-shm rollbackapp.dev.harness.test.db-wal rollbackapp.dev.harness.test.env ] |
| +[hssm] i-06a1ec144aa5226fb 771c3b69-e10e-4c4f-b110-1ba29f8cc2f3 Success exit=0 |
diff --git a/harness/evidence/run10/b4-stale.log b/harness/evidence/run10/b4-stale.log| new file mode 100644 |
| index 000000000..083f9edc1 |
| --- /dev/null |
| +++ b/harness/evidence/run10/b4-stale.log |
| @@ -0,0 +1,374 @@ |
| +== staleapp/staleapp.dev.harness.test stale (180) log dir /root/harness/r10-staleapp-stale-214351 |
| +== before (21:43:51.323Z) |
| +{ |
| + "host": "staleapp.dev.harness.test", |
| + "region": "home", |
| + "region_from": "unset, treated as home", |
| + "mode": "normal", |
| + "boot_id": "93fb1a46cb625173", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 1175, |
| + "min_seq": 1, |
| + "max_seq": 1175 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 9, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 588, |
| + "min_seq": 1, |
| + "max_seq": 588 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531920, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532280, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532340, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532400, |
| + "count": 1 |
| + |
| +-- sophie |
| + env file: /data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.env 045759299913cebd key-sha 00b8ffeae6df2174 |
| + config.env /data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.config.env (21:33:59.908): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test/instance.key sha 2c58caaf2a2f840f |
| + db /data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db 100K |
| + instance dir: .activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env |
| + .instances dir: .activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env |
| + legacy /data/staleapp.dev.harness.test: ls: cannot access '/data/staleapp.dev.harness.test': No such file or directory |
| + row: [('ready', '', '', '/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db', '')] |
| + timers: [('staleapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532640, 1792532581, '200', 16, 0, 1792531895, 1792532640, '56fd48083da1d2b41aea0cb18e0afade', '', 0, '', '')] |
| +-- dennis |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/staleapp.dev.harness.test': No such file or directory |
| + legacy /data/staleapp.dev.harness.test: ls: cannot access '/data/staleapp.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + ie lease {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:54:33.692686417Z"}; uk lease absent |
| +== start (21:43:53.471Z) |
| + try 1 21:43:55.654Z exit=0: Moving staleapp.dev.harness.test from ie to uk. staleapp.dev.harness.test is offline from now until the move opens or is aborted. Waiting for ada-letterpress in ie to confirm it has stopped staleapp.dev.harness.test. staleapp.dev.harness.test is offline. Every box in ie has stopped staleapp.dev.harn |
| +start exit=0 at 21:43:55.658Z took 2.2s |
| +== seal (21:43:55.663Z) |
| + try 1 21:44:03.804Z exit=0: Copying staleapp.dev.harness.test to uk. Copied and checked. Run carlos move offline flip --host staleapp.dev.harness.test to start it in uk. |
| +seal exit=0 at 21:44:03.808Z took 8.1s |
| +== pause 180 s between seal and flip (21:44:03.813Z) |
| + status after the pause: |
| + Moving staleapp.dev.harness.test from ie to uk. |
| + Copied and checked. Run carlos move offline flip --host staleapp.dev.harness.test to start it in uk. |
| + Item db is waiting. |
| + Item db-tail is ok: every source box is at or below TXID 64. |
| + Item build-state is ok: the request record is in the build state the replica prefix carries. |
| + Item lease is waiting. |
| + Item files is waiting. |
| + Item env is waiting. |
| + Item key-adoption is waiting. |
| + Item config is waiting. |
| + Item env-pin is waiting. |
| + Item release is waiting. |
| + Item store is unchecked. |
| + Item timers is waiting. |
| + Item next-due is waiting. |
| + Item replication is waiting. |
| + Item row is waiting. |
| + Item routing is unchecked. |
| + Item source-retire is unchecked. |
| + Item sidecar is ok: no source box keeps a sidecar file. |
| + Item restore-verify is unchecked. |
| + Item warm-cache is waiting. |
| + Item receipts is unchecked. |
| + Item class is ok. |
| + ada-letterpress is sealed as the sealer. |
| + ada-letterpress is stopped as the source. |
| + kathleen-juggling is stopped as the source. |
| +== flip (21:47:03.893Z) |
| + try 1 21:47:06.024Z exit=0: Stopped in ie. Starting in uk. uk holds everything staleapp.dev.harness.test needs. Nothing is served yet. Run carlos move offline open --host staleapp.dev.harness.test to serve it from uk. |
| +flip exit=0 at 21:47:06.030Z took 2.1s |
| +== open (21:47:06.035Z) |
| + try 1 21:47:10.203Z exit=0: this move is offline-open. run carlos move offline status --host staleapp.dev.harness.test to see what it waits for Waiting for ada-letterpress to clean up staleapp.dev.harness.test. staleapp.dev.harness.test is served from uk. Run carlos move offline finish --host staleapp.dev.harness.test once ie |
| +open exit=0 at 21:47:10.207Z took 4.2s |
| +== dennis before the first request (21:47:10.211Z): config.env must be there (defect 6) |
| +-rw------- 1 carlos carlos 301 21:47:04.640090702 /data/staleapp.dev.harness.test.config.env |
| +== first request on dennis (21:47:10.324Z) |
| + dennis /ready [404 0.005571s] 21:47:10.438Z |
| +dennis /ready [200 1.883290s] at 21:47:13.437Z: OFFLINE 200.0s since start |
| +{ |
| + "host": "staleapp.dev.harness.test", |
| + "region": "uk", |
| + "region_from": "CARLOS_REGION", |
| + "mode": "normal", |
| + "boot_id": "9d94f44b2b8ab79b", |
| + "rows": [ |
| + { |
| + "kind": "bg", |
| + "region": "home", |
| + "count": 1181, |
| + "min_seq": 1, |
| + "max_seq": 1181 |
| + }, |
| + { |
| + "kind": "bg", |
| + "region": "uk", |
| + "count": 1, |
| + "min_seq": 1182, |
| + "max_seq": 1182 |
| + }, |
| + { |
| + "kind": "tick", |
| + "region": "home", |
| + "count": 9, |
| + "min_seq": 0, |
| + "max_seq": 0 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "home", |
| + "count": 591, |
| + "min_seq": 1, |
| + "max_seq": 591 |
| + }, |
| + { |
| + "kind": "worker", |
| + "region": "uk", |
| + "count": 1, |
| + "min_seq": 592, |
| + "max_seq": 592 |
| + } |
| + ], |
| + "fired": [ |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792531920, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532160, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532220, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532280, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532340, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_at": 1792532400, |
| + "count": 1 |
| + }, |
| + { |
| + "kind": "tick", |
| + "name": "every-minute", |
| + "due_ |
| +-- backhaul through sophie's edge: [200 0.012980s] at 21:47:31.434Z, 17.9s after the first 200 on dennis (17 tries) |
| +-- backhaul through leo's edge: [200 0.009439s] at 21:47:31.549Z, 0.1s after the first 200 on dennis (1 tries) |
| +-- sophie after open |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: |
| + legacy /data/staleapp.dev.harness.test: ls: cannot access '/data/staleapp.dev.harness.test': No such file or directory |
| + row: [('moving', '', '', '/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db', '')] |
| + timers: [('staleapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532640, 1792532581, '200', 16, 0, 1792531895, 1792532640, '56fd48083da1d2b41aea0cb18e0afade', '', 0, '', '')] |
| + parked: /data/.offline-moves/staleapp.dev.harness.test/mv_rwidfnpau7xetmyhq4ooe3ogtu/sealer |
| + parked: /data/.offline-moves/staleapp.dev.harness.test/mv_rwidfnpau7xetmyhq4ooe3ogtu/source |
| +-- dennis after open |
| + env file: /data/staleapp.dev.harness.test.env 045759299913cebd key-sha 00b8ffeae6df2174 |
| + config.env /data/staleapp.dev.harness.test.config.env (21:47:04.640): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/staleapp.dev.harness.test/instance.key sha 2c58caaf2a2f840f |
| + db /data/staleapp.dev.harness.test.db 108K |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/staleapp.dev.harness.test': No such file or directory |
| + legacy /data/staleapp.dev.harness.test: instance.key |
| + row: [('ready', '', 'uk', '/data/staleapp.dev.harness.test.db', '')] |
| + timers: [('staleapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532880, 1792532836, '200', 13, 0, 1792531895, 1792532880, '56fd48083da1d2b41aea0cb18e0afade', '', 0, '', '')] |
| +Moving staleapp.dev.harness.test from ie to uk. |
| +staleapp.dev.harness.test is served from uk. Run carlos move offline finish --host staleapp.dev.harness.test once ie has cleaned up. |
| +Item db is ok. |
| +Item db-tail is ok: every source box is at or below TXID 64. |
| +Item build-state is ok: the request record is in the build state the replica prefix carries. |
| +Item lease is ok. |
| +Item files is ok. |
| +Item env is ok. |
| +Item key-adoption is ok. |
| +Item config is ok. |
| +Item env-pin is ok. |
| +Item release is ok. |
| +Item store is unchecked. |
| +Item timers is ok. |
| +Item next-due is ok. |
| +Item replication is ok. |
| +Item row is ok. |
| +Item routing is unchecked. |
| +Item source-retire is unchecked. |
| +Item sidecar is ok: no source box keeps a sidecar file. |
| +Item restore-verify is unchecked. |
| +Item warm-cache is ok. |
| +Item receipts is unchecked. |
| +Item class is ok. |
| +ada-letterpress is sealed as the sealer. |
| +ada-letterpress is parked as the source. |
| +kathleen-juggling is parked as the source. |
| +noor-marquetry is opened as the target. |
| +ada-letterpress left /data/instances/staleapp.dev.harness.test in place: the mount view of the instance directory: it stays, empty. |
| +ada-letterpress left /data/.instances/staleapp.dev.harness.test in place: the instance directory stays, empty. |
| +ada-letterpress left /opt/carlos/live/staleapp.dev.harness.test in place: root-owned: listed for the census, not renamed. |
| +== finish (21:47:31.902Z) |
| + try 1 21:52:28.226Z exit=0: Waiting for ada-letterpress to clean up staleapp.dev.harness.test. Moved. staleapp.dev.harness.test is served from uk. It was offline for 193 seconds. |
| +finish exit=0 at 21:52:28.230Z took 296.3s |
| +Move mv_rwidfnpau7xetmyhq4ooe3ogtu of staleapp.dev.harness.test is done. |
| +-- sophie after finish |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: |
| + legacy /data/staleapp.dev.harness.test: ls: cannot access '/data/staleapp.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [('staleapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792532640, 1792532581, '200', 16, 0, 1792531895, 1792532640, '56fd48083da1d2b41aea0cb18e0afade', '', 0, '', '')] |
| + parked: /data/.offline-moves/staleapp.dev.harness.test/mv_rwidfnpau7xetmyhq4ooe3ogtu/sealer |
| + parked: /data/.offline-moves/staleapp.dev.harness.test/mv_rwidfnpau7xetmyhq4ooe3ogtu/source |
| +-- dennis after finish |
| + env file: /data/staleapp.dev.harness.test.env 045759299913cebd key-sha 00b8ffeae6df2174 |
| + config.env /data/staleapp.dev.harness.test.config.env (21:47:04.640): HARNESS_NOTE MOVEPROBE_ACCESS_KEY MOVEPROBE_BUCKET MOVEPROBE_ENDPOINT MOVEPROBE_REGION MOVEPROBE_SECRET MOVEPROBE_SECRET_KEY |
| + files: /data/staleapp.dev.harness.test/instance.key sha 2c58caaf2a2f840f |
| + db /data/staleapp.dev.harness.test.db 140K |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/staleapp.dev.harness.test': No such file or directory |
| + legacy /data/staleapp.dev.harness.test: instance.key |
| + row: [('ready', '', 'uk', '/data/staleapp.dev.harness.test.db', '')] |
| + timers: [('staleapp.dev.harness.test', 'every-minute', 'schedule', 'every 1m', '/tick', 1792533180, 1792533121, '200', 24, 0, 1792531895, 1792533180, '56fd48083da1d2b41aea0cb18e0afade', '', 0, '', '')] |
| +-- leo after finish |
| + env file: absent key-sha |
| + instance dir: |
| + .instances dir: ls: cannot access '/data/.instances/staleapp.dev.harness.test': No such file or directory |
| + legacy /data/staleapp.dev.harness.test: ls: cannot access '/data/staleapp.dev.harness.test': No such file or directory |
| + row: [] |
| + timers: [] |
| + ie lease {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"1970-01-01T00:00:00Z"}; uk lease {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T22:07:11.649407101Z"} |
| +{"region":"uk","move":null,"last_move":{"id":"mv_rwidfnpau7xetmyhq4ooe3ogtu","outcome":"done","at":"2026-10-20T21:52:28.182314082Z","offline_s":192.568092385}} |
| +== sampler (changes only) |
| +sampler: 21:43:53.471Z ie {"owner":"sophie:1389","generation":1,"expires_at":"2026-10-20T21:54:33.692686417Z"} uk absent sophie:.instances=[.activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env ] |
| +sampler: 21:44:00.187Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:44:00.833049447Z"} uk absent sophie:.instances=[.activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env ] |
| +sampler: 21:44:07.059Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:44:06.943911187Z"} uk absent sophie:.instances=[.activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env ] |
| +sampler: 21:47:00.216Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:46:56.639909416Z"} uk absent sophie:.instances=[.activator-staleapp.dev.harness.test.db-litestream launch.run.json staleapp.dev.harness.test staleapp.dev.harness.test.config.env staleapp.dev.harness.test.db staleapp.dev.harness.test.db-shm staleapp.dev.harness.test.db-wal staleapp.dev.harness.test.env ] |
| +sampler: 21:47:06.953Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:47:06.412064389Z"} uk absent sophie:.instances=[] |
| +sampler: 21:47:13.685Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:47:12.207756171Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T22:02:11.643827923Z"} sophie:.instances=[] |
| +sampler: 21:47:33.713Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:47:32.912028334Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T22:02:11.643827923Z"} sophie:.instances=[] |
| +sampler: 21:52:13.758Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"2026-10-27T21:47:32.912028334Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T22:07:11.649407101Z"} sophie:.instances=[] |
| +sampler: 21:52:27.017Z ie {"owner":"moved:mv_rwidfnpau7xetmyhq4ooe3ogtu","generation":1,"expires_at":"1970-01-01T00:00:00Z"} uk {"owner":"dennis:1359","generation":1,"expires_at":"2026-10-20T22:07:11.649407101Z"} sophie:.instances=[] |
| +== sophie journal for staleapp.dev.harness.test since start |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000003f txid.db=0000000000000040 |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 INFO ltx file uploaded system=store db=staleapp.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000040 maxTXID=0000000000000040 size=12661 |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 provision: held staleapp.dev.harness.test moving by mv_rwidfnpau7xetmyhq4ooe3ogtu (was ready) |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 agent: staleapp.dev.harness.test: held for a move; asked root to re-judge what it staged |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 provision: staleapp.dev.harness.test: row is on the nested layout (db "/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db" socket "/run/carlos/instances/staleapp.dev.harness |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 activator: staleapp.dev.harness.test hibernated |
| +2026-10-20T21:43:54+00:00 sophie carlos[1389]: 2026/10/20 21:43:54 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 provision: staleapp.dev.harness.test: row is on the nested layout (db "/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db" socket "/run/carlos/instances/staleapp.dev.harness |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000040 txid.db=0000000000000040 |
| +2026-10-20T21:44:00+00:00 sophie carlos[1389]: 2026/10/20 21:44:00 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: checkpointed the local database (position 64, replica 64) |
| +2026-10-20T21:44:01+00:00 sophie carlos[1389]: 2026/10/20 21:44:01 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: sealed at TXID 64 (66 staged objects, files sha256:d66d9b471dacc53e19a05f9daf104ef22ef085215d25d40825a5e092674cbd8d |
| +2026-10-20T21:44:02+00:00 sophie carlos[1389]: 2026/10/20 21:44:02 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: sealed |
| +2026-10-20T21:44:02+00:00 sophie carlos[1389]: 2026/10/20 21:44:02 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:44:06+00:00 sophie carlos[1389]: 2026/10/20 21:44:06 provision: staleapp.dev.harness.test: row is on the nested layout (db "/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db" socket "/run/carlos/instances/staleapp.dev.harness |
| +2026-10-20T21:44:07+00:00 sophie carlos[1389]: 2026/10/20 21:44:07 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:46:56+00:00 sophie carlos[1389]: 2026/10/20 21:46:56 provision: staleapp.dev.harness.test: row is on the nested layout (db "/data/instances/staleapp.dev.harness.test/staleapp.dev.harness.test.db" socket "/run/carlos/instances/staleapp.dev.harness |
| +2026-10-20T21:47:06+00:00 sophie carlos[1389]: 2026/10/20 21:47:06 provision: not removing staleapp.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:47:06+00:00 sophie carlos[1389]: 2026/10/20 21:47:06 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: wake fence lifted |
| +2026-10-20T21:47:06+00:00 sophie carlos[1389]: 2026/10/20 21:47:06 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:47:11+00:00 sophie carlos[1389]: 2026/10/20 21:47:11 provision: not removing staleapp.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:47:12+00:00 sophie carlos[1389]: 2026/10/20 21:47:12 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:47:32+00:00 sophie carlos[1389]: 2026/10/20 21:47:32 provision: not removing staleapp.dev.harness.test (record gone): participant of a running operation |
| +2026-10-20T21:47:32+00:00 sophie carlos[1389]: 2026/10/20 21:47:32 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: waiting for reconcile to remove the row |
| +2026-10-20T21:47:33+00:00 sophie carlos[1389]: 2026/10/20 21:47:33 layout: staleapp.dev.harness.test: deferred -> status moving |
| +2026-10-20T21:52:26+00:00 sophie carlos[1389]: 2026/10/20 21:52:26 provision: removed retired instance route staleapp.dev.harness.test (record gone) |
| +2026-10-20T21:52:27+00:00 sophie carlos[1389]: 2026/10/20 21:52:27 moves: staleapp.dev.harness.test: offline move mv_rwidfnpau7xetmyhq4ooe3ogtu: tombstone lifted |
| +== dennis journal for staleapp.dev.harness.test since start |
| +2026-10-20T21:51:46+00:00 dennis carlos[1359]: 2026/10/20 21:51:46 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000005f txid.db=000000000000005f |
| +2026-10-20T21:51:48+00:00 dennis carlos[1359]: 2026/10/20 21:51:48 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000005f txid.db=000000000000005f |
| +2026-10-20T21:51:50+00:00 dennis carlos[1359]: 2026/10/20 21:51:50 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000005f txid.db=000000000000005f |
| +2026-10-20T21:51:52+00:00 dennis carlos[1359]: 2026/10/20 21:51:52 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000005f txid.db=000000000000005f |
| +2026-10-20T21:51:53+00:00 dennis carlos[1359]: 2026/10/20 21:51:53 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=000000000000005f txid.db=0000000000000060 |
| +2026-10-20T21:51:53+00:00 dennis carlos[1359]: 2026/10/20 21:51:53 INFO ltx file uploaded system=store db=staleapp.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000060 maxTXID=0000000000000060 size=12937 |
| +2026-10-20T21:51:55+00:00 dennis carlos[1359]: 2026/10/20 21:51:55 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000060 txid.db=0000000000000060 |
| +2026-10-20T21:51:57+00:00 dennis carlos[1359]: 2026/10/20 21:51:57 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000060 txid.db=0000000000000060 |
| +2026-10-20T21:51:59+00:00 dennis carlos[1359]: 2026/10/20 21:51:59 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000060 txid.db=0000000000000060 |
| +2026-10-20T21:52:01+00:00 dennis carlos[1359]: 2026/10/20 21:52:01 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000060 txid.db=0000000000000060 |
| +2026-10-20T21:52:03+00:00 dennis carlos[1359]: 2026/10/20 21:52:03 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000060 txid.db=0000000000000061 |
| +2026-10-20T21:52:03+00:00 dennis carlos[1359]: 2026/10/20 21:52:03 INFO ltx file uploaded system=store db=staleapp.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000061 maxTXID=0000000000000061 size=15167 |
| +2026-10-20T21:52:04+00:00 dennis carlos[1359]: 2026/10/20 21:52:04 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000061 |
| +2026-10-20T21:52:06+00:00 dennis carlos[1359]: 2026/10/20 21:52:06 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000061 |
| +2026-10-20T21:52:08+00:00 dennis carlos[1359]: 2026/10/20 21:52:08 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000061 |
| +2026-10-20T21:52:10+00:00 dennis carlos[1359]: 2026/10/20 21:52:10 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000061 |
| +2026-10-20T21:52:12+00:00 dennis carlos[1359]: 2026/10/20 21:52:12 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000061 |
| +2026-10-20T21:52:13+00:00 dennis carlos[1359]: 2026/10/20 21:52:13 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000061 txid.db=0000000000000062 |
| +2026-10-20T21:52:13+00:00 dennis carlos[1359]: 2026/10/20 21:52:13 INFO ltx file uploaded system=store db=staleapp.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000062 maxTXID=0000000000000062 size=6308 |
| +2026-10-20T21:52:15+00:00 dennis carlos[1359]: 2026/10/20 21:52:15 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000062 txid.db=0000000000000062 |
| +2026-10-20T21:52:17+00:00 dennis carlos[1359]: 2026/10/20 21:52:17 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000062 txid.db=0000000000000062 |
| +2026-10-20T21:52:19+00:00 dennis carlos[1359]: 2026/10/20 21:52:19 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000062 txid.db=0000000000000062 |
| +2026-10-20T21:52:21+00:00 dennis carlos[1359]: 2026/10/20 21:52:21 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000062 txid.db=0000000000000062 |
| +2026-10-20T21:52:23+00:00 dennis carlos[1359]: 2026/10/20 21:52:23 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000062 txid.db=0000000000000063 |
| +2026-10-20T21:52:23+00:00 dennis carlos[1359]: 2026/10/20 21:52:23 INFO ltx file uploaded system=store db=staleapp.dev.harness.test.db replica=s3 level=0 minTXID=0000000000000063 maxTXID=0000000000000063 size=9455 |
| +2026-10-20T21:52:24+00:00 dennis carlos[1359]: 2026/10/20 21:52:24 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000063 txid.db=0000000000000063 |
| +2026-10-20T21:52:26+00:00 dennis carlos[1359]: 2026/10/20 21:52:26 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000063 txid.db=0000000000000063 |
| +2026-10-20T21:52:28+00:00 dennis carlos[1359]: 2026/10/20 21:52:28 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000063 txid.db=0000000000000063 |
| +2026-10-20T21:52:28+00:00 dennis carlos[1359]: 2026/10/20 21:52:28 layout: deferred until asleep -> shop-small.dev.harness.test (awake (stopped for the move in 45m12s unless it sleeps first)), staleapp.dev.harness.test (awake (stopped for the move in 55m3s unl |
| +2026-10-20T21:52:30+00:00 dennis carlos[1359]: 2026/10/20 21:52:30 INFO replica sync system=store db=staleapp.dev.harness.test.db replica=s3 txid.replica=0000000000000063 txid.db=0000000000000063 |
| +[hssm] i-06a1ec144aa5226fb 3384427a-33a2-44c4-b6dd-cf0111e6c082 Success exit=0 |
diff --git a/harness/scenarios/cohort1/blocks/r10-grow.host.sh b/harness/scenarios/cohort1/blocks/r10-grow.host.sh| new file mode 100644 |
| index 000000000..b4964ebb9 |
| --- /dev/null |
| +++ b/harness/scenarios/cohort1/blocks/r10-grow.host.sh |
| @@ -0,0 +1,62 @@ |
| +#!/bin/bash |
| +# Run 10 (B2) on host, root: make shop-large look like flagship's large instance (the coordinator's discovery: |
| +# db ~535 MiB, a replica history of several GiB, a 1-minute timer, a files tree, a stale flat legacy dir). |
| +# r10-grow.host.sh <host> <db MiB> <churn MiB> <files MiB> |
| +# 1. Real rows, through SQLite on the LIVE database (the instance never sleeps: every-minute timer), as the |
| +# instance's owner (uid 989) so no file changes owner: table `orders` (id, created_at, customer, status, body: |
| +# ~1 KiB random hex), committed 4 MiB at a time with a busy timeout, paced so litestream ships as it goes. |
| +# 2. Replica history: rewrite `body` of existing rows (UPDATE in 4 MiB batches) for <churn MiB>, so the replica |
| +# holds that many MiB more of page history than the database's size. |
| +# 3. A files tree: <files MiB> of 1 MiB random files under <files dir>/media/, owner 989, 0600; plus report.token |
| +# (32 random bytes hex) beside instance.key. |
| +# 4. The stale flat legacy dir /data/<host>/ (as on the real Ireland box): copies of instance.key and report.token, |
| +# byte-identical to the nested ones, owner 989, dir 0700, files 0600. |
| +set -euo pipefail |
| +H=$1 DBMIB=$2 CHURN=$3 FMIB=$4 |
| +N=/data/instances/$H |
| +incus exec sophie -- test -f $N/$H.db |
| +cat > /root/harness/r10-grow.py <<'PY' |
| +import os, sqlite3, sys, time |
| +db, dbmib, churn = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) |
| +c = sqlite3.connect(db, timeout=60, isolation_level=None) |
| +c.execute("pragma busy_timeout=60000") |
| +c.execute("create table if not exists orders (id integer primary key, created_at integer not null, customer text not null, status text not null, body text not null)") |
| +def size(): return os.path.getsize(db) / 1048576 |
| +t0 = time.time(); batch = 4096 # rows of ~1 KiB: 4 MiB per commit |
| +n = c.execute("select coalesce(max(id),0) from orders").fetchone()[0] |
| +while size() < dbmib: |
| + c.execute("begin immediate") |
| + c.executemany("insert into orders (created_at, customer, status, body) values (?,?,?,?)", |
| + [(int(time.time()), "customer-%06d" % ((n + i) % 50000), ("paid", "sent", "open")[(n + i) % 3], os.urandom(500).hex()) for i in range(batch)]) |
| + c.execute("commit"); n += batch |
| + if n % (batch * 16) == 0: print("rows %d db %.0f MiB wal %.0f MiB at %.0fs" % (n, size(), os.path.getsize(db + "-wal") / 1048576, time.time() - t0), flush=True) |
| + time.sleep(0.15) |
| +print("insert done: rows %d db %.0f MiB in %.0fs" % (n, size(), time.time() - t0), flush=True) |
| +done, lo = 0, 1 |
| +while done < churn: |
| + c.execute("begin immediate") |
| + c.executemany("update orders set body=?, status='updated' where id=?", [(os.urandom(500).hex(), lo + i) for i in range(batch)]) |
| + c.execute("commit"); done += 4; lo += batch |
| + if lo > n: lo = 1 |
| + if done % 128 == 0: print("churn %d MiB db %.0f MiB wal %.0f MiB at %.0fs" % (done, size(), os.path.getsize(db + "-wal") / 1048576, time.time() - t0), flush=True) |
| + time.sleep(0.15) |
| +print("churn done: %d MiB, rows %d, db %.0f MiB, %.0fs" % (done, c.execute("select count(*) from orders").fetchone()[0], size(), time.time() - t0), flush=True) |
| +PY |
| +incus file push /root/harness/r10-grow.py sophie/tmp/r10-grow.py --uid 0 --gid 0 --mode 0644 -q |
| +echo "== rows and churn ($(date -u +%T))" |
| +incus exec sophie -- setpriv --reuid=989 --regid=989 --clear-groups python3 /tmp/r10-grow.py $N/$H.db $DBMIB $CHURN |
| +incus exec sophie -- rm -f /tmp/r10-grow.py; rm -f /root/harness/r10-grow.py |
| +echo "== files tree and the legacy dir ($(date -u +%T))" |
| +incus exec sophie -- bash -s -- $H $FMIB <<'BOX' |
| +set -euo pipefail |
| +H=$1 FMIB=$2 F=/data/instances/$1/$1 |
| +setpriv --reuid=989 --regid=989 --clear-groups bash -c "umask 077; mkdir -p $F/media; for i in \$(seq -w 1 $FMIB); do head -c 1048576 /dev/urandom > $F/media/upload-\$i.bin; done; od -An -N32 -tx1 /dev/urandom | tr -d ' \n' > $F/report.token" |
| +install -d -o 989 -g 989 -m 0700 /data/$H |
| +install -o 989 -g 989 -m 0600 $F/instance.key /data/$H/instance.key |
| +install -o 989 -g 989 -m 0600 $F/report.token /data/$H/report.token |
| +cmp $F/instance.key /data/$H/instance.key && cmp $F/report.token /data/$H/report.token && echo "legacy /data/$H/: instance.key and report.token = the nested copies" |
| +ls -lan /data/$H; du -sh $F /data/instances/$H/$H.db /data/instances/$H/$H.db-wal /data/instances/$H/.activator-$H.db-litestream |
| +(cd $F && find . -type f | sort | xargs sha256sum | sha256sum | cut -c1-16 | sed 's/^/files tree digest: /'; find . -type f | wc -l | sed 's/^/files: /') |
| +BOX |
| +echo "== the replica in harness-data-ie ($(date -u +%T))" |
| +runuser -l operator -c "aws s3 ls --recursive --summarize s3://harness-data-ie/ | grep -F '$H' | awk '{s+=\$3; n++} END {printf \"objects %d, %.0f MiB\\n\", n, s/1048576}'" |
diff --git a/harness/scenarios/cohort1/blocks/r10-move-precheck.host.sh b/harness/scenarios/cohort1/blocks/r10-move-precheck.host.sh| new file mode 100644 |
| index 000000000..fa995de26 |
| --- /dev/null |
| +++ b/harness/scenarios/cohort1/blocks/r10-move-precheck.host.sh |
| @@ -0,0 +1,43 @@ |
| +#!/bin/bash |
| +# Run 10 (B) pre-checks (r9-move-precheck.host.sh for Run 10's hosts). Run 9 (B) step 2a: the offline-move plan's "Morning runbook outline" P0 pre-checks, as commands (on host, root). |
| +# Source region: home/ie = leo (kathleen-juggling) and sophie (ada-letterpress, the console box). Target: dennis |
| +# (noor-marquetry, uk). Run 10 hosts: app:instance pairs below. |
| +PAIRS="shop:shop-small shop:shop-large rollbackapp:rollbackapp staleapp:staleapp" |
| +set -uo pipefail |
| +OPR() { runuser -l operator -c "$1" 2>&1 | grep -v ALERT; } |
| +B=harness-deployment |
| +echo "== P0.2 switches, R3, delegation, release ($(date -u +%T))" |
| +for b in leo sophie dennis; do |
| + incus exec $b -- bash -c ' |
| + printf "%s: host.env CARLOS_HOST_CLAIM=%s CARLOS_MOVES_PHASE1=%s CARLOS_REGION=%s CARLOS_POOLS=%s secrets_key=%s\n" "$(hostname)" \ |
| + "$(grep -c "^CARLOS_HOST_CLAIM=." /etc/carlos/host.env)" "$(grep -c "^CARLOS_MOVES_PHASE1=" /etc/carlos/host.env)" \ |
| + "$(grep "^CARLOS_REGION=" /etc/carlos/host.env | cut -d= -f2)" "$(grep "^CARLOS_POOLS=" /etc/carlos/host.env | cut -d= -f2)" "$(grep -c "^CARLOS_SECRETS_KEY=" /etc/carlos/host.env)" |
| + P=$(systemctl show -p MainPID --value carlos-edge); echo " edge env: $(tr "\0" "\n" < /proc/$P/environ | grep -cE "^CARLOS_(HOST_CLAIM=.|MOVES_PHASE1=)") claim/phase1 vars set" |
| + echo " $(systemctl show carlos-edge -p User) $(systemctl show carlos-edge -p Delegate) $(systemctl show carlos-edge -p ActiveState)" |
| + echo " carlos-stage.path $(systemctl is-active carlos-stage.path) / $(systemctl is-enabled carlos-stage.path 2>&1)" |
| + echo " $(/opt/carlos/carlos version | tail -1); guard policy: $(test -f /opt/carlos/system/guard/policy.json && python3 -c "import json;print(json.load(open(\"/opt/carlos/system/guard/policy.json\")).get(\"mode\",\"?\"))" || echo absent)"' |
| +done |
| +incus exec sophie -- bash -c 'P=$(systemctl show -p MainPID --value console.service); echo "console: $(tr "\0" "\n" < /proc/$P/environ | grep -E "^CARLOS_(HOST_CLAIM|MOVES_PHASE1|SECRETS_PUBKEY)=" | sed "s/PUBKEY=.*/PUBKEY=<set>/" | tr "\n" " ") exe $(sha256sum < /proc/$P/exe | cut -c1-16)"' |
| +OPR 'carlos system status --console http://127.0.0.1:8090' | tail -3 |
| +echo "== P0.2 edges files: offline-move-1, secrets_key_fp, hosts" |
| +OPR "aws s3 ls s3://$B/control/edges/" |
| +for b in ada-letterpress kathleen-juggling noor-marquetry; do |
| + OPR "aws s3 cp --quiet s3://$B/control/edges/$b.json -" | jq -c --arg b $b '{box:$b, region, pools, caps:.capabilities, secrets_key_fp, hibernation_configured, serves_shared_pool, hosts:(.hosts|keys)}' |
| +done |
| +echo "== P0.3 claims and backfill absent, certificates, layout" |
| +for p in $PAIRS; do a=${p%%:*} h=${p#*:} |
| + OPR "aws s3api head-object --bucket $B --key control/hosts/$h.dev.harness.test.json >/dev/null 2>&1 && echo 'control/hosts/$h... PRESENT' || echo 'control/hosts/$h.dev.harness.test.json absent'" |
| + OPR "aws s3 ls s3://$B/control/certs/$h.dev.harness.test | wc -l | sed 's/^/control\/certs\/$h: /'" |
| +done |
| +for a in shop rollbackapp staleapp; do OPR "aws s3 ls --recursive s3://$B/apps/bab/$a/ | awk '{print \$4}' | grep -E 'admission|instances/' "; done |
| +OPR "aws s3api head-object --bucket $B --key control/hosts-backfill.json >/dev/null 2>&1 && echo 'hosts-backfill PRESENT' || echo 'control/hosts-backfill.json absent'" |
| +OPR 'carlos status --layout --console http://127.0.0.1:8090' |
| +for b in leo sophie dennis; do echo "$b: $(incus exec $b -- journalctl -u carlos-edge --no-pager -b | grep -E 'layout: |nested by default' | tail -1)"; done |
| +echo "== P0.4 staging lane: list harness-data-uk moves/ with sophie's box credentials (the IE box role)" |
| +eval "$(python3 -c 'import json; u=json.load(open("/root/harness/minio-users.json"))["sophie"]; print("export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s" % (u["access_key"], u["secret_key"]))')" |
| +aws --endpoint-url http://203.0.113.10:9000 --region eu-west-2 s3api list-objects-v2 --bucket harness-data-uk --prefix moves/ --max-items 3 --query 'KeyCount' 2>&1 | sed 's/^/data-uk moves\/ KeyCount: /' |
| +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY |
| +echo "== P0.6 sizes, sidecars, local copies" |
| +for b in leo sophie dennis; do incus exec $b -- bash -c 'for h in shop-small.dev.harness.test shop-large.dev.harness.test rollbackapp.dev.harness.test staleapp.dev.harness.test; do |
| + for d in /data/instances/$h /data/$h.db /data/$h; do [ -e $d ] && echo "$(hostname) $h: $(du -sh $d | cut -f1) $d"; done |
| + ls /data/$h-sidecar.env /data/instances/$h/$h-sidecar.env 2>/dev/null | sed "s/^/SIDECAR /"; done; true'; done |
diff --git a/harness/scenarios/cohort1/blocks/r10-move-setup.host.sh b/harness/scenarios/cohort1/blocks/r10-move-setup.host.sh| new file mode 100644 |
| index 000000000..999524bc3 |
| --- /dev/null |
| +++ b/harness/scenarios/cohort1/blocks/r10-move-setup.host.sh |
| @@ -0,0 +1,92 @@ |
| +#!/bin/bash |
| +# Run 10 (B) setup (on host, root), from `post-candidate`: Run 9's r9-move-setup.host.sh (H17 secrets key, H13b test |
| +# CA in each box's trust store, H18 store binding as settings), for these apps and instances in account bab, |
| +# claims off, region home, woken on sophie only: |
| +# shop instances shop-small (B1) and shop-large (B2): two instances of ONE app, moved one at a time |
| +# (the coordinator: flagship's real app holds several instances under one app) |
| +# rollbackapp instance rollbackapp (B3, the abort after flip) |
| +# staleapp instance staleapp (B4, the stale-report pause) |
| +# Each app: moveprobe @ 54ae7fb9 (cmd/moveprobe is unchanged at e0986eab) as mp1 on edge; sealed secret; store |
| +# binding as settings; HARNESS_NOTE; `schedule set every-minute --every 1m --path /tick`. |
| +# Defect 6 check: for each instance, whether <host>.config.env is in the nested directory BEFORE the first request, |
| +# and how long the first wake takes to /ready 200 (Run 9: rollbackapp ran 12 min without config.env). |
| +set -euo pipefail |
| +K=/run/harness/keys; MC=/root/harness/bin/mc |
| +OP() { runuser -l operator -c "set -euo pipefail; C=http://127.0.0.1:8090; $1"; } |
| +declare -A INST=([shop]="shop-small shop-large" [rollbackapp]="rollbackapp" [staleapp]="staleapp") |
| +APPS="shop rollbackapp staleapp" |
| +# RESUME=<app> skips H17/H13b/H13/H18 and every app before <app>, and <app>'s `apps create` (the console's upload |
| +# limit, "too many uploads - try again in 59s", stopped the first run at staleapp's ship). |
| +RESUME=${RESUME:-} |
| +if [ -z "$RESUME" ]; then |
| +echo "== H17 secrets key ($(date -u +%T.%3NZ))" |
| +if [ ! -s $K/secrets.env ]; then (umask 077; /usr/local/bin/carlos secrets genkey > $K/secrets.env 2>/dev/null); fi |
| +. $K/secrets.env |
| +for b in leo sophie dennis; do |
| + incus exec $b -- bash -c "grep -q '^CARLOS_SECRETS_KEY=' /etc/carlos/host.env || printf 'CARLOS_SECRETS_KEY=%s\n' '$CARLOS_SECRETS_KEY' >> /etc/carlos/host.env" |
| +done |
| +incus exec sophie -- bash -c "grep -q '^CARLOS_SECRETS_PUBKEY=' /etc/carlos/host.env || printf 'CARLOS_SECRETS_PUBKEY=%s\n' '$CARLOS_SECRETS_PUBKEY' >> /etc/carlos/host.env" |
| +echo "== H13b test CA in each box's trust store" |
| +for b in leo sophie dennis; do |
| + incus file push $K/ca.pem $b/usr/local/share/ca-certificates/harness-test-ca.crt --uid 0 --gid 0 --mode 0644 -q |
| + incus exec $b -- sh -c 'update-ca-certificates 2>&1 | tail -1' |
| +done |
| +for b in leo sophie dennis; do incus exec $b -- bash -c 'systemctl restart carlos-edge.service; echo "$(hostname) edge $(systemctl is-active carlos-edge.service)"'; done |
| +incus exec sophie -- bash -c 'systemctl restart console.service; for i in $(seq 1 40); do [ "$(curl -s -o /dev/null -w %{http_code} http://127.0.0.1:8090/healthz)" = 200 ] && break; sleep 0.5; done; echo "console $(systemctl is-active console.service) healthz 200"' |
| +echo "secrets pubkey fp: $(printf %s "$CARLOS_SECRETS_PUBKEY" | base64 -d | sha256sum | cut -c1-16)" |
| +echo "== H13 certificates for the new hosts" |
| +bash /root/harness/certs.sh $(for app in $APPS; do for i in ${INST[$app]}; do echo $i.dev.harness.test; done; done) |
| +echo "== H18 MinIO stores ($(date -u +%T.%3NZ))" |
| +( set -a; . /root/harness/minio-root.env; set +a; $MC alias set harness http://203.0.113.10:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null ) |
| +for app in $APPS; do |
| + bk=harness-store-$app; u=store-$app; sk=$(od -An -N20 -tx1 /dev/urandom | tr -d ' \n') |
| + $MC mb --ignore-existing harness/$bk >/dev/null |
| + printf '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/*"]}]}' $bk $bk > /root/harness/policy-$u.json |
| + $MC admin policy create harness $u /root/harness/policy-$u.json >/dev/null |
| + $MC admin user add harness $u $sk >/dev/null |
| + $MC admin policy attach harness $u --user $u >/dev/null 2>&1 || true |
| + (umask 077; printf 'MOVEPROBE_BUCKET=%s\nMOVEPROBE_REGION=eu-west-1\nMOVEPROBE_ENDPOINT=http://203.0.113.10:9000\nMOVEPROBE_ACCESS_KEY=%s\nMOVEPROBE_SECRET_KEY=%s\nMOVEPROBE_SECRET=%s\n' $bk $u $sk "$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')" > $K/$app.store.env) |
| + install -o operator -g operator -m 0600 $K/$app.store.env /home/operator/.$app.store.env |
| + echo "$app: bucket $bk user $u" |
| +done |
| +else . $K/secrets.env; fi |
| +echo "== apps ($(date -u +%T.%3NZ))" |
| +install -d -o operator -g operator -m 0755 /home/operator/r10 |
| +aws s3 cp --quiet s3://carlos-harness-640012953448/artefacts/moveprobe-54ae7fb9/moveprobe /home/operator/r10/moveprobe |
| +chown operator:operator /home/operator/r10/moveprobe; chmod 0755 /home/operator/r10/moveprobe |
| +skip=${RESUME:+1} |
| +for app in $APPS; do |
| + [ "$app" = "$RESUME" ] && skip=; [ -n "$skip" ] && continue |
| + [ "$app" = "$RESUME" ] && create=: || create="carlos apps create --app $app --account bab --console \$C" |
| + OP "F=~/r10/moveprobe; sha256sum \$F; export CARLOS_SECRETS_PUBKEY='$CARLOS_SECRETS_PUBKEY' |
| +$create |
| +carlos ship --app $app --account bab --console \$C --version mp1 --label 'moveprobe 54ae7fb9 (harness Run 10)' --no-prompt \$F |
| +carlos promote --app $app --account bab --console \$C --no-prompt mp1 edge |
| +AWS_PROFILE=harness CARLOS_DEPLOYMENT_BUCKET=harness-deployment carlos instances enable --app $app --account bab --domain dev.harness.test --channel edge |
| +set -a; . ~/.$app.store.env; set +a |
| +carlos secrets set --app $app --account bab --console \$C MOVEPROBE_SECRET=\"\$MOVEPROBE_SECRET\" MOVEPROBE_ACCESS_KEY=\"\$MOVEPROBE_ACCESS_KEY\" MOVEPROBE_SECRET_KEY=\"\$MOVEPROBE_SECRET_KEY\" |
| +carlos env set --app $app --account bab --console \$C MOVEPROBE_BUCKET=\"\$MOVEPROBE_BUCKET\" MOVEPROBE_REGION=\"\$MOVEPROBE_REGION\" MOVEPROBE_ENDPOINT=\"\$MOVEPROBE_ENDPOINT\" HARNESS_NOTE=r10-$app |
| +carlos schedule set --app $app --account bab --console \$C --name every-minute --every 1m --path /tick" |
| + for i in ${INST[$app]}; do OP "carlos instances create --app $app --account bab --host $i.dev.harness.test --console \$C --no-prompt"; done |
| +done |
| +echo "== config.env before the first wake (defect 6) ($(date -u +%T.%3NZ))" |
| +sleep 40 |
| +for app in $APPS; do for i in ${INST[$app]}; do h=$i.dev.harness.test |
| + incus exec sophie -- bash -c "ls -la /data/instances/$h/ 2>&1 | grep -E 'config.env|\.env|total|No such' | sed 's/^/ $i: /'" |
| +done; done |
| +echo "== first wake on sophie only ($(date -u +%T.%3NZ))" |
| +for app in $APPS; do for i in ${INST[$app]}; do h=$i.dev.harness.test |
| + T=$(date +%s.%N) |
| + for n in $(seq 1 30); do |
| + c=$(incus exec sophie -- curl -s -m 40 --cacert /root/harness-ca.pem -o /dev/null -w '%{http_code}' --resolve $h:443:127.0.0.1 https://$h/ready); [ "$c" = 200 ] && break; sleep 5 |
| + done |
| + echo "$i /ready $c after $n tries, $(echo "$(date +%s.%N) $T" | awk '{printf "%.1f", $1-$2}')s" |
| + incus exec sophie -- bash -c "ls /data/instances/$h/ | tr '\n' ' '"; echo |
| +done; done |
| +for app in $APPS; do for i in ${INST[$app]}; do h=$i.dev.harness.test |
| + S=$(grep ^MOVEPROBE_SECRET= $K/$app.store.env | cut -d= -f2) |
| + incus exec sophie -- curl -s -m 20 --cacert /root/harness-ca.pem -H "Authorization: Bearer $S" --resolve $h:443:127.0.0.1 https://$h/report | jq -c '{host,region,rows,file_key_sha256,platform_key_sha256,object:.object.key,has_secret,worker_writes_failed}' |
| +done; done |
| +echo "== routes" |
| +for b in leo sophie dennis; do echo "-- $b"; incus exec $b -- python3 -c "import sqlite3; c=sqlite3.connect('file:/data/registry.db?mode=ro',uri=True); [print(r) for r in c.execute('select host,app,pool,region,channel,status from routes order by host')]" 2>&1; done |
| +OP 'carlos status --layout --console $C' 2>&1 | grep -v ALERT |
diff --git a/harness/scenarios/cohort1/blocks/r10-move.host.sh b/harness/scenarios/cohort1/blocks/r10-move.host.sh| new file mode 100644 |
| index 000000000..fd3614148 |
| --- /dev/null |
| +++ b/harness/scenarios/cohort1/blocks/r10-move.host.sh |
| @@ -0,0 +1,111 @@ |
| +#!/bin/bash |
| +# Run 10 (B): one offline operator move, sophie (home/ie) -> dennis (uk), doors back to back, timed (on host, root). |
| +# r10-move.host.sh <app> <instance> move [once_in_s] start, seal, flip, open, first request, finish |
| +# r10-move.host.sh <app> <instance> abort start, seal, flip, abort, finish (B3) |
| +# r10-move.host.sh <app> <instance> stale <pause_s> as move, with a pause between seal and flip (B4) |
| +# The candidate CLI (harness v0.28.8) against sophie's console. Nothing runs between the doors but the door |
| +# itself (the evidence is read after open, or by the background sampler): |
| +# - sampler (every 5 s, from start to the end): the source lease object harness-data-ie instances/<host>/lock.json |
| +# (owner, generation, expires_at) and sophie's /data/.instances/<host>/ listing (defect 8). |
| +# - MVR retries a door every 5 s on a refusal that is a wait (MV-O178 "not current", "Waiting for", exit 4/5) and |
| +# logs every try, so a retry (MV-O178) is visible. |
| +# once_in_s: arm a once-timer through the app on sophie that many seconds before start (B1: after the move; B2: due |
| +# during the move). |
| +set -uo pipefail |
| +APP=$1 I=$2 MODE=$3 ARG=${4:-} |
| +H=$I.dev.harness.test C=http://127.0.0.1:8090 K=/run/harness/keys |
| +S=$(grep ^MOVEPROBE_SECRET= $K/$APP.store.env | cut -d= -f2) |
| +L=/root/harness/r10-$I-$MODE-$(date -u +%H%M%S); mkdir -p $L |
| +ts() { date -u +%T.%3NZ; } |
| +el() { echo "$(date +%s.%N) $1" | awk '{printf "%.1f", $1-$2}'; } |
| +MV() { runuser -l operator -c "carlos move offline $1 --host $H --app $APP --account bab --console $C ${2:-}" 2>&1 | grep -v ALERT; return ${PIPESTATUS[0]}; } |
| +MVR() { local i out rc; for i in $(seq 1 120); do out=$(MV "$1" "${2:-}"); rc=$?; echo " try $i $(ts) exit=$rc: $(echo "$out" | tr '\n' ' ' | cut -c1-300)" |
| + case $rc in 0|3) return $rc;; esac # 0 done, 3 aborted (M8); a door that succeeded can still print "Waiting for ..." as progress (B1's first run) |
| + case "$rc:$out" in *"not current"*|*"Waiting for"*|5:*"still runs a process"*|4:*) sleep 5;; *) return $rc;; esac; done; return $rc; } |
| +REQ() { incus exec $1 -- curl -s -m 60 --cacert /root/harness-ca.pem -H "Authorization: Bearer $S" --resolve $H:443:127.0.0.1 -w '\n[%{http_code} %{time_total}s]' "https://$H$2"; } |
| +lockj() { runuser -l operator -c "aws s3 cp --quiet s3://$1/instances/$H/lock.json - 2>/dev/null" | jq -c '{owner, generation, expires_at}' 2>/dev/null || echo absent; } |
| +facts() { incus exec $1 -- bash -s -- $H <<'BOX' |
| +H=$1 |
| +for d in /data/instances/$H /data; do [ -f $d/$H.env ] && { E=$d/$H.env; break; }; done |
| +echo " env file: ${E:-absent} $( [ -n "${E:-}" ] && sha256sum < $E | cut -c1-16) key-sha $( [ -n "${E:-}" ] && grep ^CARLOS_INSTANCE_KEY= $E | cut -d= -f2- | tr -d '"\n' | sha256sum | cut -c1-16)" |
| +for f in /data/instances/$H/$H.config.env /data/$H.config.env; do [ -f $f ] && echo " config.env $f ($(stat -c %y $f | cut -c12-23)): $(cut -d= -f1 $f | sort | tr '\n' ' ')"; done |
| +for f in /data/instances/$H/$H/instance.key /data/$H/instance.key; do [ -f $f ] && echo " files: $f sha $(sha256sum < $f | cut -c1-16)"; done |
| +for d in /data/instances/$H/$H /data/$H; do [ -d $d/media ] && echo " files tree $d: $(cd $d && find . -type f | sort | xargs sha256sum | sha256sum | cut -c1-16) ($(find $d -type f | wc -l) files, $(du -sh $d | cut -f1))"; done |
| +for d in /data/instances/$H /data; do [ -f $d/$H.db ] && echo " db $d/$H.db $(du -sh $d/$H.db | cut -f1)"; done |
| +echo " instance dir: $(ls -A /data/instances/$H 2>/dev/null | tr '\n' ' ')" |
| +echo " .instances dir: $(ls -A /data/.instances/$H 2>&1 | tr '\n' ' ')" |
| +echo " legacy /data/$H: $(ls -A /data/$H 2>&1 | tr '\n' ' ')" |
| +python3 - $H <<'PY' |
| +import sqlite3,sys |
| +h=sys.argv[1]; c=sqlite3.connect('file:/data/registry.db?mode=ro',uri=True) |
| +print(' row:', c.execute('select status,last_error,region,db_path,replication from routes where host=?',(h,)).fetchall()) |
| +try: print(' timers:', [(r[0],r[1],r[2]) for r in c.execute('select name,kind,next_due from timers where host=?',(h,)).fetchall()]) |
| +except Exception as e: |
| + try: print(' timers:', c.execute('select * from timers where host=?',(h,)).fetchall()) |
| + except Exception as e2: print(' timers: ?', e2) |
| +PY |
| +ls -d /data/.offline-moves/$H/*/* 2>/dev/null | sed 's/^/ parked: /' |
| +BOX |
| +} |
| +sampler() { while :; do echo "$(ts) ie $(lockj harness-data-ie) uk $(lockj harness-data-uk) sophie:.instances=[$(incus exec sophie -- sh -c "ls -A /data/.instances/$H 2>&1" | tr '\n' ' ')]"; sleep 5; done; } |
| +echo "== $APP/$H $MODE ${ARG:+($ARG) }log dir $L" |
| +echo "== before ($(ts))" |
| +REQ sophie /report | head -c 1200; echo |
| +for b in sophie dennis; do echo "-- $b"; facts $b; done |
| +echo " ie lease $(lockj harness-data-ie); uk lease $(lockj harness-data-uk)" |
| +if [ "$MODE" = move ] && [ -n "$ARG" ]; then |
| + echo "== arm a once-timer in $ARG s ($(ts))"; incus exec sophie -- curl -s -m 20 --cacert /root/harness-ca.pem -X POST -H "Authorization: Bearer $S" --resolve $H:443:127.0.0.1 "https://$H/arm-once?in=$ARG"; echo |
| +fi |
| +sampler > $L/sampler.log 2>&1 & SP=$! |
| +trap 'kill $SP 2>/dev/null' EXIT |
| +echo "== start ($(ts))"; T_START=$(date +%s.%N); T=$T_START |
| +MVR start "--to uk"; echo "start exit=$? at $(ts) took $(el $T)s" |
| +echo "== seal ($(ts))"; T=$(date +%s.%N) |
| +MVR seal; echo "seal exit=$? at $(ts) took $(el $T)s" |
| +T_SEALED=$(date +%s.%N) |
| +if [ "$MODE" = stale ]; then |
| + echo "== pause $ARG s between seal and flip ($(ts))"; sleep "$ARG" |
| + echo " status after the pause:"; MV status | sed 's/^/ /' |
| +fi |
| +echo "== flip ($(ts))"; T=$(date +%s.%N) |
| +MVR flip; echo "flip exit=$? at $(ts) took $(el $T)s" |
| +if [ "$MODE" = abort ]; then |
| + echo "== abort ($(ts))"; T=$(date +%s.%N) |
| + MVR abort; echo "abort exit=$? at $(ts) took $(el $T)s (exit 3 = aborted, M8)" |
| + echo "== finish ($(ts))"; T=$(date +%s.%N); MVR finish; echo "finish exit=$? at $(ts) took $(el $T)s" |
| + echo "== source serves again ($(ts))" |
| + for i in $(seq 1 60); do c=$(REQ sophie /ready | tail -1); case "$c" in *"[200 "*) break;; esac; sleep 1; done |
| + echo "sophie /ready $c at $(ts): OFFLINE $(el $T_START)s since start" |
| + MV status; REQ sophie /report | head -c 1200; echo |
| + for b in sophie dennis; do echo "-- $b"; facts $b; done |
| + echo " ie lease $(lockj harness-data-ie); uk lease $(lockj harness-data-uk)" |
| + runuser -l operator -c "aws s3 ls --recursive s3://harness-data-uk/moves/ | grep -F $H | head -3" |
| + runuser -l operator -c "aws s3 cp --quiet s3://harness-deployment/apps/bab/$APP/instances/$H.json -" | jq -c '{region, move, last_move}' |
| + kill $SP; echo "== sampler (changes only)"; awk '{k=substr($0,14)} k!=p {print "sampler: " $0} {p=k}' $L/sampler.log; exit 0 |
| +fi |
| +echo "== open ($(ts))"; T=$(date +%s.%N) |
| +MVR open; echo "open exit=$? at $(ts) took $(el $T)s" |
| +echo "== dennis before the first request ($(ts)): config.env must be there (defect 6)" |
| +incus exec dennis -- bash -c "ls -la --time-style=+%T.%N /data/$H.config.env /data/instances/$H/$H.config.env 2>&1 | grep -v 'No such'" |
| +echo "== first request on dennis ($(ts))" |
| +for i in $(seq 1 120); do c=$(REQ dennis /ready | tail -1); case "$c" in *"[200 "*) break;; esac; echo " dennis /ready $c $(ts)"; sleep 1; done |
| +echo "dennis /ready $c at $(ts): OFFLINE $(el $T_START)s since start" |
| +REQ dennis /report | head -c 1500; echo |
| +# B1 saw 404 through both IE edges 7 s after open: poll each until 200 and time the window. |
| +for b in sophie leo; do T=$(date +%s.%N); codes=; for i in $(seq 1 180); do r=$(REQ $b /ready); c=$(echo "$r" | tail -1); case "$c" in *"[200 "*) break;; esac |
| + codes="$codes $(ts | cut -c4-8):${c:1:3}"; [ $i = 1 ] && echo " first non-200 body via $b: $(echo "$r" | head -2 | tr '\n' ' ' | cut -c1-200)"; sleep 1; done |
| + echo "-- backhaul through $b's edge: $c at $(ts), $(el $T)s after the first 200 on dennis ($i tries;${codes:- none before})"; done |
| +for b in sophie dennis; do echo "-- $b after open"; facts $b; done |
| +MV status |
| +echo "== finish ($(ts))"; T=$(date +%s.%N) |
| +MVR finish; echo "finish exit=$? at $(ts) took $(el $T)s" |
| +MV status |
| +for b in sophie dennis leo; do echo "-- $b after finish"; facts $b; done |
| +echo " ie lease $(lockj harness-data-ie); uk lease $(lockj harness-data-uk)" |
| +runuser -l operator -c "aws s3 cp --quiet s3://harness-deployment/apps/bab/$APP/instances/$H.json -" | jq -c '{region, move, last_move}' |
| +kill $SP |
| +echo "== sampler (changes only)"; awk '{k=substr($0,14)} k!=p {print "sampler: " $0} {p=k}' $L/sampler.log |
| +echo "== sophie journal for $H since start" |
| +incus exec sophie -- journalctl -u carlos-edge --since "@${T_START%.*}" --no-pager -o short-iso | grep -F "$H" | grep -viE "GET /|POST /tick" | cut -c1-260 | tail -40 |
| +echo "== dennis journal for $H since start" |
| +incus exec dennis -- journalctl -u carlos-edge --since "@${T_START%.*}" --no-pager -o short-iso | grep -F "$H" | grep -viE "GET /|POST /tick" | cut -c1-260 | tail -30 |
diff --git a/harness/scenarios/cohort1/blocks/r10-report.host.sh b/harness/scenarios/cohort1/blocks/r10-report.host.sh| new file mode 100644 |
| index 000000000..347cbdc1f |
| --- /dev/null |
| +++ b/harness/scenarios/cohort1/blocks/r10-report.host.sh |
| @@ -0,0 +1,18 @@ |
| +#!/bin/bash |
| +# Run 10: a compact /report for app:instance pairs through a box's edge (on host, root). |
| +# r10-report.host.sh <box> <app>:<instance> [...] |
| +# moveprobe's report can carry control characters in the object body, so it is parsed with strict=False. |
| +B=$1; shift |
| +for p in "$@"; do a=${p%%:*} i=${p#*:}; H=$i.dev.harness.test; S=$(grep ^MOVEPROBE_SECRET= /run/harness/keys/$a.store.env | cut -d= -f2) |
| +incus exec $B -- curl -s -m 60 --cacert /root/harness-ca.pem -H "Authorization: Bearer $S" --resolve $H:443:127.0.0.1 https://$H/report | python3 -c ' |
| +import json,sys |
| +raw=sys.stdin.read() |
| +try: d=json.loads(raw,strict=False) |
| +except Exception as e: print(sys.argv[1], "via", sys.argv[2], "unparsed:", raw[:200]); sys.exit() |
| +rows=" ".join("%s/%s n=%d %d-%d" % (r["kind"],r["region"],r["count"],r["min_seq"],r["max_seq"]) for r in d.get("rows",[])) |
| +once=[(f["name"],f["due_at"],f["count"]) for f in d.get("fired",[]) if f.get("kind")=="once"] |
| +ticks=sum(1 for f in d.get("fired",[]) if f.get("kind")=="tick") |
| +o=d.get("object") or {} |
| +print("%s via %s: region %s boot %s | %s | ticks %d once %s | pk %s fk %s | object %s | worker_failed %s" % (sys.argv[1], sys.argv[2], d.get("region"), d.get("boot_id"), rows, ticks, once, d.get("platform_key_sha256","")[:16], d.get("file_key_sha256","")[:16], o.get("key") if isinstance(o,dict) else o, d.get("worker_writes_failed"))) |
| +' $i $B |
| +done |