amadan
Merge branch 'moves-offline-t2' into moves-offline
4 files changed,
+1135
−0
internal/lease/leasetest/s3server.go+11 −0internal/lease/leasetest/s3server_test.go+32 −0internal/lease/tombstone.go+308 −0internal/lease/tombstone_test.go+784 −0
CI failed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.012s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 312.736s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.920s
ok github.com/carlosframework/platform/cmd/moveprobe 93.300s
ok github.com/carlosframework/platform/cmd/zddprobe 0.714s
ok github.com/carlosframework/platform/internal/activator 249.731s
ok github.com/carlosframework/platform/internal/agentsock 0.015s
ok github.com/carlosframework/platform/internal/alerting 0.004s
ok github.com/carlosframework/platform/internal/analytics 0.019s
ok github.com/carlosframework/platform/internal/appconfig 2.500s
ok github.com/carlosframework/platform/internal/apperr 0.070s
ok github.com/carlosframework/platform/internal/applog 0.124s
ok github.com/carlosframework/platform/internal/awsinv 0.004s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.733s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.693s
ok github.com/carlosframework/platform/internal/changelog 0.336s
ok github.com/carlosframework/platform/internal/claimactivation 0.774s
ok github.com/carlosframework/platform/internal/clamd 0.054s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 407.479s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.076s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.004s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.005s
ok github.com/carlosframework/platform/internal/domains 2.406s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 44.888s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.149s
ok github.com/carlosframework/platform/internal/gate 1.656s
ok github.com/carlosframework/platform/internal/genselect 0.014s
ok github.com/carlosframework/platform/internal/geo 0.015s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.176s
ok github.com/carlosframework/platform/internal/host 183.144s
ok github.com/carlosframework/platform/internal/hostactors 6.298s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.014s
ok github.com/carlosframework/platform/internal/hostbox 2.834s
ok github.com/carlosframework/platform/internal/hostclaim 0.592s
ok github.com/carlosframework/platform/internal/hostenv 0.014s
ok github.com/carlosframework/platform/internal/hostevidence 0.014s
ok github.com/carlosframework/platform/internal/hostexec 4.941s
ok github.com/carlosframework/platform/internal/hostfence 0.048s
ok github.com/carlosframework/platform/internal/hostfiles 10.518s
ok github.com/carlosframework/platform/internal/hostfloor 0.014s
ok github.com/carlosframework/platform/internal/hostguard 1.035s
ok github.com/carlosframework/platform/internal/hostops 1.610s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.014s
ok github.com/carlosframework/platform/internal/hostquarantine 0.014s
ok github.com/carlosframework/platform/internal/hostrelocate 0.477s
ok github.com/carlosframework/platform/internal/hostresolve 0.168s
ok github.com/carlosframework/platform/internal/hostwrite 0.014s
ok github.com/carlosframework/platform/internal/infraview 0.017s
ok github.com/carlosframework/platform/internal/instancectl 5.250s
ok github.com/carlosframework/platform/internal/instanceexec 2.781s
ok github.com/carlosframework/platform/internal/instancefs 0.018s
ok github.com/carlosframework/platform/internal/instancekey 0.016s
ok github.com/carlosframework/platform/internal/instanceupdates 0.022s
ok github.com/carlosframework/platform/internal/layout 1.567s
ok github.com/carlosframework/platform/internal/lbhealth 0.002s
ok github.com/carlosframework/platform/internal/lease 9.881s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.005s
ok github.com/carlosframework/platform/internal/ledger 0.015s
ok github.com/carlosframework/platform/internal/mail 0.169s
ok github.com/carlosframework/platform/internal/maintenance 0.082s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.529s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.090s
ok github.com/carlosframework/platform/internal/metrics 0.005s
ok github.com/carlosframework/platform/internal/moves 21.028s
ok github.com/carlosframework/platform/internal/moves/archive 0.792s
ok github.com/carlosframework/platform/internal/objstore 0.186s
ok github.com/carlosframework/platform/internal/ops 0.195s
ok github.com/carlosframework/platform/internal/park 0.216s
ok github.com/carlosframework/platform/internal/privatefile 0.006s
2026/10/20 00:32:13 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/20 00:32:13 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:32:13 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:32:13 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/20 00:32:13 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/20 00:32:13 provision: an app's records could not be listed: they are skipped
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:13 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:32:13 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/20 00:32:13 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:32:13 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:32:13 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:32:13 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:32:13 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:32:13 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:32:13 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:32:13 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:32:13 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/20 00:32:13 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/20 00:32:13 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/20 00:32:13 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:13 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:13 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:13 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:13 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:13 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:13 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:13 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:14 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:32:14 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/20 00:32:14 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/20 00:32:14 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/20 00:32:14 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:14 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:14 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (4.64s)
plan14_root_reads_test.go:940: root read census: 8 discrepancies
changed root read internal/lease/leasetest/s3server.go:NewS3Server; review class, task, sites and digest
changed root source envelope internal/lease
changed root source envelope internal/lease/leasetest
changed root source envelope internal/moves
changed root source envelope internal/provision
unclassified root read internal/lease/tombstone.go:readLock file:ReadAll@417
unclassified root read internal/provision/offline_move.go:OfflineStart file:Open@184
changed legacy declaration internal/lease/leasetest/s3server.go:NewS3Server
--- FAIL: TestPlan9B8RecordWriteCensus (0.55s)
plan9_b8_census_test.go:372: record-write census rejected 2 site(s); update the explicit inventory only after reviewing the source:
changed site internal/provision/provision.go:updateRecordSeeding: source differs from the reviewed digest
unlisted site internal/moves/offline.go:var OfflineManifest [lit:instances]: review it, classify it, and update its explicit inventory entry
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:32:42 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/20 00:32:42 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/20 00:32:42 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/20 00:32:42 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io hibernate false -> true
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged acme.go.tito.io status-named false -> true
2026/10/20 00:32:42 provision: converged acme.go.tito.io status-named true -> false
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:32:42 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:32:42 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:32:42 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/20 00:32:42 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/20 00:32:42 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/20 00:32:42 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:32:42 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/20 00:32:42 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/20 00:32:42 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/20 00:32:42 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:42 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/20 00:32:42 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/20 00:32:42 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/20 00:32:42 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:42 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:32:42 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:42 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:42 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/20 00:32:42 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:32:42 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:42 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/20 00:32:42 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/20 00:32:42 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/20 00:32:42 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:42 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:32:42 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:42 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/20 00:32:42 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/20 00:32:42 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:32:42 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:32:42 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:32:42 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/20 00:32:43 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:32:43 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/20 00:32:43 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/20 00:32:43 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:43 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:32:43 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/20 00:32:43 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/20 00:32:43 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/20 00:32:43 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:32:43 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/20 00:32:43 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/20 00:32:43 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/20 00:32:43 provision: dry-run: would converge owned.go.tito.io seed
2026/10/20 00:32:43 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/20 00:32:43 provision: converged owned.go.tito.io channel stable -> edge
2026/10/20 00:32:43 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/20 00:32:43 provision: converged owned.go.tito.io hibernate false -> true
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:43 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/20 00:32:43 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/20 00:32:43 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/20 00:32:43 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/20 00:32:43 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: removed retired static route site.go.tito.io (record gone)
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/20 00:32:43 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/20 00:32:43 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:43 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:32:43 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/20 00:32:43 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/20 00:32:43 provision: converged acme.go.tito.io channel stable -> edge
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:32:43 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/20 00:32:43 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/20 00:32:43 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/20 00:32:43 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/20 00:32:43 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:32:43 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/20 00:32:43 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/20 00:32:43 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/20 00:32:43 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/20 00:32:43 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/20 00:32:43 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/20 00:32:43 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/20 00:32:43 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/20 00:32:43 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:43 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/20 00:32:44 provision: a.shop.example.com: swapped for import sd_jb4vzsgs3p3wcwneyjnce4f7l4 (the original route is journaled)
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_jb4vzsgs3p3wcwneyjnce4f7l4 committed: its gate is cleared and its journal row dropped
2026/10/20 00:32:44 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_ny3jkgyc53hfr7lfauzck7vpiu rolled back to its original route
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_ny3jkgyc53hfr7lfauzck7vpiu: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_ny3jkgyc53hfr7lfauzck7vpiu is cleaned up and its journal row retired
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_k5aip7v2bhqvcnmmrdogw5mpxm rolled back to its original route
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_k5aip7v2bhqvcnmmrdogw5mpxm: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_k5aip7v2bhqvcnmmrdogw5mpxm is cleaned up and its journal row retired
2026/10/20 00:32:44 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_2rwo3mvesq4wehklc5bizaozm4
2026/10/20 00:32:44 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/20 00:32:44 provision: dry-run: would converge a.shop.example.com seed source
2026/10/20 00:32:44 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/20 00:32:44 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/20 00:32:44 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_kxgys32p4ilvhlnbco7i43hgaq cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_w7po3xlhrswbkxiazfro56qfma cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_vcpho2epbh5e6scj2gwyp7xvii is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_iaoiv3wgeawvb76m73fvv6obo4 is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_jhwdjzeci6ciwty66bwepwatdy is journaled, but this box cannot act on imports; its row is left alone
2026/10/20 00:32:44 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/20 00:32:44 provision: b.shop.example.com: import sd_vibg3egr6vssgpqa3a2n77luam left alone this pass: the import journal could not be read
2026/10/20 00:32:44 provision: c.shop.example.com: import sd_g55hzj2ldgfzhjnwqpthhfghmi left alone this pass: the import journal could not be read
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_d3w7oal6r4z4lnjix55gj3nznu committed: its gate is cleared and its journal row dropped
2026/10/20 00:32:44 provision: a.shop.example.com: not converging import sd_jesrmtcjzdklzvn4tgggu27k7y: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:32:44 provision: a.shop.example.com: not converging import sd_rrcnbjudkm72apzqi6mfctqyv4: it is swapped but not committed, and the row on this box has no import gate
2026/10/20 00:32:44 provision: a.shop.example.com: not converging import sd_yibohir4wbzsfcrcijwu4nyggm: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:32:44 provision: a.shop.example.com: not converging import sd_c5qwu3kwmiono5xz72yvr5v62u: it is placed but not committed, and the row on this box has no import gate
2026/10/20 00:32:44 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/20 00:32:44 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: a.shop.example.com: import sd_phfpnf3vtw756enhar7hmvc7by committed: its gate is cleared and its journal row dropped
2026/10/20 00:32:44 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:44 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:44 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/20 00:32:44 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/20 00:32:44 provision: acme.go.tito.io: aborted seed sd_dg7xuuw24vnutibx4phuovh44y: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:32:44 provision: acme.go.tito.io: aborted seed sd_dg7xuuw24vnutibx4phuovh44y rolled back (no box had swapped it)
2026/10/20 00:32:44 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/20 00:32:44 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:44 provision: removed instance route acme.go.tito.io (its seed sd_quncdnyvb2qbzrxqrcd35y6kzm was aborted); the next pass rolls it back
2026/10/20 00:32:44 provision: acme.go.tito.io: aborted seed sd_quncdnyvb2qbzrxqrcd35y6kzm: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/20 00:32:44 provision: acme.go.tito.io: aborted seed sd_quncdnyvb2qbzrxqrcd35y6kzm rolled back (no box had swapped it)
2026/10/20 00:32:45 provision: acme.go.tito.io: aborted seed sd_pfy2xwucthrcwjidpux7xkf2se has a row this pass may not remove; leaving the seed aborted
2026/10/20 00:32:45 provision: acme.go.tito.io: aborted seed sd_pjomayl5gw25vqf32dpyaiqyh4 rolled back (no box had swapped it)
2026/10/20 00:32:45 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:45 provision: removed instance route acme.go.tito.io (its seed sd_nb444upihundsl3f5zeegijpo4 was aborted); the next pass rolls it back
2026/10/20 00:32:45 provision: acme.go.tito.io: aborted seed sd_nb444upihundsl3f5zeegijpo4 has no route on this box: its local data is removed
2026/10/20 00:32:45 provision: acme.go.tito.io: aborted seed sd_nb444upihundsl3f5zeegijpo4 rolled back (no box had swapped it)
2026/10/20 00:32:45 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_7zguvc6lippbn2izfwpphniwwa (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_7zguvc6lippbn2izfwpphniwwa rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_7zguvc6lippbn2izfwpphniwwa is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_exyix42vnjdyja6t7adcqdjfne (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_exyix42vnjdyja6t7adcqdjfne rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_exyix42vnjdyja6t7adcqdjfne is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_sam6dn27wlrhwnr2fw272t74ri (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_sam6dn27wlrhwnr2fw272t74ri rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_sam6dn27wlrhwnr2fw272t74ri: the host's replica is left alone: its record no longer names this seed
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_sam6dn27wlrhwnr2fw272t74ri is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_beiad7nmdkakpvwqbs5mfgtn3a (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_beiad7nmdkakpvwqbs5mfgtn3a rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_beiad7nmdkakpvwqbs5mfgtn3a rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_beiad7nmdkakpvwqbs5mfgtn3a is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_hynjjdavblk7brk5pjedxzdpxm (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_hynjjdavblk7brk5pjedxzdpxm rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_tr6uwdcrymqeq4l2uwnckybwhe (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_tr6uwdcrymqeq4l2uwnckybwhe is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_r4r3r4unrhq4ewtruxhdoe7y7e (the original route is journaled)
2026/10/20 00:32:45 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_r4r3r4unrhq4ewtruxhdoe7y7e left alone this pass: its record could not be read
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_7qcmhuh4zp6463c3sleb2rwloq (the original route is journaled)
2026/10/20 00:32:45 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_7qcmhuh4zp6463c3sleb2rwloq left alone this pass: its record could not be read
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ss2jvm4rzcl7wywnrxl3tbx5la (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ss2jvm4rzcl7wywnrxl3tbx5la: close the import gate: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ss2jvm4rzcl7wywnrxl3tbx5la rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ss2jvm4rzcl7wywnrxl3tbx5la is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_o5x75757fbysyv5l3hhjamnhy4 (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_o5x75757fbysyv5l3hhjamnhy4: close the import gate: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_o5x75757fbysyv5l3hhjamnhy4 rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_o5x75757fbysyv5l3hhjamnhy4 is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_j3smkoffmmsw5z3kgzcqmdclg4 (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_j3smkoffmmsw5z3kgzcqmdclg4 rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_j3smkoffmmsw5z3kgzcqmdclg4 rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_j3smkoffmmsw5z3kgzcqmdclg4 is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_d5atjyutxo3rbq636vnbv4ptge (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_d5atjyutxo3rbq636vnbv4ptge rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_d5atjyutxo3rbq636vnbv4ptge is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_jnzeo2wmmavvmghy7xbc2g7oue (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jnzeo2wmmavvmghy7xbc2g7oue: retire its instance key: injected failure at retire-key
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jnzeo2wmmavvmghy7xbc2g7oue rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jnzeo2wmmavvmghy7xbc2g7oue is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ld5kuho3t2xgijwbbsb4xd5s4a (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ld5kuho3t2xgijwbbsb4xd5s4a rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ld5kuho3t2xgijwbbsb4xd5s4a is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_bfk425hmxfxbkpw4nyvlcmr5ba (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_bfk425hmxfxbkpw4nyvlcmr5ba: remove its local data: injected failure at remove-local
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_bfk425hmxfxbkpw4nyvlcmr5ba rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_bfk425hmxfxbkpw4nyvlcmr5ba is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_2ya6yglyveh25gp6pfpfod5pzm (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_2ya6yglyveh25gp6pfpfod5pzm rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_2ya6yglyveh25gp6pfpfod5pzm is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_btugkt5qzate45botsps5xruqa (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_btugkt5qzate45botsps5xruqa: remove its local data: injected failure at remove-local-done
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_btugkt5qzate45botsps5xruqa rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_btugkt5qzate45botsps5xruqa is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_xihonc7hsjazgku3gxn5elrvia (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_xihonc7hsjazgku3gxn5elrvia rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_xihonc7hsjazgku3gxn5elrvia is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_n6replyl2gry5p2i6vf6gtg3du (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_n6replyl2gry5p2i6vf6gtg3du: restore the original route: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_n6replyl2gry5p2i6vf6gtg3du rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_n6replyl2gry5p2i6vf6gtg3du is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_gyvwxbica6kukk5wosfs75uijq (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_gyvwxbica6kukk5wosfs75uijq: restore the original route: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_gyvwxbica6kukk5wosfs75uijq rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_gyvwxbica6kukk5wosfs75uijq is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_64oskt2ggitfldczg74ost3gs4 (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_64oskt2ggitfldczg74ost3gs4 is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_jrtkn67nwmgspp2zbdhypi6fcm (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jrtkn67nwmgspp2zbdhypi6fcm rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jrtkn67nwmgspp2zbdhypi6fcm: delete its snapshot: injected failure at delete-snapshot
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_jrtkn67nwmgspp2zbdhypi6fcm is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_qlsjgee56ceznd4ozolcn4xywy (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_qlsjgee56ceznd4ozolcn4xywy rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_qlsjgee56ceznd4ozolcn4xywy is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_wb7ermm26codv5shvbjwmtjgre (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_wb7ermm26codv5shvbjwmtjgre rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_wb7ermm26codv5shvbjwmtjgre: delete the host's replica: injected failure at delete-replica
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_wb7ermm26codv5shvbjwmtjgre is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_pa6y4zdoyn7t4jpq5y7mfd2fb4 (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_pa6y4zdoyn7t4jpq5y7mfd2fb4 rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_pa6y4zdoyn7t4jpq5y7mfd2fb4 is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ps556tgwdu3ogkrejxatuvr5za (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ps556tgwdu3ogkrejxatuvr5za rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ps556tgwdu3ogkrejxatuvr5za: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ps556tgwdu3ogkrejxatuvr5za is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_vbnfcs7vqvm4yz6ejvisxn7kmu (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_vbnfcs7vqvm4yz6ejvisxn7kmu rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_vbnfcs7vqvm4yz6ejvisxn7kmu: retire its journal row: constraint failed: injected (1811)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_vbnfcs7vqvm4yz6ejvisxn7kmu is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_qmusstoj6z7mg4jyinyuyhtaka (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_qmusstoj6z7mg4jyinyuyhtaka rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_qmusstoj6z7mg4jyinyuyhtaka is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: mark import sd_qmusstoj6z7mg4jyinyuyhtaka rolled back: injected failure at cas
2026/10/20 00:32:45 provision: a.shop.example.com: aborted seed sd_qmusstoj6z7mg4jyinyuyhtaka rolled back (no box had swapped it)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_6xssb7g6lmeigoltm4yaich7iu (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_6xssb7g6lmeigoltm4yaich7iu rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_6xssb7g6lmeigoltm4yaich7iu is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: aborted seed sd_6xssb7g6lmeigoltm4yaich7iu rolled back (no box had swapped it)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_yacwmplc3uqttfcxxwfd2ezpbu (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_yacwmplc3uqttfcxxwfd2ezpbu rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_yacwmplc3uqttfcxxwfd2ezpbu rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_yacwmplc3uqttfcxxwfd2ezpbu rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_yacwmplc3uqttfcxxwfd2ezpbu is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ubcllqkzcvtptj26p6mw6jo4ue (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ubcllqkzcvtptj26p6mw6jo4ue rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ubcllqkzcvtptj26p6mw6jo4ue is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ujoi4kqdpyuuozxyrplmap6bqe (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ujoi4kqdpyuuozxyrplmap6bqe rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ujoi4kqdpyuuozxyrplmap6bqe is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_3tnojkukuuekordtueg22le7da (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_3tnojkukuuekordtueg22le7da rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_3tnojkukuuekordtueg22le7da is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_fzthnzjzyraqbejtbtggkbvmgm (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_fzthnzjzyraqbejtbtggkbvmgm rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_fzthnzjzyraqbejtbtggkbvmgm is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_xh6j37amfhi2ne2nc57s3icuum (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_xh6j37amfhi2ne2nc57s3icuum rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_xh6j37amfhi2ne2nc57s3icuum is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ymwdjvzwunobpdqekqkjtbjfru (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ymwdjvzwunobpdqekqkjtbjfru rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ymwdjvzwunobpdqekqkjtbjfru is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_kcamahkqlcjtmzex3xafglioom (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_kcamahkqlcjtmzex3xafglioom rolled back to its original route
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_kcamahkqlcjtmzex3xafglioom is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_lqhbmjgsxa6rvpsocdr35kyg54 (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_lqhbmjgsxa6rvpsocdr35kyg54 has no route on this box: its local data is removed and no route is put back
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_lqhbmjgsxa6rvpsocdr35kyg54 is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_yqxexv3ma3slks2a54egwu6hta (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_x6ky2ut7l7uy5hyca3lal5cu6i (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_ad5pyvl5ueeqiwwnw77hmo65hu (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_ad5pyvl5ueeqiwwnw77hmo65hu cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_gjmnbxbmypvjg335wtk56vfzcm (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: import sd_gjmnbxbmypvjg335wtk56vfzcm is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/20 00:32:45 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/20 00:32:45 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_u45pdiiz6hoop5an3hukqllvim (the original route is journaled)
2026/10/20 00:32:45 provision: a.shop.example.com: swapped for import sd_dkwayofdyzduonp55qvmztxl64 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_67baitqxcdqvizatxndhvyymiy (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_ql465mmzjpw5eu4g2zd3opfh44 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_t4rwcq62nupzolikxca2iouzo4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_t4rwcq62nupzolikxca2iouzo4 rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_khkgax7xgcv4fgmksc72ruvi5a (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_khkgax7xgcv4fgmksc72ruvi5a has no route on this box: its local data is removed and no route is put back
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_khkgax7xgcv4fgmksc72ruvi5a is cleaned up and its journal row retired
2026/10/20 00:32:46 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_5ifbtdqaarw2jxd3ohheluemka (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_5ifbtdqaarw2jxd3ohheluemka: remove its local data: injected failure at remove-local
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_5ifbtdqaarw2jxd3ohheluemka is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_ov2b24ymnsuznblr7dfxo33jcq (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_ov2b24ymnsuznblr7dfxo33jcq: a route appeared at the host, so its local database is left and the rollback waits
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_vet4d6mhxglsfhb3bgh4rhnfe4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_vet4d6mhxglsfhb3bgh4rhnfe4 rollback deferred: no fence generation: registry busy
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_6dsfn3uhazu6yiqcxtdd3qfrb4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_6dsfn3uhazu6yiqcxtdd3qfrb4 rollback deferred: no fence generation: the generation source answered 0
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_zvijvlbtjo477fu6esld75cp3m (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_tbv5i3ivdssecjimy2j2774kq4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_n5rirak4ut3s54h76k3a2piul4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_47dsmajfybbi7o3kzqe2zl2mda (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_47dsmajfybbi7o3kzqe2zl2mda rolled back to its original route
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_47dsmajfybbi7o3kzqe2zl2mda is cleaned up and its journal row retired
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_fnanlpq2bdv5xgpds4og4qq6am (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_fnanlpq2bdv5xgpds4og4qq6am rolled back to its original route
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_fnanlpq2bdv5xgpds4og4qq6am is cleaned up and its journal row retired
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_5rjxrhzuelw3fpvjg2svu3iaby (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_5rjxrhzuelw3fpvjg2svu3iaby rolled back to its original route
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_5rjxrhzuelw3fpvjg2svu3iaby is cleaned up and its journal row retired
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_p6q767vfofduplq6tuidxeckrq (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_p6q767vfofduplq6tuidxeckrq: retire its instance key: the env file is a symlink
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_p6q767vfofduplq6tuidxeckrq rolled back to its original route
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_p6q767vfofduplq6tuidxeckrq is cleaned up and its journal row retired
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_ubemgdhdp7bfftkttzi7ujnu5q (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_t7bi7owwr53xzw3zhq5tae77ay (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_cv3gtfae4ic3nwjdfepvsvx2ly refused: there is no route at a.shop.example.com to replace
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_glvenpamtyc72t3rj47aftt2oy refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_a635jb4qkljhfjvcddx62pi5we refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_a6riz2ilg52qdmhzh5vlbtw4s4 refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_regsqlm4j7fl7t6v66b2dsax44 refused: the route belongs to app other
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_jdbechgshaa6vqpvh7zlacly2a refused: the route is filed under another account
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_gthi2hw74lg2rutj3siydiv26q refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_4h2nbsch3ggasbsuc4z2v6vcza refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_uc4qmvoiqb7e2ub7yde2j5cpku refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_mt6tcjzcvisataz5toy2wgq4ba refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_dqw6uqzif2ii2axs37c4gehhli refused: the route is not an address route this import can take over (socket)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_szejw4p6g2qpvza7zh6iohrx6y refused: the route is not an address route this import can take over (no address)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_fsmqswbefo3zgghxodhm5btayi refused: the route is not an address route this import can take over (provisioned)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_o22umll7wguclmgrzablmhl2uu refused: the route is not an address route this import can take over (kind static)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_kcd4x5esyw5coi25kkzg6yh674 refused: the route is not an address route this import can take over (backing)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_klvdtcfakjsrbrgmgjzp7csds4 refused: the route is not an address route this import can take over (tunnel)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_dtghdnkhypch2xprsuxtoooxqy refused: the route is not an address route this import can take over (caps)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_jeeeszofgfyi22snlz4wzxg6d4 refused: the route is not an address route this import can take over (platform)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_4rqleaqkz5myqtud53ofv4vn6m refused: the route is not an address route this import can take over (canary)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_benkctpkab7yhvn2dqntqjmoie refused: the route is not an address route this import can take over (kind sidecar)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_zkke5ktyd5oy5oalqablvuzota refused: the route is not an address route this import can take over (region)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_mdx6dmvjn42ykbpvjfsc2xn2zu refused: the route is not an address route this import can take over (fleet_box)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_eglvcsmsgbimyp67ybxji4hdxu refused: the route is not an address route this import can take over (hibernate)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_q44du7vm6yy5sy7cuuk5vqajfq refused: the route is not an address route this import can take over (claim_owner)
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_fgfztoyddzw4wknlzfvrzpfcci refused: this box does not run instances nested
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_ghdxfvsgk4lf7ai3agcr5wrb6a refused: this box does not run instances nested
2026/10/20 00:32:46 provision: a.shop.example.com: import sd_hj6uv6x7lolyjc2jouci75xjpq refused: claim control is on for this box
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_xajqedy4jqxbvsghdugpzqzsc4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_topb4z4wpndi2lpphkngddkiy4 (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_bip5ymdvd634kelxclvu6ceqhi: its verify hash is not a sha256
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_lqkcpkbpta4p2hrg4v5tqasxte: its verify hash is not a sha256
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_xy7utg4abnh5hpfbdufo2awvwm: its verify hash is not a sha256
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/20 00:32:46 provision: A.shop.example.com/..: not swapping for import sd_wdj6orqg553phzcbebvgev7oce: its host or seed id is malformed (seeds: not a host)
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_fu7tlyvf7zqna6jzpi56amo2sa: its record does not describe a nested, hibernating instance
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_soed2fgvbnob3clprhn22uwz7a: its record does not describe a nested, hibernating instance
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_b4xe3lgetuomptti5j7uu6bc3i: its record does not describe a nested, hibernating instance
2026/10/20 00:32:46 provision: a.shop.example.com: not swapping for import sd_dw364hdbg6qj5xqpaotl3imwsa: its record does not describe a nested, hibernating instance
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_infhewsqu2tjaoubzvtpkqoxyu (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_iissgsxda7b74rfjgmwu7gnbjq (the original route is journaled)
2026/10/20 00:32:46 provision: a.shop.example.com: swapped for import sd_bjxhtw722324nu4g3o3ydn6wja (the original route is journaled)
2026/10/20 00:32:47 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/20 00:32:47 provision: dry-run: would consider swapping a.shop.example.com for import sd_pkkeouyu4g5ebvxyrdnhivczim
2026/10/20 00:32:47 provision: a.shop.example.com: import sd_wazqgraio5awzgk2oe7l6fn2si rolled back to its original route
2026/10/20 00:32:47 provision: a.shop.example.com: import sd_wazqgraio5awzgk2oe7l6fn2si: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/20 00:32:47 provision: a.shop.example.com: import sd_wazqgraio5awzgk2oe7l6fn2si is cleaned up and its journal row retired
2026/10/20 00:32:47 provision: a.shop.example.com: import sd_usluyghj3jhqhaogcihvz22l6u left alone: the import journal holds seed sd_dgpu2tzstr3dm7y2p4mpkydibu at swapped
2026/10/20 00:32:47 provision: a.shop.example.com: not minting import sd_6cciqpwdwrqq2czm5nqwevmfie: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:32:47 provision: a.shop.example.com: not minting import sd_74yw6x66sqpmqcnjpujyq4aojm: it is placed but not committed, and only its swap makes its row
2026/10/20 00:32:47 provision: a.shop.example.com: swapped for import sd_4v4y3nq7nmtirdymq5idicstmq (the original route is journaled)
2026/10/20 00:32:47 provision: a.shop.example.com: not minting import sd_4v4y3nq7nmtirdymq5idicstmq: it is swapped but not committed, and only its swap makes its row
2026/10/20 00:32:47 provision: a.shop.example.com: not minting import sd_4v4y3nq7nmtirdymq5idicstmq: it is placed but not committed, and only its swap makes its row
2026/10/20 00:32:47 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/20 00:32:47 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/20 00:32:47 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/20 00:32:47 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/20 00:32:47 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/20 00:32:47 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/20 00:32:47 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:32:47 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/20 00:32:47 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/20 00:32:47 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/20 00:32:47 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:32:47 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/20 00:32:47 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/20 00:32:47 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/20 00:32:47 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/20 00:32:47 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/20 00:32:47 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/20 00:32:47 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/20 00:32:47 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/20 00:32:47 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/20 00:32:47 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/20 00:32:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:47 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/20 00:32:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:47 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:32:47 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:32:47 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:32:47 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/20 00:32:47 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/20 00:32:47 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/20 00:32:47 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL github.com/carlosframework/platform/internal/provision 34.222s
ok github.com/carlosframework/platform/internal/publicrelease 0.010s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.457s
ok github.com/carlosframework/platform/internal/relay 7.228s
ok github.com/carlosframework/platform/internal/release 24.621s
ok github.com/carlosframework/platform/internal/rollout 3.769s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/rootparticipation 0.057s
ok github.com/carlosframework/platform/internal/rootstate 0.027s
ok github.com/carlosframework/platform/internal/runs 0.580s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.500s
ok github.com/carlosframework/platform/internal/seeds 0.012s
ok github.com/carlosframework/platform/internal/sim/hostlife 9.433s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 3.636s
ok github.com/carlosframework/platform/internal/status 0.063s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.279s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.004s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.557s
FAIL
make: *** [Makefile:18: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/lease/leasetest/s3server.go b/internal/lease/leasetest/s3server.go| index 0b07ae29d..051b9f391 100644 |
| --- a/internal/lease/leasetest/s3server.go |
| +++ b/internal/lease/leasetest/s3server.go |
| @@ -217,6 +217,17 @@ func NewS3Server(f *VersionedBucket, serves ...string) (http.Handler, func() int |
| } |
| w.WriteHeader(http.StatusNoContent) |
| case r.Method == http.MethodDelete && key != "": |
| + if match := r.Header.Get("If-Match"); match != "" { // as S3 enforces it on DeleteObject |
| + current := f.buckets[bucket][key] |
| + if len(current) == 0 || current[0].deleteMarker { |
| + s3err(w, http.StatusNotFound, "NoSuchKey") |
| + return |
| + } |
| + if current[0].etag != strings.Trim(match, `"`) { |
| + s3err(w, http.StatusPreconditionFailed, "PreconditionFailed") |
| + return |
| + } |
| + } |
| id, err := f.bareDeleteLocked(bucket, key) // versioning on: a delete marker |
| if err != nil { |
| writeErr(w, err) |
diff --git a/internal/lease/leasetest/s3server_test.go b/internal/lease/leasetest/s3server_test.go| index b4546f938..3b3581c2b 100644 |
| --- a/internal/lease/leasetest/s3server_test.go |
| +++ b/internal/lease/leasetest/s3server_test.go |
| @@ -255,3 +255,35 @@ func TestS3ServerConditionalReplaceUsesReadETag(t *testing.T) { |
| t.Fatal("CAS over delete marker accepted", got.Code) |
| } |
| } |
| + |
| +// DeleteObject honours If-Match as S3 does: a stale ETag is a 412 and removes |
| +// nothing, a missing key is a 404, the current ETag deletes, and a plain |
| +// delete (no header) is unchanged. |
| +func TestS3ServerConditionalDeleteUsesReadETag(t *testing.T) { |
| + m := NewVersionedBucket() |
| + m.AddBucket("data") |
| + h, _ := NewS3Server(m) |
| + do(t, h, http.MethodPut, "/data/lock.json", "one", map[string]string{"If-None-Match": "*"}) |
| + stale := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag") |
| + do(t, h, http.MethodPut, "/data/lock.json", "two", map[string]string{"If-Match": stale}) |
| + current := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag") |
| + before := m.Snapshot() |
| + if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": stale}); got.Code != http.StatusPreconditionFailed { |
| + t.Fatal("stale conditional delete accepted", got.Code) |
| + } |
| + if m.Snapshot() != before { |
| + t.Fatal("stale conditional delete mutated versions") |
| + } |
| + if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNoContent { |
| + t.Fatal("current conditional delete refused", got.Code) |
| + } |
| + if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound { |
| + t.Fatal("conditional delete of a deleted key", got.Code) |
| + } |
| + if got := do(t, h, http.MethodDelete, "/data/never", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound { |
| + t.Fatal("conditional delete of a missing key", got.Code) |
| + } |
| + if got := do(t, h, http.MethodDelete, "/data/never", "", nil); got.Code != http.StatusNoContent { |
| + t.Fatal("plain delete changed", got.Code) |
| + } |
| +} |
diff --git a/internal/lease/tombstone.go b/internal/lease/tombstone.go| new file mode 100644 |
| index 000000000..c0054208f |
| --- /dev/null |
| +++ b/internal/lease/tombstone.go |
| @@ -0,0 +1,308 @@ |
| +package lease |
| + |
| +// The tombstone lease: the offline operator move's linearization point. |
| +// |
| +// A move that has stopped a host on its source box writes a lease at the |
| +// host's own lock.json under the owner "moved:<id>" and a long expiry. From |
| +// then on lease.Acquire on that location fails with a *HeldError naming the |
| +// move, so no box can wake the host and write to the replica behind the |
| +// move's back. Three operations manage it, each a conditional write over the |
| +// client AcquireTTL builds (IfNoneMatch "*" on create, IfMatch on replace |
| +// or delete) and none of them starts a renewer: the move's sealer re-asserts |
| +// the tombstone itself on every pass. |
| +// |
| +// - Tombstone writes it. It never replaces a live holder's lease. |
| +// - Reassert pushes the expiry out again, only on a lease the same move |
| +// owns, with the ETag it just read. |
| +// - LiftTombstone lifts it, only when the owner is a moved:<id> lease the |
| +// caller claims as its own, with the ETag it just read. It lifts by a |
| +// conditional PUT that rewrites the lease as already expired, never by a |
| +// DELETE: Ceph RGW and MinIO are not known to enforce If-Match on a |
| +// DELETE, and the conditional PUT is the primitive the rest of this |
| +// package relies on. IsLiftedTombstone reads that state back. |
| +// |
| +// "Free" is exactly what Acquire means by it (leaseFree): the lease's expiry |
| +// has passed. There is no RenewMargin here. A tombstone that is free to a |
| +// wake is free to the next Tombstone, and the two cannot drift apart. |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "strings" |
| + "time" |
| + |
| + "github.com/aws/aws-sdk-go-v2/aws" |
| + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| + "github.com/benbjohnson/litestream" |
| + lss3 "github.com/benbjohnson/litestream/s3" |
| +) |
| + |
| +// TombstonePrefix starts the owner of every tombstone lease. The three |
| +// functions here refuse any other owner. |
| +const TombstonePrefix = "moved:" |
| + |
| +// MaxTombstoneTTL is the longest a tombstone is ever written for. The moves |
| +// package's TombstoneTTL equals it; this package cannot import that one. |
| +const MaxTombstoneTTL = 7 * 24 * time.Hour |
| + |
| +var ( |
| + // ErrTombstoneOwner refuses an owner that is not moved:<id>. |
| + ErrTombstoneOwner = errors.New("lease: a tombstone owner is moved:<id>") |
| + // ErrTombstoneTTL refuses a TTL that is not positive or exceeds |
| + // MaxTombstoneTTL. |
| + ErrTombstoneTTL = errors.New("lease: a tombstone lives for at most 7 days") |
| + // ErrTombstoneChanged reports that the lease was rewritten or removed |
| + // between the read and the conditional write. Nothing was overwritten; |
| + // the caller's next pass reads it afresh. |
| + ErrTombstoneChanged = errors.New("lease: the lock changed under the tombstone write") |
| +) |
| + |
| +// leaseFree is the predicate litestream's AcquireLease applies to the lease |
| +// it reads (Lease.IsExpired: now is after the expiry), which is what decides |
| +// whether lease.Acquire takes a lock over. Tombstone, Reassert and |
| +// IsLiftedTombstone use it so a lease is free to them exactly when it is free |
| +// to a wake. TestTombstoneAndAcquireAgreeOnFree pins the two together. |
| +func leaseFree(expires, now time.Time) bool { return now.After(expires) } |
| + |
| +// liftedExpiry is the expiry LiftTombstone writes: long past, so every |
| +// reading (leaseFree, and LockState's RenewMargin) agrees the lease is free. |
| +var liftedExpiry = time.Unix(0, 0).UTC() |
| + |
| +// IsLiftedTombstone reports whether a lock read as owner and expires (from |
| +// LockState) is a lifted tombstone: a moved:<id> lease that has expired. That |
| +// is what LiftTombstone leaves behind, and it is free to every Acquire (which |
| +// already treats any expired lease as free) and to the next Tombstone. It is |
| +// false for an ordinary process's lease, expired or not, and for a tombstone |
| +// that is still unexpired. An absent lock is not "lifted" here: callers read |
| +// absence from LockState's zero owner and expiry and treat it the same way. |
| +func IsLiftedTombstone(owner string, expires time.Time) bool { |
| + return tombstoneOwnerOK(owner) && !expires.IsZero() && leaseFree(expires, time.Now()) |
| +} |
| + |
| +func tombstoneOwnerOK(owner string) bool { |
| + return strings.HasPrefix(owner, TombstonePrefix) && len(owner) > len(TombstonePrefix) |
| +} |
| + |
| +func tombstoneTTLOK(ttl time.Duration) bool { return ttl > 0 && ttl <= MaxTombstoneTTL } |
| + |
| +// tombstoneLeaser is the leaser a tombstone call writes with, built over the |
| +// client AcquireTTL builds, and the lock's key. ttl is zero for calls that |
| +// write no expiry (LiftTombstone). |
| +func tombstoneLeaser(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) (*lss3.Leaser, string, error) { |
| + key, err := lockKey(bucket, location) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + client, _, err := newLeaserClient(ctx, region) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + leaser := lss3.NewLeaser() |
| + leaser.SetClient(client) |
| + leaser.Bucket = bucket |
| + leaser.Path = strings.TrimSuffix(location, "/") |
| + leaser.TTL = ttl |
| + leaser.Owner = owner |
| + return leaser, key, nil |
| +} |
| + |
| +// readLock reads the lock with the ETag it was read at. Only S3's explicit |
| +// NoSuchKey is absence (lockAbsent); a garbled body, an oversized one or a |
| +// lock with no expiry is an error, never "free". |
| +func readLock(ctx context.Context, leaser *lss3.Leaser, key string) (lock *litestream.Lease, absent bool, err error) { |
| + out, err := leaser.Client().GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(leaser.Bucket), Key: aws.String(key)}) |
| + if err != nil { |
| + if lockAbsent(err) { |
| + return nil, true, nil |
| + } |
| + return nil, false, fmt.Errorf("lease: read %s: %w", key, err) |
| + } |
| + defer out.Body.Close() |
| + body, err := io.ReadAll(io.LimitReader(out.Body, lockBodyLimit+1)) |
| + if err != nil { |
| + return nil, false, fmt.Errorf("lease: read %s: %w", key, err) |
| + } |
| + if len(body) > lockBodyLimit { |
| + return nil, false, fmt.Errorf("lease: %s is larger than a lock", key) |
| + } |
| + var l litestream.Lease |
| + if err := json.Unmarshal(body, &l); err != nil { |
| + return nil, false, fmt.Errorf("lease: decode %s: %w", key, err) |
| + } |
| + if l.ExpiresAt.IsZero() { |
| + return nil, false, fmt.Errorf("lease: %s has no expiry", key) |
| + } |
| + if out.ETag == nil || *out.ETag == "" { |
| + return nil, false, fmt.Errorf("lease: %s came back without an ETag", key) |
| + } |
| + l.ETag = *out.ETag |
| + return &l, false, nil |
| +} |
| + |
| +func heldErr(l *litestream.Lease) error { |
| + return fmt.Errorf("acquire lease: %w", &HeldError{Owner: l.Owner, ExpiresAt: l.ExpiresAt}) |
| +} |
| + |
| +// Tombstone writes the lease at s3://bucket/<location>/lock.json owned by |
| +// owner (moved:<id>), expiring ttl from now (at most MaxTombstoneTTL). |
| +// |
| +// - absent, or expired (as Acquire reads it): created (IfNoneMatch "*") or |
| +// replaced (IfMatch on the expired lease's ETag); |
| +// - live and held by owner: nil and no write; the caller re-asserts with |
| +// Reassert; |
| +// - live and held by anyone else: a *HeldError, nothing written; |
| +// - the lock changed under the write and cannot be read back: |
| +// ErrTombstoneChanged. |
| +// |
| +// A live holder is never replaced: the replace is conditional on the ETag of |
| +// the lease that was read as expired, so a holder that renews in between |
| +// wins and Tombstone reports it as held. |
| +func Tombstone(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) error { |
| + if !tombstoneOwnerOK(owner) { |
| + return ErrTombstoneOwner |
| + } |
| + if !tombstoneTTLOK(ttl) { |
| + return ErrTombstoneTTL |
| + } |
| + leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, owner, ttl) |
| + if err != nil { |
| + return err |
| + } |
| + cur, absent, err := readLock(ctx, leaser, key) |
| + if err != nil { |
| + return err |
| + } |
| + if !absent && !leaseFree(cur.ExpiresAt, time.Now()) { |
| + if cur.Owner == owner { |
| + return nil |
| + } |
| + return heldErr(cur) |
| + } |
| + // Absent or expired. AcquireLease reads again and writes conditionally: |
| + // IfNoneMatch "*" when absent, IfMatch on the ETag it reads otherwise. |
| + // If a holder appeared since our read it answers LeaseExistsError. |
| + if _, err := leaser.AcquireLease(ctx); err != nil { |
| + var lexists *litestream.LeaseExistsError |
| + if errors.As(err, &lexists) { |
| + switch lexists.Owner { |
| + case "": |
| + // The write lost its condition and AcquireLease could not |
| + // read the winner back (it was removed again, or the read |
| + // failed). There is no holder to name. |
| + return fmt.Errorf("tombstone lease: %w", ErrTombstoneChanged) |
| + case owner: |
| + return nil |
| + } |
| + return fmt.Errorf("acquire lease: %w", &HeldError{Owner: lexists.Owner, ExpiresAt: lexists.ExpiresAt}) |
| + } |
| + return fmt.Errorf("tombstone lease: %w", err) |
| + } |
| + return nil |
| +} |
| + |
| +// Reassert moves the expiry of a tombstone owner already holds to now+ttl: |
| +// |
| +// - live and held by owner: rewritten with IfMatch on the ETag just read. |
| +// If the lock changed since, ErrTombstoneChanged and nothing is written; |
| +// - absent, or expired (including lifted): Tombstone, which writes a new |
| +// one; |
| +// - live and held by anyone else: a *HeldError, nothing written. |
| +// |
| +// Callers must read LockState first and treat absent or expired as a lost |
| +// stop; never Reassert after Lift. Reassert cannot tell a tombstone that was |
| +// kept from one that lapsed and was re-created, and in between a wake may |
| +// have acquired, written and released the lease. |
| +func Reassert(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) error { |
| + if !tombstoneOwnerOK(owner) { |
| + return ErrTombstoneOwner |
| + } |
| + if !tombstoneTTLOK(ttl) { |
| + return ErrTombstoneTTL |
| + } |
| + leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, owner, ttl) |
| + if err != nil { |
| + return err |
| + } |
| + cur, absent, err := readLock(ctx, leaser, key) |
| + if err != nil { |
| + return err |
| + } |
| + if absent || leaseFree(cur.ExpiresAt, time.Now()) { |
| + return Tombstone(ctx, bucket, location, region, owner, ttl) |
| + } |
| + if cur.Owner != owner { |
| + return heldErr(cur) |
| + } |
| + if _, err := leaser.RenewLease(ctx, cur); err != nil { |
| + if errors.Is(err, litestream.ErrLeaseNotHeld) { |
| + return fmt.Errorf("reassert tombstone: %w", ErrTombstoneChanged) |
| + } |
| + return fmt.Errorf("reassert tombstone: %w", err) |
| + } |
| + return nil |
| +} |
| + |
| +// LiftTombstone lifts the tombstone at s3://bucket/<location>/lock.json by |
| +// rewriting it as an expired lease of the same owner and generation, with |
| +// IfMatch on the ETag just read. It never deletes the object. |
| +// |
| +// - absent: (false, nil), and mine is not consulted; |
| +// - owner is not a moved:<id> that mine accepts: (false, *HeldError), |
| +// nothing written. This holds for an expired lease too: only the move |
| +// that wrote a tombstone lifts it; |
| +// - mine and already lifted (expired, IsLiftedTombstone): (false, nil), |
| +// nothing written; |
| +// - mine and live: rewritten, (true, nil). Acquire takes the lease from |
| +// here, as it takes any expired lease, and so does Tombstone; |
| +// - the lock was rewritten since the read: (false, ErrTombstoneChanged), |
| +// and the rewrite stands. One that was removed since is (false, nil). |
| +// |
| +// After a lift, LockState reports held=false with the move's owner and a |
| +// past expiry; IsLiftedTombstone recognises that. Never Reassert after it. |
| +func LiftTombstone(ctx context.Context, bucket, location, region string, mine func(owner string) bool) (lifted bool, err error) { |
| + leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, "", 0) |
| + if err != nil { |
| + return false, err |
| + } |
| + cur, absent, err := readLock(ctx, leaser, key) |
| + if err != nil { |
| + return false, err |
| + } |
| + if absent { |
| + return false, nil |
| + } |
| + if !tombstoneOwnerOK(cur.Owner) || mine == nil || !mine(cur.Owner) { |
| + return false, heldErr(cur) |
| + } |
| + if IsLiftedTombstone(cur.Owner, cur.ExpiresAt) { |
| + return false, nil |
| + } |
| + body, err := json.Marshal(&litestream.Lease{Generation: cur.Generation, ExpiresAt: liftedExpiry, Owner: cur.Owner}) |
| + if err != nil { |
| + return false, fmt.Errorf("lift tombstone: %w", err) |
| + } |
| + _, err = leaser.Client().PutObject(ctx, &awss3.PutObjectInput{ |
| + Bucket: aws.String(leaser.Bucket), |
| + Key: aws.String(key), |
| + Body: bytes.NewReader(body), |
| + ContentType: aws.String("application/json"), |
| + IfMatch: aws.String(cur.ETag), |
| + }) |
| + if err == nil { |
| + return true, nil |
| + } |
| + if code, status := apiCode(err); code == "PreconditionFailed" || status == http.StatusPreconditionFailed || code == "NoSuchKey" || status == http.StatusNotFound { |
| + // Rewritten, or gone (AWS answers a conditional PUT on a missing key |
| + // 404, some stores 412). Read it back to tell which. |
| + if _, gone, rerr := readLock(ctx, leaser, key); rerr == nil && gone { |
| + return false, nil |
| + } |
| + return false, fmt.Errorf("lift tombstone: %w", ErrTombstoneChanged) |
| + } |
| + return false, fmt.Errorf("lift tombstone: %w", err) |
| +} |
diff --git a/internal/lease/tombstone_test.go b/internal/lease/tombstone_test.go| new file mode 100644 |
| index 000000000..9dc522871 |
| --- /dev/null |
| +++ b/internal/lease/tombstone_test.go |
| @@ -0,0 +1,784 @@ |
| +package lease_test |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/lease/leasetest" |
| +) |
| + |
| +const ( |
| + tombLoc = "instances/h.example.com" |
| + tombMove = "moved:mv_01AAAAAAAAAAAAAAAAAAAAAAAA" |
| + tombOther = "moved:mv_01BBBBBBBBBBBBBBBBBBBBBBBB" |
| + tombRegion = "test-region" |
| +) |
| + |
| +// tombStore is leasetest's S3 server behind a wrapper that runs a hook in the |
| +// window between the code under test's read of lock.json and its conditional |
| +// write, so a race is a request ordering, not a sleep. The conditions |
| +// themselves (If-Match, If-None-Match) are leasetest's own. |
| +type tombStore struct { |
| + *leasetest.VersionedBucket |
| + mu sync.Mutex |
| + before func(method string) |
| + writes []string // "PUT" and "DELETE" requests that reached lock.json, in order |
| +} |
| + |
| +// beforeWrite arms hook for the next write to lock.json, once. |
| +func (s *tombStore) beforeWrite(hook func(method string)) { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + s.before = hook |
| +} |
| + |
| +func tombFixture(t *testing.T) *tombStore { |
| + t.Helper() |
| + bucket := leasetest.NewVersionedBucket() |
| + bucket.AddBucket("data") |
| + inner, _ := leasetest.NewS3Server(bucket) |
| + s := &tombStore{VersionedBucket: bucket} |
| + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + key := strings.TrimPrefix(r.URL.Path, "/data/") |
| + if strings.HasSuffix(key, "lock.json") && (r.Method == http.MethodPut || r.Method == http.MethodDelete) { |
| + s.mu.Lock() |
| + hook := s.before |
| + s.before = nil |
| + s.writes = append(s.writes, r.Method) |
| + s.mu.Unlock() |
| + if hook != nil { |
| + hook(r.Method) |
| + } |
| + } |
| + inner.ServeHTTP(w, r) |
| + })) |
| + t.Cleanup(srv.Close) |
| + t.Setenv("AWS_ACCESS_KEY_ID", "test-key") |
| + t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") |
| + t.Setenv("AWS_SESSION_TOKEN", "") |
| + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") |
| + t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent") |
| + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent") |
| + t.Setenv("CARLOS_S3_ENDPOINT", srv.URL) |
| + return s |
| +} |
| + |
| +func (s *tombStore) methods() []string { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return append([]string(nil), s.writes...) |
| +} |
| + |
| +func (s *tombStore) writeCount() int { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return len(s.writes) |
| +} |
| + |
| +type tombLock struct { |
| + Generation int64 `json:"generation"` |
| + ExpiresAt time.Time `json:"expires_at"` |
| + Owner string `json:"owner"` |
| +} |
| + |
| +func (s *tombStore) putLock(t *testing.T, loc, owner string, gen int64, expires time.Time) { |
| + t.Helper() |
| + body, err := json.Marshal(tombLock{Generation: gen, ExpiresAt: expires.UTC(), Owner: owner}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s.Put("data", loc+"/lock.json", body) |
| +} |
| + |
| +func (s *tombStore) lock(t *testing.T, loc string) (tombLock, []byte, bool) { |
| + t.Helper() |
| + body, ok := s.Current("data", loc+"/lock.json") |
| + if !ok { |
| + return tombLock{}, nil, false |
| + } |
| + var l tombLock |
| + if err := json.Unmarshal(body, &l); err != nil { |
| + t.Fatalf("lock.json is not a lock: %v", err) |
| + } |
| + return l, body, true |
| +} |
| + |
| +func wantHeld(t *testing.T, err error, owner string) { |
| + t.Helper() |
| + var held *lease.HeldError |
| + if !errors.As(err, &held) { |
| + t.Fatalf("err = %v, want a *HeldError", err) |
| + } |
| + if held.Owner != owner { |
| + t.Fatalf("held by %q, want %q", held.Owner, owner) |
| + } |
| +} |
| + |
| +func near(t *testing.T, got time.Time, want time.Duration) { |
| + t.Helper() |
| + if d := time.Until(got) - want; d < -time.Minute || d > time.Minute { |
| + t.Fatalf("expiry %s is %s from now, want about %s", got, time.Until(got), want) |
| + } |
| +} |
| + |
| +func isMine(owner string) func(string) bool { |
| + return func(o string) bool { return o == owner } |
| +} |
| + |
| +// A tombstone is a lease like any other: Acquire on the host's location |
| +// fails with a HeldError that names the move. |
| +func TestTombstoneBlocksAcquire(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 2*time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + l, _, ok := s.lock(t, tombLoc) |
| + if !ok || l.Owner != tombMove { |
| + t.Fatalf("lock = %+v present %v, want owner %s", l, ok, tombMove) |
| + } |
| + near(t, l.ExpiresAt, 2*time.Hour) |
| + _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion) |
| + wantHeld(t, err, tombMove) |
| + var held *lease.HeldError |
| + errors.As(err, &held) |
| + near(t, held.ExpiresAt, 2*time.Hour) |
| + // The trailing-slash form ReplicaSource hands over reads the same lock. |
| + got, owner, _, err := lease.LockState(ctx, "data", tombLoc+"/", tombRegion) |
| + if err != nil || !got || owner != tombMove { |
| + t.Fatalf("LockState = held %v owner %q err %v", got, owner, err) |
| + } |
| +} |
| + |
| +func TestTombstoneNeverReplacesALiveHolder(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { _ = held.Release(context.Background()) }) |
| + _, before, _ := s.lock(t, tombLoc) |
| + writes := s.writeCount() |
| + err = lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour) |
| + var he *lease.HeldError |
| + if !errors.As(err, &he) || strings.HasPrefix(he.Owner, "moved:") { |
| + t.Fatalf("Tombstone over a live holder = %v, want a *HeldError naming the holder", err) |
| + } |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatalf("lock.json changed: %s -> %s", before, after) |
| + } |
| + if s.writeCount() != writes { |
| + t.Fatal("Tombstone wrote to a live holder's lock") |
| + } |
| + if err := held.Err(); err != nil { |
| + t.Fatalf("the holder lost its lease: %v", err) |
| + } |
| + // Reassert is no way round it either. |
| + err = lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour) |
| + if !errors.As(err, &he) { |
| + t.Fatalf("Reassert over a live holder = %v, want a *HeldError", err) |
| + } |
| + if s.writeCount() != writes { |
| + t.Fatal("Reassert wrote to a live holder's lock") |
| + } |
| +} |
| + |
| +// Free means what Acquire means by it: the expiry has passed. There is no |
| +// RenewMargin grace, so a lease that lapsed a moment ago is taken by Tombstone |
| +// exactly as lease.Acquire would take it. The two are run side by side on |
| +// identical locks so they cannot drift. |
| +func TestTombstoneAndAcquireAgreeOnFree(t *testing.T) { |
| + ctx := context.Background() |
| + for name, tc := range map[string]struct { |
| + expires time.Time |
| + free bool |
| + }{ |
| + "expired a second ago": {time.Now().Add(-time.Second), true}, |
| + "expired within the margin": {time.Now().Add(-lease.RenewMargin / 2), true}, |
| + "expires in a minute": {time.Now().Add(time.Minute), false}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, "box-a:1", 3, tc.expires) |
| + _, before, _ := s.lock(t, tombLoc) |
| + acq, acqErr := lease.Acquire(ctx, "data", tombLoc, tombRegion) |
| + if acqErr == nil { |
| + _ = acq.Release(ctx) |
| + } |
| + // Put the same lock back for Tombstone. |
| + s.putLock(t, tombLoc, "box-a:1", 3, tc.expires) |
| + tombErr := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour) |
| + if (acqErr == nil) != tc.free || (tombErr == nil) != tc.free { |
| + t.Fatalf("free %v: Acquire err %v, Tombstone err %v", tc.free, acqErr, tombErr) |
| + } |
| + if !tc.free { |
| + wantHeld(t, tombErr, "box-a:1") |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatal("a live lease was replaced") |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// An own tombstone that has expired is free like any other: Tombstone writes |
| +// a fresh one rather than reporting the dead one held. |
| +func TestTombstoneRewritesItsOwnExpiredTombstone(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-time.Second)) |
| + if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + l, _, _ := s.lock(t, tombLoc) |
| + if l.Owner != tombMove || l.Generation != 5 { |
| + t.Fatalf("lock = %+v, want owner %s at generation 5", l, tombMove) |
| + } |
| + near(t, l.ExpiresAt, time.Hour) |
| +} |
| + |
| +// The write loses its condition and AcquireLease cannot read a winner back: |
| +// there is no holder to name, so the answer is ErrTombstoneChanged, never a |
| +// HeldError with an empty owner. |
| +func TestTombstoneWriteLostThenReadLostIsChanged(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) |
| + s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") }) |
| + err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour) |
| + var he *lease.HeldError |
| + if !errors.Is(err, lease.ErrTombstoneChanged) || errors.As(err, &he) { |
| + t.Fatalf("err = %v, want ErrTombstoneChanged and no HeldError", err) |
| + } |
| + if _, _, ok := s.lock(t, tombLoc); ok { |
| + t.Fatal("Tombstone wrote over the vanished lock") |
| + } |
| +} |
| + |
| +// The write loses its condition and the read-back shows this very owner |
| +// (another pass of the same move got there first): nothing to do. |
| +func TestTombstoneWriteLostToItsOwnOwnerIsNil(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) |
| + var theirs []byte |
| + s.beforeWrite(func(string) { |
| + s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(2*time.Hour)) |
| + _, theirs, _ = s.lock(t, tombLoc) |
| + }) |
| + if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatalf("err = %v, want nil", err) |
| + } |
| + if _, now, _ := s.lock(t, tombLoc); string(now) != string(theirs) { |
| + t.Fatalf("the winner's lock was overwritten: %s -> %s", theirs, now) |
| + } |
| +} |
| + |
| +func TestTombstoneReplacesAnExpiredLease(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) |
| + if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + l, _, _ := s.lock(t, tombLoc) |
| + if l.Owner != tombMove || l.Generation != 4 { |
| + t.Fatalf("lock = %+v, want owner %s at generation 4", l, tombMove) |
| + } |
| + near(t, l.ExpiresAt, time.Hour) |
| +} |
| + |
| +func TestTombstoneByTheSameOwnerIsANoOp(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + _, first, _ := s.lock(t, tombLoc) |
| + writes := s.writeCount() |
| + // A longer ttl on the repeat changes nothing: Reassert is the call that moves it. |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, second, _ := s.lock(t, tombLoc); string(second) != string(first) || s.writeCount() != writes { |
| + t.Fatalf("the repeat wrote: %s -> %s", first, second) |
| + } |
| +} |
| + |
| +func TestTombstoneRefusesAnotherMove(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + _, before, _ := s.lock(t, tombLoc) |
| + wantHeld(t, lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove) |
| + wantHeld(t, lease.Reassert(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove) |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatalf("another move's tombstone was overwritten: %s -> %s", before, after) |
| + } |
| +} |
| + |
| +func TestTombstoneRefusesWhatItShouldNotWrite(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + for _, owner := range []string{"", "box-a:1", "moved:", "move:mv_1", "Moved:mv_1"} { |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) { |
| + t.Fatalf("Tombstone owner %q = %v, want ErrTombstoneOwner", owner, err) |
| + } |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) { |
| + t.Fatalf("Reassert owner %q = %v, want ErrTombstoneOwner", owner, err) |
| + } |
| + } |
| + for _, ttl := range []time.Duration{0, -time.Hour, lease.MaxTombstoneTTL + time.Second, 30 * 24 * time.Hour} { |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) { |
| + t.Fatalf("Tombstone ttl %s = %v, want ErrTombstoneTTL", ttl, err) |
| + } |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) { |
| + t.Fatalf("Reassert ttl %s = %v, want ErrTombstoneTTL", ttl, err) |
| + } |
| + } |
| + for _, loc := range []string{"", "/abs", "../x", "a/../b", `a\b`} { |
| + if err := lease.Tombstone(ctx, "data", loc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrLockLocation) { |
| + t.Fatalf("Tombstone location %q = %v, want ErrLockLocation", loc, err) |
| + } |
| + } |
| + if s.writeCount() != 0 { |
| + t.Fatal("a refused call wrote") |
| + } |
| + // Seven days exactly is the longest there is. |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, lease.MaxTombstoneTTL); err != nil { |
| + t.Fatal(err) |
| + } |
| + l, _, _ := s.lock(t, tombLoc) |
| + near(t, l.ExpiresAt, lease.MaxTombstoneTTL) |
| + if time.Until(l.ExpiresAt) > lease.MaxTombstoneTTL { |
| + t.Fatalf("expiry %s is beyond 7 days", l.ExpiresAt) |
| + } |
| +} |
| + |
| +func TestReassertMovesTheExpiry(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + first, _, _ := s.lock(t, tombLoc) |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + second, _, _ := s.lock(t, tombLoc) |
| + if second.Owner != tombMove || second.Generation != first.Generation { |
| + t.Fatalf("lock = %+v, want the same owner and generation as %+v", second, first) |
| + } |
| + near(t, second.ExpiresAt, 48*time.Hour) |
| + // And it can be pulled in again: the expiry is what the caller says. |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + third, _, _ := s.lock(t, tombLoc) |
| + near(t, third.ExpiresAt, time.Hour) |
| + _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion) |
| + wantHeld(t, err, tombMove) |
| +} |
| + |
| +func TestReassertWithNothingToReassertTombstones(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatalf("absent: %v", err) |
| + } |
| + if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove { |
| + t.Fatalf("absent: lock = %+v present %v", l, ok) |
| + } |
| + // An expired tombstone of its own is replaced and live again. |
| + s.putLock(t, tombLoc, tombMove, 5, time.Now().Add(-time.Hour)) |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatalf("expired: %v", err) |
| + } |
| + l, _, _ := s.lock(t, tombLoc) |
| + if l.Owner != tombMove || l.Generation != 6 { |
| + t.Fatalf("expired: lock = %+v, want owner %s at generation 6", l, tombMove) |
| + } |
| + near(t, l.ExpiresAt, time.Hour) |
| + // An expired lease of someone else is replaced the same way. |
| + s.putLock(t, tombLoc, tombOther, 2, time.Now().Add(-time.Hour)) |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatalf("expired other: %v", err) |
| + } |
| + if l, _, _ := s.lock(t, tombLoc); l.Owner != tombMove { |
| + t.Fatalf("expired other: lock = %+v", l) |
| + } |
| +} |
| + |
| +func TestReassertRefusesAHolderThatIsNotTheMove(t *testing.T) { |
| + s := tombFixture(t) |
| + s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(time.Hour)) |
| + _, before, _ := s.lock(t, tombLoc) |
| + wantHeld(t, lease.Reassert(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour), "box-a:1") |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatal("Reassert overwrote a live holder") |
| + } |
| +} |
| + |
| +// The lock is rewritten between Reassert's read and its write: the write |
| +// carries the stale ETag, S3 refuses it, and the rewrite stands. |
| +func TestReassertFailsWhenTheLockChangesUnderIt(t *testing.T) { |
| + for name, rewrite := range map[string]func(s *tombStore, t *testing.T){ |
| + "another move takes it": func(s *tombStore, t *testing.T) { |
| + s.putLock(t, tombLoc, tombOther, 9, time.Now().Add(3*time.Hour)) |
| + }, |
| + "the same move rewrites it": func(s *tombStore, t *testing.T) { |
| + s.putLock(t, tombLoc, tombMove, 9, time.Now().Add(3*time.Hour)) |
| + }, |
| + "it is removed": func(s *tombStore, t *testing.T) { |
| + s.BareDelete("data", tombLoc+"/lock.json") |
| + }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + var after []byte |
| + var present bool |
| + s.beforeWrite(func(method string) { |
| + if method != http.MethodPut { |
| + t.Errorf("Reassert sent a %s", method) |
| + } |
| + rewrite(s, t) |
| + _, after, present = s.lock(t, tombLoc) |
| + }) |
| + err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour) |
| + if !errors.Is(err, lease.ErrTombstoneChanged) { |
| + t.Fatalf("err = %v, want ErrTombstoneChanged", err) |
| + } |
| + _, now, ok := s.lock(t, tombLoc) |
| + if ok != present || string(now) != string(after) { |
| + t.Fatalf("Reassert overwrote the rewrite: %s -> %s", after, now) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Tombstone's own windows: the lock is absent or expired when it reads, and a |
| +// live holder is there when it writes. |
| +func TestTombstoneLosesToAHolderThatArrivesFirst(t *testing.T) { |
| + for name, setup := range map[string]func(s *tombStore, t *testing.T){ |
| + "absent": func(s *tombStore, t *testing.T) {}, |
| + "expired": func(s *tombStore, t *testing.T) { s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + s := tombFixture(t) |
| + setup(s, t) |
| + var holder []byte |
| + s.beforeWrite(func(string) { |
| + s.putLock(t, tombLoc, "box-b:2", 7, time.Now().Add(15*time.Minute)) |
| + _, holder, _ = s.lock(t, tombLoc) |
| + }) |
| + err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour) |
| + wantHeld(t, err, "box-b:2") |
| + if _, now, _ := s.lock(t, tombLoc); string(now) != string(holder) { |
| + t.Fatalf("Tombstone replaced the holder that arrived first: %s -> %s", holder, now) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestLiftTombstone(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + called := "" |
| + mine := func(o string) bool { called = o; return o == tombMove } |
| + |
| + // Absent: nothing to lift, and mine is not consulted. |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine); lifted || err != nil || called != "" { |
| + t.Fatalf("absent: lifted %v err %v consulted %q", lifted, err, called) |
| + } |
| + |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + live, before, _ := s.lock(t, tombLoc) |
| + |
| + // Another move's tombstone is not ours to lift. |
| + lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombOther)) |
| + wantHeld(t, err, tombMove) |
| + if lifted { |
| + t.Fatal("lifted another move's tombstone") |
| + } |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatal("the tombstone changed") |
| + } |
| + // No ownership test at all lifts nothing. |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, nil); lifted || err == nil { |
| + t.Fatalf("nil mine: lifted %v err %v", lifted, err) |
| + } |
| + |
| + // Ours lifts: by a conditional PUT, leaving the same owner and generation |
| + // with a long-past expiry, never by a DELETE. |
| + writes := s.writeCount() |
| + lifted, err = lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine) |
| + if err != nil || !lifted || called != tombMove { |
| + t.Fatalf("lift: lifted %v err %v consulted %q", lifted, err, called) |
| + } |
| + if got := s.methods()[writes:]; len(got) != 1 || got[0] != http.MethodPut { |
| + t.Fatalf("lift wrote %v, want one PUT", got) |
| + } |
| + l, _, ok := s.lock(t, tombLoc) |
| + if !ok || l.Owner != tombMove || l.Generation != live.Generation || !l.ExpiresAt.Before(time.Now().Add(-lease.RenewMargin)) { |
| + t.Fatalf("after the lift: %+v present %v, want %s at generation %d, long expired", l, ok, tombMove, live.Generation) |
| + } |
| + if held, owner, expires, err := lease.LockState(ctx, "data", tombLoc, tombRegion); err != nil || held || !lease.IsLiftedTombstone(owner, expires) { |
| + t.Fatalf("LockState after the lift: held %v owner %q expires %v err %v", held, owner, expires, err) |
| + } |
| + // The host can be acquired again, and so can a new tombstone. |
| + held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion) |
| + if err != nil { |
| + t.Fatalf("Acquire after the lift: %v", err) |
| + } |
| + if err := held.Release(ctx); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func TestLiftedTombstoneIsFreeToTheNextMove(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted { |
| + t.Fatalf("lifted %v err %v", lifted, err) |
| + } |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour); err != nil { |
| + t.Fatalf("Tombstone over a lifted one: %v", err) |
| + } |
| + if l, _, _ := s.lock(t, tombLoc); l.Owner != tombOther { |
| + t.Fatalf("lock = %+v", l) |
| + } |
| +} |
| + |
| +// Already lifted, however it came about, is (false, nil) with nothing written. |
| +func TestLiftTombstoneAlreadyLifted(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted { |
| + t.Fatalf("first lift: %v %v", lifted, err) |
| + } |
| + writes := s.writeCount() |
| + for i := 0; i < 2; i++ { |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil { |
| + t.Fatalf("already lifted: lifted %v err %v", lifted, err) |
| + } |
| + } |
| + // An own tombstone that simply ran out is lifted too. |
| + s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-48*time.Hour)) |
| + writes = s.writeCount() |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil { |
| + t.Fatalf("expired own: lifted %v err %v", lifted, err) |
| + } |
| + if s.writeCount() != writes { |
| + t.Fatal("lifting an already-lifted tombstone wrote") |
| + } |
| + // And so is an absent one. |
| + s.BareDelete("data", tombLoc+"/lock.json") |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil { |
| + t.Fatalf("absent: lifted %v err %v", lifted, err) |
| + } |
| +} |
| + |
| +func TestIsLiftedTombstone(t *testing.T) { |
| + past, future := time.Now().Add(-time.Second), time.Now().Add(time.Hour) |
| + for name, tc := range map[string]struct { |
| + owner string |
| + expires time.Time |
| + want bool |
| + }{ |
| + "expired move": {tombMove, past, true}, |
| + "long expired move": {tombMove, time.Unix(0, 0), true}, |
| + "live move": {tombMove, future, false}, |
| + "expired ordinary lease": {"box-a:1", past, false}, |
| + "live ordinary lease": {"box-a:1", future, false}, |
| + "absent (zero owner/exp)": {"", time.Time{}, false}, |
| + "bare prefix": {"moved:", past, false}, |
| + "zero expiry": {tombMove, time.Time{}, false}, |
| + } { |
| + if got := lease.IsLiftedTombstone(tc.owner, tc.expires); got != tc.want { |
| + t.Errorf("%s: IsLiftedTombstone(%q, %v) = %v, want %v", name, tc.owner, tc.expires, got, tc.want) |
| + } |
| + } |
| +} |
| + |
| +// Whatever mine says, only a moved:<id> lease is lifted: a live process's |
| +// lease is never this function's to touch. |
| +func TestLiftTombstoneLeavesOrdinaryLeasesAlone(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + for name, expires := range map[string]time.Time{"live": time.Now().Add(time.Hour), "expired": time.Now().Add(-time.Hour)} { |
| + s.putLock(t, tombLoc, "box-a:1", 3, expires) |
| + _, before, _ := s.lock(t, tombLoc) |
| + lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, func(string) bool { return true }) |
| + wantHeld(t, err, "box-a:1") |
| + if lifted { |
| + t.Fatalf("%s: lifted an ordinary lease", name) |
| + } |
| + if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) { |
| + t.Fatalf("%s: the lease changed", name) |
| + } |
| + } |
| + if s.writeCount() != 0 { // putLock writes the bucket directly; only requests count |
| + t.Fatal("a refused lift wrote") |
| + } |
| +} |
| + |
| +func TestLiftTombstoneFailsWhenTheLockChangesUnderIt(t *testing.T) { |
| + ctx := context.Background() |
| + t.Run("rewritten", func(t *testing.T) { |
| + s := tombFixture(t) |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + var after []byte |
| + s.beforeWrite(func(method string) { |
| + if method != http.MethodPut { |
| + t.Errorf("LiftTombstone sent a %s", method) |
| + } |
| + // Still ours by owner, but a different lock: Reassert beat the lift. |
| + s.putLock(t, tombLoc, tombMove, 8, time.Now().Add(72*time.Hour)) |
| + _, after, _ = s.lock(t, tombLoc) |
| + }) |
| + lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)) |
| + if lifted || !errors.Is(err, lease.ErrTombstoneChanged) { |
| + t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err) |
| + } |
| + if _, now, ok := s.lock(t, tombLoc); !ok || string(now) != string(after) { |
| + t.Fatalf("the lift overwrote the rewrite: %s", now) |
| + } |
| + }) |
| + t.Run("taken over by a holder", func(t *testing.T) { |
| + s := tombFixture(t) |
| + // Live tombstone read; before the write a holder cannot legitimately |
| + // appear, but a rewrite by anything must still stand. |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + var after []byte |
| + s.beforeWrite(func(string) { |
| + s.putLock(t, tombLoc, "box-b:2", 5, time.Now().Add(15*time.Minute)) |
| + _, after, _ = s.lock(t, tombLoc) |
| + }) |
| + lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)) |
| + if lifted || !errors.Is(err, lease.ErrTombstoneChanged) { |
| + t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err) |
| + } |
| + if _, now, _ := s.lock(t, tombLoc); string(now) != string(after) { |
| + t.Fatalf("the lift overwrote a live holder's lock: %s", now) |
| + } |
| + }) |
| + t.Run("removed", func(t *testing.T) { |
| + s := tombFixture(t) |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") }) |
| + if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil { |
| + t.Fatalf("lifted %v err %v, want (false, nil): already gone", lifted, err) |
| + } |
| + if _, _, ok := s.lock(t, tombLoc); ok { |
| + t.Fatal("the lift re-created a removed lock") |
| + } |
| + }) |
| +} |
| + |
| +// The tombstone is a lock.json like any other, and MirrorPrefix, which copies |
| +// a host's replica to a staging prefix, leaves lock.json alone both ways. |
| +func TestMirrorPrefixStillSkipsTheTombstone(t *testing.T) { |
| + s := tombFixture(t) |
| + ctx := context.Background() |
| + s.Put("data", "instances/h.example.com/generations/a/snapshot", []byte("snap")) |
| + if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + // A staging lease of the target's own, under the destination prefix. |
| + s.putLock(t, "moves/h.example.com", "box-z:9", 1, time.Now().Add(time.Hour)) |
| + _, staged, _ := s.lock(t, "moves/h.example.com") |
| + |
| + m := bucketMirror{s.VersionedBucket} |
| + r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{}) |
| + if err != nil || r.Changed != 1 { |
| + t.Fatalf("round = %+v, %v; want only the snapshot to move", r, err) |
| + } |
| + if _, now, _ := s.lock(t, "moves/h.example.com"); string(now) != string(staged) { |
| + t.Fatal("the staging lease was touched") |
| + } |
| + if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove { |
| + t.Fatalf("the tombstone was touched: %+v", l) |
| + } |
| + // Reasserting the tombstone is churn the next round does not see. |
| + if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 24*time.Hour); err != nil { |
| + t.Fatal(err) |
| + } |
| + if r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{}); err != nil || r.Changed != 0 { |
| + t.Fatalf("caught-up round = %+v, %v; want 0 despite the reassert", r, err) |
| + } |
| +} |
| + |
| +// bucketMirror is a MirrorStore over the versioned bucket the S3 server also |
| +// serves, so MirrorPrefix runs over the very lock.json objects the tombstone |
| +// calls wrote (the S3 server does not speak ListObjectsV2). |
| +type bucketMirror struct{ b *leasetest.VersionedBucket } |
| + |
| +func (m bucketMirror) List(ctx context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) { |
| + vs, err := m.b.ListVersions(ctx, bucket, prefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + out := map[string]lease.MirrorObject{} |
| + for _, v := range vs { |
| + if v.IsLatest && !v.DeleteMarker { |
| + out[v.Key] = lease.MirrorObject{ETag: v.ETag, Size: v.Size} |
| + } |
| + } |
| + return out, nil |
| +} |
| + |
| +func (m bucketMirror) Copy(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) { |
| + vs, err := m.b.ListVersions(ctx, srcBucket, srcKey) |
| + if err != nil { |
| + return "", err |
| + } |
| + for _, v := range vs { |
| + if v.Key == srcKey && v.IsLatest && !v.DeleteMarker && v.ETag == srcETag { |
| + _, etag, err := m.b.CopyVersion(ctx, srcBucket, srcKey, v.VersionID, dstBucket, dstKey, false) |
| + return etag, err |
| + } |
| + } |
| + return "", lease.ErrMirrorNotFound |
| +} |
| + |
| +func (m bucketMirror) Delete(_ context.Context, bucket, key string) error { |
| + m.b.BareDelete(bucket, key) |
| + return nil |
| +} |
| + |
| +func (m bucketMirror) Get(_ context.Context, bucket, key string) ([]byte, error) { |
| + if body, ok := m.b.Current(bucket, key); ok { |
| + return body, nil |
| + } |
| + return nil, lease.ErrMirrorNotFound |
| +} |
| + |
| +func (m bucketMirror) Put(_ context.Context, bucket, key string, body []byte) error { |
| + m.b.Put(bucket, key, body) |
| + return nil |
| +} |