amadan
ratelimit: limiter with fair share under pressure and backstop buckets (#358)
Limiter.Admit takes a backstop token from the (host, rule, client) bucket, then sheds a client at its share while the host is busy, then admits, holding a slot unless holdSlot is false. observe lets refusals through with a slot. Bucket arithmetic is integer (billionths of a token per ns). Memory is bounded by maxKeys: only zero-holder entries are evicted, those with refilled buckets first, then least recently used, in batches of an eighth so the sweep is amortised. A map full of holders admits new clients untracked. Shares is the concurrency-only half, for the backhaul.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+861
−0
internal/ratelimit/limiter.go+271 −0internal/ratelimit/limiter_test.go+415 −0internal/ratelimit/shares.go+71 −0internal/ratelimit/shares_test.go+104 −0
CI failed — run details
CI log
=== gate ===
.amadan/test-gate.sh
go: downloading github.com/aws/aws-sdk-go-v2 v1.44.0
go: downloading github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5
go: downloading github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0
go: downloading github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0
go: downloading github.com/aws/smithy-go v1.28.1
go: downloading github.com/superfly/ltx v0.5.1
go: downloading golang.org/x/net v0.60.0
go: downloading modernc.org/sqlite v1.55.0
go: downloading github.com/aws/aws-sdk-go-v2/config v1.32.6
go: downloading github.com/aws/aws-sdk-go-v2/service/sns v1.43.0
go: downloading github.com/aws/aws-sdk-go-v2/service/autoscaling v1.74.0
go: downloading github.com/aws/aws-sdk-go-v2/service/ec2 v1.324.0
go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.19.6
go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.45.6
go: downloading github.com/aws/aws-sdk-go-v2/service/route53 v1.64.1
go: downloading golang.org/x/crypto v0.57.0
go: downloading github.com/oschwald/maxminddb-golang v1.13.1
go: downloading github.com/benbjohnson/litestream v0.5.14
go: downloading github.com/aws/aws-sdk-go-v2/service/acm v1.45.0
go: downloading github.com/aws/aws-sdk-go-v2/service/iam v1.58.2
go: downloading github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.37.0
go: downloading golang.org/x/sys v0.48.0
go: downloading github.com/pion/logging v0.2.4
go: downloading github.com/pion/stun/v3 v3.1.5
go: downloading github.com/pion/turn/v4 v4.1.4
go: downloading github.com/aws/aws-sdk-go-v2/service/sesv2 v1.66.6
go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40
go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4
go: downloading github.com/aws/aws-sdk-go-v2/service/signin v1.0.4
go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.30.8
go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12
go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40
go: downloading github.com/pion/dtls/v3 v3.1.4
go: downloading github.com/pion/transport/v4 v4.0.2
go: downloading github.com/pion/randutil v0.1.0
go: downloading github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14
go: downloading github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38
go: downloading github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19
go: downloading github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23
go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40
go: downloading github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31
go: downloading github.com/prometheus/client_golang v1.17.0
go: downloading golang.org/x/sync v0.23.0
go: downloading github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
go: downloading github.com/mattn/go-isatty v0.0.20
go: downloading github.com/lmittmann/tint v1.1.3
go: downloading github.com/pierrec/lz4/v4 v4.1.22
go: downloading github.com/wlynxg/anet v0.0.5
go: downloading github.com/prometheus/common v0.45.0
go: downloading github.com/prometheus/procfs v0.12.0
go: downloading google.golang.org/protobuf v1.36.10
go: downloading github.com/cespare/xxhash/v2 v2.3.0
go: downloading github.com/beorn7/perks v1.0.1
go: downloading github.com/prometheus/client_model v0.5.0
go: downloading golang.org/x/text v0.42.0
go: downloading modernc.org/sqlite v1.55.0
go: downloading github.com/aws/aws-sdk-go-v2/service/ec2 v1.324.0
go: downloading github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0
go: downloading github.com/aws/aws-sdk-go-v2/service/ssm v1.74.0
go: downloading github.com/pierrec/lz4/v4 v4.1.22
go: downloading github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38
go: downloading github.com/klauspost/compress v1.18.0
go: downloading golang.org/x/text v0.42.0
go: downloading google.golang.org/protobuf v1.36.10
go: downloading github.com/wlynxg/anet v0.0.5
go: downloading modernc.org/libc v1.74.1
go: downloading modernc.org/libc v1.74.1
go: downloading modernc.org/memory v1.11.0
go: downloading github.com/dustin/go-humanize v1.0.1
go: downloading github.com/google/uuid v1.6.0
go: downloading modernc.org/mathutil v1.7.1
go: downloading modernc.org/memory v1.11.0
go: downloading github.com/google/uuid v1.6.0
go: downloading github.com/dustin/go-humanize v1.0.1
go: downloading modernc.org/mathutil v1.7.1
go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec
go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec
=== launchctx (3 s)
ok amadan.net/carlos/platform.git/launchctx 0.012s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
=== vet (46 s)
go: downloading github.com/aws/aws-sdk-go-v2/service/ec2 v1.324.0
go: downloading modernc.org/sqlite v1.55.0
go: downloading github.com/aws/aws-sdk-go-v2/service/ssm v1.74.0
go: downloading golang.org/x/tools v0.49.0
go: downloading github.com/pierrec/lz4/v4 v4.1.22
go: downloading github.com/klauspost/compress v1.18.0
go: downloading github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38
go: downloading google.golang.org/protobuf v1.36.10
go: downloading golang.org/x/text v0.42.0
go: downloading github.com/wlynxg/anet v0.0.5
go: downloading golang.org/x/mod v0.41.0
go: downloading modernc.org/libc v1.74.1
go: downloading modernc.org/memory v1.11.0
go: downloading github.com/google/uuid v1.6.0
go: downloading modernc.org/mathutil v1.7.1
go: downloading github.com/dustin/go-humanize v1.0.1
go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec
=== gofmt (0 s)
gofmt -l .: clean
=== rest (198 s)
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 3.616s
ok github.com/carlosframework/platform/cmd/moveprobe 11.894s
ok github.com/carlosframework/platform/cmd/zddprobe 0.736s
ok github.com/carlosframework/platform/internal/activator 59.053s
ok github.com/carlosframework/platform/internal/agentsock 0.043s
ok github.com/carlosframework/platform/internal/alerting 0.008s
ok github.com/carlosframework/platform/internal/analytics 0.034s
ok github.com/carlosframework/platform/internal/appconfig 2.806s
ok github.com/carlosframework/platform/internal/apperr 0.166s
ok github.com/carlosframework/platform/internal/applog 0.316s
ok github.com/carlosframework/platform/internal/awsinv 0.008s
ok github.com/carlosframework/platform/internal/boxchoice 0.012s
ok github.com/carlosframework/platform/internal/boxid 0.892s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.014s
ok github.com/carlosframework/platform/internal/caps 0.017s
ok github.com/carlosframework/platform/internal/certwarm 0.003s
ok github.com/carlosframework/platform/internal/cgroup 0.771s
ok github.com/carlosframework/platform/internal/changelog 0.605s
ok github.com/carlosframework/platform/internal/claimactivation 1.407s
ok github.com/carlosframework/platform/internal/clamd 0.068s
? github.com/carlosframework/platform/internal/cliflag [no test files]
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 8.276s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.014s
ok github.com/carlosframework/platform/internal/dbquery 0.006s
ok github.com/carlosframework/platform/internal/dbreads 2.103s
? github.com/carlosframework/platform/internal/dbreads/dbreadstest [no test files]
ok github.com/carlosframework/platform/internal/dbverify 0.009s
ok github.com/carlosframework/platform/internal/dnsrec 0.006s
ok github.com/carlosframework/platform/internal/docsite 0.012s
ok github.com/carlosframework/platform/internal/domains 2.697s
ok github.com/carlosframework/platform/internal/doorkey 0.010s
ok github.com/carlosframework/platform/internal/economics 0.006s
ok github.com/carlosframework/platform/internal/edge 50.228s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.137s
ok github.com/carlosframework/platform/internal/gate 1.665s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.014s
ok github.com/carlosframework/platform/internal/geoip 0.030s
ok github.com/carlosframework/platform/internal/historyreadback 0.164s
ok github.com/carlosframework/platform/internal/host 147.305s
ok github.com/carlosframework/platform/internal/hostactors 6.491s
ok github.com/carlosframework/platform/internal/hostactors/awsmembership 0.009s
ok github.com/carlosframework/platform/internal/hostbox 3.233s
ok github.com/carlosframework/platform/internal/hostclaim 0.800s
ok github.com/carlosframework/platform/internal/hostenv 0.006s
ok github.com/carlosframework/platform/internal/hostevidence 0.006s
ok github.com/carlosframework/platform/internal/hostexec 5.805s
ok github.com/carlosframework/platform/internal/hostfence 0.074s
ok github.com/carlosframework/platform/internal/hostfiles 10.426s
ok github.com/carlosframework/platform/internal/hostfloor 0.024s
ok github.com/carlosframework/platform/internal/hostguard 1.075s
ok github.com/carlosframework/platform/internal/hostops 2.045s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.005s
ok github.com/carlosframework/platform/internal/hostquarantine 0.007s
ok github.com/carlosframework/platform/internal/hostrelocate 0.646s
ok github.com/carlosframework/platform/internal/hostresolve 0.219s
ok github.com/carlosframework/platform/internal/hostwrite 0.002s
ok github.com/carlosframework/platform/internal/infraview 0.004s
ok github.com/carlosframework/platform/internal/instancectl 5.247s
ok github.com/carlosframework/platform/internal/instanceexec 2.811s
ok github.com/carlosframework/platform/internal/instancefs 0.011s
ok github.com/carlosframework/platform/internal/instancekey 0.004s
ok github.com/carlosframework/platform/internal/instanceupdates 0.014s
ok github.com/carlosframework/platform/internal/layout 1.853s
ok github.com/carlosframework/platform/internal/lbhealth 0.005s
ok github.com/carlosframework/platform/internal/lease 10.083s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.594s
ok github.com/carlosframework/platform/internal/ledger 0.027s
ok github.com/carlosframework/platform/internal/mail 0.218s
ok github.com/carlosframework/platform/internal/maintenance 0.085s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.601s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.115s
ok github.com/carlosframework/platform/internal/metrics 0.007s
ok github.com/carlosframework/platform/internal/moves 29.781s
ok github.com/carlosframework/platform/internal/moves/archive 1.035s
ok github.com/carlosframework/platform/internal/objstore 0.332s
ok github.com/carlosframework/platform/internal/ops 0.184s
ok github.com/carlosframework/platform/internal/park 0.312s
ok github.com/carlosframework/platform/internal/plans 0.006s
ok github.com/carlosframework/platform/internal/privatefile 0.008s
2026/10/24 20:49:04 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:04 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:04 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:04 provision: skipping record apps/bac/tito/instances/one.example.com.json: unreadable: read refused
2026/10/24 20:49:04 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:04 provision: an app's records could not be listed: they are skipped
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:04 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 20:49:04 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/24 20:49:04 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:49:04 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:49:04 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:49:04 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:49:04 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 20:49:04 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:49:04 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:49:04 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:49:04 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:49:04 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:04 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/24 20:49:04 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:49:04 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/24 20:49:04 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/24 20:49:04 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:49:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:05 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:49:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:05 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:05 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:05 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 20:49:05 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/24 20:49:05 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/24 20:49:05 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/24 20:49:05 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:05 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:05 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:49:08 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/24 20:49:08 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/24 20:49:08 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 20:49:08 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:08 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/24 20:49:08 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged acme.go.tito.io hibernate false -> true
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged acme.go.tito.io status-named false -> true
2026/10/24 20:49:09 provision: converged acme.go.tito.io status-named true -> false
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:49:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:49:09 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:49:09 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/24 20:49:09 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/24 20:49:09 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/24 20:49:09 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/24 20:49:09 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/24 20:49:09 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/24 20:49:09 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:09 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/24 20:49:09 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/24 20:49:09 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/24 20:49:09 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:09 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 20:49:09 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:09 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/24 20:49:09 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 20:49:09 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:09 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/24 20:49:09 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:09 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/24 20:49:09 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:09 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:49:09 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:09 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/24 20:49:09 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/24 20:49:09 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/24 20:49:09 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:09 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:49:09 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/24 20:49:09 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/24 20:49:09 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/24 20:49:09 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/24 20:49:09 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/24 20:49:09 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/24 20:49:09 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/24 20:49:09 provision: dry-run: would converge owned.go.tito.io seed
2026/10/24 20:49:09 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:09 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/24 20:49:09 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/24 20:49:09 provision: converged owned.go.tito.io channel stable -> edge
2026/10/24 20:49:09 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/24 20:49:09 provision: converged owned.go.tito.io hibernate false -> true
2026/10/24 20:49:09 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/24 20:49:09 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/24 20:49:09 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: removed retired static route site.go.tito.io (record gone)
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/24 20:49:09 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/24 20:49:09 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:09 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:10 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:10 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:10 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 20:49:10 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/24 20:49:10 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/24 20:49:10 provision: converged acme.go.tito.io channel stable -> edge
2026/10/24 20:49:10 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:49:10 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/24 20:49:10 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:49:10 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:49:10 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/24 20:49:10 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 20:49:10 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/24 20:49:10 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/24 20:49:10 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/24 20:49:10 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/24 20:49:10 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/24 20:49:10 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/24 20:49:10 provision: minted instance route acme.example.com (app "shop" acct "bac")
2026/10/24 20:49:10 provision: minted instance route plain.example.com (app "shop" acct "bac")
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/liar.shop.example.com.json: disagrees with its own path (host="other.shop.example.com" account="bac" app="shop")
2026/10/24 20:49:10 provision: skipping record apps/bac/shop/instances/garbled.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:10 provision: listing apps/bac/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:11 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:11 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/24 20:49:11 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:11 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/24 20:49:11 provision: minted sidecar route bot.shop.bac.sidecar.internal (app "shop" acct "bac" args ["run"])
2026/10/24 20:49:11 provision: removed retired sidecar route bot.shop.bac.sidecar.internal (record gone)
2026/10/24 20:49:11 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 20:49:11 provision: minted sidecar route bot.shop.bum.sidecar.internal (app "shop" acct "bum" args ["run"])
2026/10/24 20:49:11 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:11 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:11 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 20:49:11 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:11 provision: sidecar bot.shop.bum.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:11 provision: removed retired sidecar route bot.shop.bum.sidecar.internal (record gone)
2026/10/24 20:49:11 provision: a.shop.example.com: swapped for import sd_rynbjjnqvcyxawqx62aur2uk2i (the original route is journaled)
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_rynbjjnqvcyxawqx62aur2uk2i committed: its gate is cleared and its journal row dropped
2026/10/24 20:49:11 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_pyxsma3lhhkk4imwtxfn5x6tgm rolled back to its original route
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_pyxsma3lhhkk4imwtxfn5x6tgm: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_pyxsma3lhhkk4imwtxfn5x6tgm is cleaned up and its journal row retired
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_ctjrrp4qw4byl4cwlzfluef3sa rolled back to its original route
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_ctjrrp4qw4byl4cwlzfluef3sa: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_ctjrrp4qw4byl4cwlzfluef3sa is cleaned up and its journal row retired
2026/10/24 20:49:11 provision: dry-run: would redo a.shop.example.com's swap record CAS for import sd_dkb7vcofrbkha645jhfwxrzhfe
2026/10/24 20:49:11 provision: dry-run: would clear a.shop.example.com's import gate
2026/10/24 20:49:11 provision: dry-run: would converge a.shop.example.com seed source
2026/10/24 20:49:11 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:11 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (connection reset)
2026/10/24 20:49:11 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (provision: invalid maintenance projection or protected fields)
2026/10/24 20:49:11 provision: a.shop.example.com: the import's record apps/bac/shop/instances/a.shop.example.com.json is not one this pass can use (the record under apps/bac/shop/instances/a.shop.example.com.json is apps/bac/shop/instances/b.shop.example.com.json's)
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_a74gfbci5ixk7yg3hf4xqe36yy cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_flnmggf62m667jgy6fugw7gone cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_x46krvjcwhodp6tqx25asr3gsm is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:49:11 provision: a.shop.example.com: import sd_qe6hdvaihc5n7m5ko7qjlfheh4 is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_xn2hn4dbffdhibp5ytzf7lqax4 is journaled, but this box cannot act on imports; its row is left alone
2026/10/24 20:49:12 provision: the import journal could not be read: registry: import journal a.shop.example.com: original row: unexpected end of JSON input (no import is acted on and no route is removed this pass)
2026/10/24 20:49:12 provision: b.shop.example.com: import sd_hpdj3srnxq345jbevlyxjwvp7u left alone this pass: the import journal could not be read
2026/10/24 20:49:12 provision: c.shop.example.com: import sd_lrmbp4qioo54twqxyylfuubgpu left alone this pass: the import journal could not be read
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_upyv632adl36x45d3apwf4xcpe committed: its gate is cleared and its journal row dropped
2026/10/24 20:49:12 provision: a.shop.example.com: not converging import sd_7h4lvtdvq4pzptojvc5jwgpa5i: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 20:49:12 provision: a.shop.example.com: not converging import sd_ozpvajlsomhwhdh5z5j4nxp2fu: it is swapped but not committed, and the row on this box has no import gate
2026/10/24 20:49:12 provision: a.shop.example.com: not converging import sd_eavhkgdriiw6rkcynv3kr4j764: it is placed but not committed, and the row on this box has no import gate
2026/10/24 20:49:12 provision: a.shop.example.com: not converging import sd_2hkefvmfyqopdiqbznhpbanjry: it is placed but not committed, and the row on this box has no import gate
2026/10/24 20:49:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:12 provision: converge seed source on a.shop.example.com: constraint failed: seed_source is read-only here (1811)
2026/10/24 20:49:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ktiptsvxa47dr57buhirxtss6e committed: its gate is cleared and its journal row dropped
2026/10/24 20:49:12 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:12 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:12 provision: acme.go.tito.io: not minting a seeded instance: this box does not mint nested (<nil>)
2026/10/24 20:49:12 provision: acme.go.tito.io: not minting a seeded instance: claim control is on for this box
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_riqnicltucq7upaorfkrn5x3xi: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_riqnicltucq7upaorfkrn5x3xi rolled back (no box had swapped it)
2026/10/24 20:49:12 provision: acme.go.tito.io: not minting a seeded instance: its record does not describe a hibernating instance
2026/10/24 20:49:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:12 provision: removed instance route acme.go.tito.io (its seed sd_ql5zl4ktrg63itrwbdp3plljhm was aborted); the next pass rolls it back
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_ql5zl4ktrg63itrwbdp3plljhm: this box cannot read seed markers, and places no seed; nothing local is removed
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_ql5zl4ktrg63itrwbdp3plljhm rolled back (no box had swapped it)
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_bhtsjvalsol44joushoxukswda has a row this pass may not remove; leaving the seed aborted
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_rx76bihelob7jfxyvxdy34doya rolled back (no box had swapped it)
2026/10/24 20:49:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:12 provision: removed instance route acme.go.tito.io (its seed sd_eewffahk4didru2drgmpvffpgu was aborted); the next pass rolls it back
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_eewffahk4didru2drgmpvffpgu has no route on this box: its local data is removed
2026/10/24 20:49:12 provision: acme.go.tito.io: aborted seed sd_eewffahk4didru2drgmpvffpgu rolled back (no box had swapped it)
2026/10/24 20:49:12 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_6nikdt74algrglrfzuz3njdvmy (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_6nikdt74algrglrfzuz3njdvmy rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_6nikdt74algrglrfzuz3njdvmy is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_lb4r2wosztjn3pxtanfxfhlqyu (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_lb4r2wosztjn3pxtanfxfhlqyu rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_lb4r2wosztjn3pxtanfxfhlqyu is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_mdsebgubrhjrsrpog4egyd7rsy (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mdsebgubrhjrsrpog4egyd7rsy rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mdsebgubrhjrsrpog4egyd7rsy: the host's replica is left alone: its record no longer names this seed
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mdsebgubrhjrsrpog4egyd7rsy is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_5chmxtn4ijxccwfnmalev3xz6q (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5chmxtn4ijxccwfnmalev3xz6q rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5chmxtn4ijxccwfnmalev3xz6q rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5chmxtn4ijxccwfnmalev3xz6q is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_ow7crz5of56klx4nliapnp6ypy (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ow7crz5of56klx4nliapnp6ypy rollback deferred: the host is not fenced and stopped yet (stop unproven)
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_modfogi6a46cboh4fmm4qhodfa (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_modfogi6a46cboh4fmm4qhodfa is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_czapzpyo6e3flquekcht6vx4i4 (the original route is journaled)
2026/10/24 20:49:12 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_czapzpyo6e3flquekcht6vx4i4 left alone this pass: its record could not be read
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_oheps7phzovvf7rrfivkap5epm (the original route is journaled)
2026/10/24 20:49:12 provision: skipping record apps/bac/shop/instances/a.shop.example.com.json: read failed: connection reset
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_oheps7phzovvf7rrfivkap5epm left alone this pass: its record could not be read
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_h4k3zsq4aawj76hnnveah3l24y (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_h4k3zsq4aawj76hnnveah3l24y: close the import gate: constraint failed: injected (1811)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_h4k3zsq4aawj76hnnveah3l24y rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_h4k3zsq4aawj76hnnveah3l24y is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_5qijk26brvxixiupzqcojsadmi (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5qijk26brvxixiupzqcojsadmi: close the import gate: constraint failed: injected (1811)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5qijk26brvxixiupzqcojsadmi rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5qijk26brvxixiupzqcojsadmi is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_5zrd2ccp4jxg2neyzguac6x2be (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5zrd2ccp4jxg2neyzguac6x2be rollback deferred: the host is not fenced and stopped yet (injected failure at fence)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5zrd2ccp4jxg2neyzguac6x2be rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_5zrd2ccp4jxg2neyzguac6x2be is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_ehj7zf6gapcqvgrfhk5dqt2qcq (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ehj7zf6gapcqvgrfhk5dqt2qcq rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ehj7zf6gapcqvgrfhk5dqt2qcq is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_ofeoxdfvn7dmmskzo767vlhjsq (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ofeoxdfvn7dmmskzo767vlhjsq: retire its instance key: injected failure at retire-key
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ofeoxdfvn7dmmskzo767vlhjsq rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_ofeoxdfvn7dmmskzo767vlhjsq is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_yr7ergqwvvgbyv7lfkhg6jowp4 (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_yr7ergqwvvgbyv7lfkhg6jowp4 rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_yr7ergqwvvgbyv7lfkhg6jowp4 is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_at5lkk7fed35pruhotkschrvhq (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_at5lkk7fed35pruhotkschrvhq: remove its local data: injected failure at remove-local
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_at5lkk7fed35pruhotkschrvhq rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_at5lkk7fed35pruhotkschrvhq is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_3ifnkmmamualw3mlcqeq3dd2ye (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_3ifnkmmamualw3mlcqeq3dd2ye rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_3ifnkmmamualw3mlcqeq3dd2ye is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_mli32uiuxcmmfasd6dgzftnmxi (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mli32uiuxcmmfasd6dgzftnmxi: remove its local data: injected failure at remove-local-done
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mli32uiuxcmmfasd6dgzftnmxi rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_mli32uiuxcmmfasd6dgzftnmxi is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_yu5zm6jtlwwuwvbcvkdhwxhez4 (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_yu5zm6jtlwwuwvbcvkdhwxhez4 rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_yu5zm6jtlwwuwvbcvkdhwxhez4 is cleaned up and its journal row retired
2026/10/24 20:49:12 provision: a.shop.example.com: swapped for import sd_6hyitiennqhydtyeos53gip6jy (the original route is journaled)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_6hyitiennqhydtyeos53gip6jy: restore the original route: constraint failed: injected (1811)
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_6hyitiennqhydtyeos53gip6jy rolled back to its original route
2026/10/24 20:49:12 provision: a.shop.example.com: import sd_6hyitiennqhydtyeos53gip6jy is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_3xrm4tkh2qcrt4ll23vz3bwpgq (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_3xrm4tkh2qcrt4ll23vz3bwpgq: restore the original route: constraint failed: injected (1811)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_3xrm4tkh2qcrt4ll23vz3bwpgq rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_3xrm4tkh2qcrt4ll23vz3bwpgq is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_lznorcg22f5shf2vbpf64wntvy (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_lznorcg22f5shf2vbpf64wntvy is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_7nz7lfzhapownwoz6kpr5lqk4u (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_7nz7lfzhapownwoz6kpr5lqk4u rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_7nz7lfzhapownwoz6kpr5lqk4u: delete its snapshot: injected failure at delete-snapshot
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_7nz7lfzhapownwoz6kpr5lqk4u is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_2hzwg4azw6b37yzf2mn2martkq (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2hzwg4azw6b37yzf2mn2martkq rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2hzwg4azw6b37yzf2mn2martkq is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_sovjhe74i2acmcazni4ebfvb3a (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_sovjhe74i2acmcazni4ebfvb3a rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_sovjhe74i2acmcazni4ebfvb3a: delete the host's replica: injected failure at delete-replica
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_sovjhe74i2acmcazni4ebfvb3a is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_p5qsro4usaoghpzxeomc27tfae (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_p5qsro4usaoghpzxeomc27tfae rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_p5qsro4usaoghpzxeomc27tfae is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_5uruppairjnjukrknseualdbma (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_5uruppairjnjukrknseualdbma rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_5uruppairjnjukrknseualdbma: retire its journal row: constraint failed: injected (1811)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_5uruppairjnjukrknseualdbma is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_pcwszqu5ueydogzvcumra4wtce (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pcwszqu5ueydogzvcumra4wtce rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pcwszqu5ueydogzvcumra4wtce: retire its journal row: constraint failed: injected (1811)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pcwszqu5ueydogzvcumra4wtce is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_ax2hi7zxk3m53svqftsedyfemi (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_ax2hi7zxk3m53svqftsedyfemi rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_ax2hi7zxk3m53svqftsedyfemi is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: mark import sd_ax2hi7zxk3m53svqftsedyfemi rolled back: injected failure at cas
2026/10/24 20:49:13 provision: a.shop.example.com: aborted seed sd_ax2hi7zxk3m53svqftsedyfemi rolled back (no box had swapped it)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_pl6x47x5c5e2fmiiu64hikcpk4 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pl6x47x5c5e2fmiiu64hikcpk4 rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pl6x47x5c5e2fmiiu64hikcpk4 is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: aborted seed sd_pl6x47x5c5e2fmiiu64hikcpk4 rolled back (no box had swapped it)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_wzdrqpobzbdc664x6b6xtnos7e (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_wzdrqpobzbdc664x6b6xtnos7e rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_wzdrqpobzbdc664x6b6xtnos7e rollback deferred: the host is not fenced and stopped yet (<nil>)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_wzdrqpobzbdc664x6b6xtnos7e rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_wzdrqpobzbdc664x6b6xtnos7e is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_6jfayi322qge5nm5fpenilk65i (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_6jfayi322qge5nm5fpenilk65i rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_6jfayi322qge5nm5fpenilk65i is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_zx25dxzcggrnhfdnjy5mzszdey (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_zx25dxzcggrnhfdnjy5mzszdey rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_zx25dxzcggrnhfdnjy5mzszdey is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_fg5gy6dxtqqhtmun6amwjty4ze (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_fg5gy6dxtqqhtmun6amwjty4ze rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_fg5gy6dxtqqhtmun6amwjty4ze is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_zqc7wwrbhso3ogit57vopjfynu (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_zqc7wwrbhso3ogit57vopjfynu rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_zqc7wwrbhso3ogit57vopjfynu is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_lnayvq5namiqmfjr7hchtpe6mi (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_lnayvq5namiqmfjr7hchtpe6mi rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_lnayvq5namiqmfjr7hchtpe6mi is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_dkzb3ssdxoiffsxqoamswujh7q (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_dkzb3ssdxoiffsxqoamswujh7q rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_dkzb3ssdxoiffsxqoamswujh7q is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_cj6jgu6ujhkobbd3gazdbv3esi (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_cj6jgu6ujhkobbd3gazdbv3esi rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_cj6jgu6ujhkobbd3gazdbv3esi is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "other" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_rx7adzbvcy677jdom2itqdir2a (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_rx7adzbvcy677jdom2itqdir2a has no route on this box: its local data is removed and no route is put back
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_rx7adzbvcy677jdom2itqdir2a is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_kuyv2dz5cmhml4r4vav6ekgvky (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_j4s7kcfpwyrfrb34ks7yeztz5q (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_lzpynzykef62dsf7ywekv62jja (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_lzpynzykef62dsf7ywekv62jja cannot roll back on this box: the activator is not wired; it stays journaled and gated
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_5a5ljbc4dzvggobz47c2r3wwjy (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_5a5ljbc4dzvggobz47c2r3wwjy is not rolled back: the route changed under the fence, so nothing there is touched
2026/10/24 20:49:13 provision: a.shop.example.com: import "../not-a-seed" names no snapshot and no replica; only its journal row is cleaned up
2026/10/24 20:49:13 provision: a.shop.example.com: import ../not-a-seed is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_jljw3cdwm5pl3hfeck66hfmb5i (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_k3h74hjsro5sxdrxyxb6t54lvm (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_p7yxoi6tmrdc2gxt4tsq7tuzqi (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_3wwohxivc4236ahcexgsebaqwq (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_pryxxfvfy2wgkypypel7mkh4i4 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_pryxxfvfy2wgkypypel7mkh4i4 rollback deferred: the host is not fenced and stopped yet (stop evidence unreadable)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_kpfgvcvqtrk5fuduhc3tijaa34 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_kpfgvcvqtrk5fuduhc3tijaa34 has no route on this box: its local data is removed and no route is put back
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_kpfgvcvqtrk5fuduhc3tijaa34 is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_tpsut2dtgsgix5vlgri6fkcrcq (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_tpsut2dtgsgix5vlgri6fkcrcq: remove its local data: injected failure at remove-local
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_tpsut2dtgsgix5vlgri6fkcrcq is not rolled back: the route at the host is not the import's, so nothing there is touched
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_4mc5icskoavsazyjshkkxkgwxq (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_4mc5icskoavsazyjshkkxkgwxq: a route appeared at the host, so its local database is left and the rollback waits
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_2rbhi5unh5se63fczxs52uofna (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2rbhi5unh5se63fczxs52uofna rollback deferred: no fence generation: registry busy
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_rvpn6n72ei4u5hnibqqlc5zaka (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_rvpn6n72ei4u5hnibqqlc5zaka rollback deferred: no fence generation: the generation source answered 0
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_76owdbcw2nfv5i5sh7f4gbcjai (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_wekjjwx5rpn5lavtq3uruepae4 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_5msb6a2buomchvxwy7cizulxeu (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import deferred: this box cannot read the instance's env file
2026/10/24 20:49:13 provision: a.shop.example.com: import deferred: the instance's env file could not be read: a symlink
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_big4go4qaxysj4e336yffxajce (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_big4go4qaxysj4e336yffxajce rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_big4go4qaxysj4e336yffxajce is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_2ur3pm6i57qxkjhg47yosxtd3m (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2ur3pm6i57qxkjhg47yosxtd3m rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2ur3pm6i57qxkjhg47yosxtd3m is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_2gqcicvgz4q6vc4hjcpze6srla (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2gqcicvgz4q6vc4hjcpze6srla rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_2gqcicvgz4q6vc4hjcpze6srla is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_qc324zas5gzhl4g6jsilylo6q4 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_qc324zas5gzhl4g6jsilylo6q4: retire its instance key: the env file is a symlink
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_qc324zas5gzhl4g6jsilylo6q4 rolled back to its original route
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_qc324zas5gzhl4g6jsilylo6q4 is cleaned up and its journal row retired
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_3kpd46qida3235pmovscl25od4 (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: row is on the nested layout (db "/data/instances/a.shop.example.com/a.shop.example.com.db" socket "/run/carlos/instances/a.shop.example.com/app.sock"); the record's shared paths are not drift
2026/10/24 20:49:13 provision: a.shop.example.com: swapped for import sd_2h5b2cpx3mljxxnmnxucrnr24y (the original route is journaled)
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_xp5pynx6yqrf4adzqrxmszj2mq refused: there is no route at a.shop.example.com to replace
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_dpvwjhgkflhctyldmavxz57nay refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_an7xc7djqaiqluix5w7magwmwa refused: the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_yp6pigz76lcpyfxg6tjpeyijsu refused: the route's address is 127.0.0.1:9, not 127.0.0.1:8401
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_7dgthidyyv3mtgdban7ctbsobq refused: the route belongs to app other
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_kcqnonzreoblevlu6yynlcefnq refused: the route is filed under another account
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_fzzf7mbn72zhfxf7vdaquk6bti refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_o6pmbx7dw3ymxffadmewwtesfm refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:49:13 provision: a.shop.example.com: import sd_j3vz3gmqgtqx6dgl7prmw5haau refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_6gzhgzspapfjxkviwuqcffxmfy refused: the route is filed under the operator account, and no assignment maps shop to this app
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_inttqmshuubcuhlkpl4anh3ee4 refused: the route is not an address route this import can take over (socket)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_afpxo3xwf4dwdd7fk6dkatyvxm refused: the route is not an address route this import can take over (no address)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_bzskvpvkmaqwi2w34fryfd3fhm refused: the route is not an address route this import can take over (provisioned)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_cobbpyrdvoabgo7pskrnaozvnq refused: the route is not an address route this import can take over (kind static)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_h5sbrgdyobgocbr5ianw4htsom refused: the route is not an address route this import can take over (backing)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_2qx4sb4dmfhha2kqi624dwrhxa refused: the route is not an address route this import can take over (tunnel)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_x2hre5o5cvxlnlovwx75242iy4 refused: the route is not an address route this import can take over (caps)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_4ow54izxsro3hk6voya24epkqi refused: the route is not an address route this import can take over (platform)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_qxrr3itnoa2n3xorx4rg5cxdd4 refused: the route is not an address route this import can take over (canary)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_xbgkf7jkikqzwf5pwecbawhr5u refused: the route is not an address route this import can take over (kind sidecar)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_wxulr3qp5s2iqfawhvfogemqkq refused: the route is not an address route this import can take over (region)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_2s7xtitqi4zd5x3k72mbprzu6i refused: the route is not an address route this import can take over (fleet_box)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_dgn2kyi2cr7b6nf44ril5ffpri refused: the route is not an address route this import can take over (hibernate)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_dtxilxwwbm7hq2ngwcwqkykroq refused: the route is not an address route this import can take over (claim_owner)
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_njeh7oqffmhwkabdqzjxzuxmjy refused: this box does not run instances nested
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_f7zqmukuaoxua7xumzqygj2qfq refused: this box does not run instances nested
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_hks2jkwt72hlomuf2oe2sqozua refused: claim control is on for this box
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_k2ax5xpqefnjpfze5ogtsntugy (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_gvv36ekjx2msltyvk6pc7dwd7q (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: import deferred: this box cannot read the legacy route assignments
2026/10/24 20:49:14 provision: a.shop.example.com: import deferred: the legacy route assignments could not be read: store down
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_ouia6xan7ppcwewiikjibjsplu: its verify hash is not a sha256
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_f7ohwx5dygcsqx5pwpfqekofqi: its verify hash is not a sha256
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_3qxcbaoh7yjonjwc4i46gytcrq: its verify hash is not a sha256
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_../../x: its host or seed id is malformed (seeds: not a seed id)
2026/10/24 20:49:14 provision: A.shop.example.com/..: not swapping for import sd_dltoi6fsww63kp2mbak565dwoq: its host or seed id is malformed (seeds: not a host)
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_pb3jvgwwp26muq3ezmr63ijuga: its record does not describe a nested, hibernating instance
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_uuxuvktmvi7cdfdxtv26afu44a: its record does not describe a nested, hibernating instance
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_iwfsulmlxglc62ifxkxvthcvla: its record does not describe a nested, hibernating instance
2026/10/24 20:49:14 provision: a.shop.example.com: not swapping for import sd_yroxnxwfancwr3dp73cfa6b6y4: its record does not describe a nested, hibernating instance
2026/10/24 20:49:14 provision: a.shop.example.com: import deferred: the box's layout intent could not be read: store down
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_tfl3ntenb4vpcehr4ofxba27hq (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: import deferred: the box's nested probe did not answer: ready probe timed out
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_4fwzkuau3rm7oqwzgwsbft2cwq (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_dhymd2m7ld4zjorqx5uutzv2iq (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: import deferred: the route is held by operation op_1
2026/10/24 20:49:14 provision: dry-run: would consider swapping a.shop.example.com for import sd_aqhhn3uij7lo6ygi5w7tstqtmy
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_oedncqtpajgmqc57ujomvnj7g4 rolled back to its original route
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_oedncqtpajgmqc57ujomvnj7g4: the host's replica is left alone: the record apps/bac/shop/instances/a.shop.example.com.json holds the host
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_oedncqtpajgmqc57ujomvnj7g4 is cleaned up and its journal row retired
2026/10/24 20:49:14 provision: a.shop.example.com: import sd_sn3fzfqmwayosyogj2dyoyq3s4 left alone: the import journal holds seed sd_2bmksjuqflxxwvrqeksujemcjq at swapped
2026/10/24 20:49:14 provision: a.shop.example.com: not minting import sd_nj64cfnotsvoscpve4u3ljvasi: it is swapped but not committed, and only its swap makes its row
2026/10/24 20:49:14 provision: a.shop.example.com: not minting import sd_4qm4jclcz2f2m3fgqvvtb72eea: it is placed but not committed, and only its swap makes its row
2026/10/24 20:49:14 provision: a.shop.example.com: swapped for import sd_vkqs4bg2rl5bqkr42prfqgjcfm (the original route is journaled)
2026/10/24 20:49:14 provision: a.shop.example.com: not minting import sd_vkqs4bg2rl5bqkr42prfqgjcfm: it is swapped but not committed, and only its swap makes its row
2026/10/24 20:49:14 provision: a.shop.example.com: not minting import sd_vkqs4bg2rl5bqkr42prfqgjcfm: it is placed but not committed, and only its swap makes its row
2026/10/24 20:49:14 provision: minted instance route a.shop.example.com (app "shop" acct "bac")
2026/10/24 20:49:14 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/24 20:49:14 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/24 20:49:14 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/24 20:49:14 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/24 20:49:14 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/24 20:49:14 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 20:49:14 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/24 20:49:14 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/24 20:49:14 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/24 20:49:14 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:49:14 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/24 20:49:14 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/24 20:49:14 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/24 20:49:14 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/24 20:49:14 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/24 20:49:14 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/24 20:49:14 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/24 20:49:14 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/24 20:49:14 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/24 20:49:14 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/24 20:49:14 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:14 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/24 20:49:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:49:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:49:15 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 20:49:15 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/24 20:49:15 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/24 20:49:15 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/24 20:49:15 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/24 20:49:15 provision: record view: change index unusable (release: change index unreadable: store has no conditional read: unsupported operation): the walk records no generations
2026/10/24 20:49:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:15 provision: record view: change index unusable (release: change index malformed: invalid character 'n' looking for beginning of object key string): walking
2026/10/24 20:49:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: listing apps/bqd/: listdirs refused — its records are unseen this pass
2026/10/24 20:49:15 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 20:49:15 provision: listing apps/bqc/shop/instances/: context canceled — its records are unseen this pass
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: record view: re-reading apps/bqc/shop failed: retrying on the next refresh
2026/10/24 20:49:15 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:15 provision: record view: change index unusable (get refused): walking
2026/10/24 20:49:15 provision: listing apps/bqc/shop/instances/: list refused — its records are unseen this pass
2026/10/24 20:49:15 provision: an app's records could not be listed: they are skipped
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:49:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:49:15 provision: skipping accounts/bqd/account.json: malformed account manifest: invalid character 'n' looking for beginning of object key string
2026/10/24 20:49:15 provision: record view: re-reading accounts/bqd failed: retrying on the next refresh
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: skipping accounts/bqd/account.json: read failed: list refused
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:49:15 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 20:49:15 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 20:49:15 provision: skipping record apps/bum/shop/instances/one.example.com.json: empty channel
2026/10/24 20:49:15 provision: removed retired instance route one.example.com (record gone)
2026/10/24 20:49:15 provision: not removing one.example.com: its record exists but could not be read cleanly this pass
2026/10/24 20:49:15 provision: minted instance route one.example.com (app "shop" acct "bum")
2026/10/24 20:49:15 provision: removed retired instance route one.example.com (record gone)
2026/10/24 20:49:15 provision: record view: change index unusable (context deadline exceeded): walking
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/24 20:49:15 provision: record view: change index unusable (context deadline exceeded): the walk records no generations
2026/10/24 20:49:15 provision: skipping record apps/bqe/store/instances/bad.store.example.com.json: malformed: provision: invalid maintenance projection or protected fields
--- FAIL: TestPlan14RootReadCensus (41.68s)
plan14_root_reads_test.go:600: root read census: 1 discrepancies
changed root source envelope internal/ratelimit
make census-card shows what changed; make census-propose NAME=<branch> CLAIMS=<file> writes the overlay and its approval card (AGENTS.md, The census)
FAIL
FAIL github.com/carlosframework/platform/internal/provision 52.357s
ok github.com/carlosframework/platform/internal/publicrelease 0.011s
ok github.com/carlosframework/platform/internal/ratelimit 0.114s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 5.939s
ok github.com/carlosframework/platform/internal/relay 7.239s
ok github.com/carlosframework/platform/internal/release 34.445s
ok github.com/carlosframework/platform/internal/rollout 4.089s
ok github.com/carlosframework/platform/internal/rolloutwords 0.004s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/rootparticipation 0.063s
ok github.com/carlosframework/platform/internal/rootstate 0.027s
ok github.com/carlosframework/platform/internal/runs 0.536s
ok github.com/carlosframework/platform/internal/s3compat 0.006s
ok github.com/carlosframework/platform/internal/safefile 0.006s
ok github.com/carlosframework/platform/internal/schedule 2.916s
ok github.com/carlosframework/platform/internal/seeds 0.011s
ok github.com/carlosframework/platform/internal/sim/engines 3.042s
ok github.com/carlosframework/platform/internal/sim/hostlife 10.906s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.465s
ok github.com/carlosframework/platform/internal/status 0.106s
ok github.com/carlosframework/platform/internal/steering 0.007s
ok github.com/carlosframework/platform/internal/stopevidence 0.307s
ok github.com/carlosframework/platform/internal/svcrole 0.005s
ok github.com/carlosframework/platform/internal/telemetry 0.006s
ok github.com/carlosframework/platform/internal/termfence 0.005s
ok github.com/carlosframework/platform/internal/tunnelclient 0.005s
ok github.com/carlosframework/platform/internal/unixdial 0.006s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.006s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.666s
FAIL
=== internal_console@0 (91 s)
ok github.com/carlosframework/platform/internal/console 89.297s
=== internal_console@1 (50 s)
ok github.com/carlosframework/platform/internal/console 47.633s
=== internal_console@2 (44 s)
ok github.com/carlosframework/platform/internal/console 42.243s
=== internal_console@3 (47 s)
ok github.com/carlosframework/platform/internal/console 45.013s
=== internal_console@4 (49 s)
ok github.com/carlosframework/platform/internal/console 46.912s
=== internal_console@5 (50 s)
ok github.com/carlosframework/platform/internal/console 48.064s
=== internal_console@6 (47 s)
ok github.com/carlosframework/platform/internal/console 45.032s
=== internal_console@7 (52 s)
ok github.com/carlosframework/platform/internal/console 50.216s
=== root@0 (113 s)
ok github.com/carlosframework/platform 111.824s
=== root@1 (83 s)
ok github.com/carlosframework/platform 82.314s
=== root@2 (95 s)
ok github.com/carlosframework/platform 93.782s
FAILED: rest
make: *** [Makefile:19: test] Error 1
amadan: step gate failed: exit status 2
amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/ratelimit/limiter.go b/internal/ratelimit/limiter.go| new file mode 100644 |
| index 000000000..9510d9358 |
| --- /dev/null |
| +++ b/internal/ratelimit/limiter.go |
| @@ -0,0 +1,271 @@ |
| +package ratelimit |
| + |
| +import ( |
| + "net/netip" |
| + "sync" |
| + "time" |
| +) |
| + |
| +// Decision is the limiter's verdict on one request. |
| +type Decision int |
| + |
| +const ( |
| + // Allow lets the request through. |
| + Allow Decision = iota |
| + // Shed refuses a client that already holds its share of a busy instance. |
| + Shed |
| + // OverCeiling refuses a client whose backstop bucket is empty. |
| + OverCeiling |
| +) |
| + |
| +func (d Decision) String() string { |
| + switch d { |
| + case Allow: |
| + return "Allow" |
| + case Shed: |
| + return "Shed" |
| + case OverCeiling: |
| + return "OverCeiling" |
| + } |
| + return "Decision(?)" |
| +} |
| + |
| +// nano is one token in the bucket's unit. Counting billionths of a token |
| +// against nanoseconds makes a refill of rate tokens a second exactly rate |
| +// units a nanosecond, so the bucket is integer arithmetic with no rounding. |
| +const nano = int64(time.Second) |
| + |
| +// key is one client's allowance under one rule of one host. A path rule's |
| +// requests count against its own key, so heavy API use leaves the client's |
| +// base allowance alone. |
| +type key struct { |
| + host string |
| + rule int // index into Effective.Paths, -1 for the base rule |
| + client netip.Prefix |
| +} |
| + |
| +type entry struct { |
| + k key |
| + tokens int64 // in billionths of a token |
| + last int64 // ns since Limiter.epoch of the last refill |
| + fullAt int64 // when the bucket is full again; from then on the entry says nothing a fresh one would not |
| + holders int // slots held now; an entry with holders is never evicted |
| + |
| + prev, next *entry // links in the idle list, set only while holders == 0 |
| +} |
| + |
| +// Limiter decides, per request, whether an instance's edge lets it through: |
| +// a token bucket per client that always applies (the backstop), and a share |
| +// of the instance's in-flight slots per client that applies only while the |
| +// instance is busy. One mutex covers everything; the work under it is a map |
| +// lookup and a little arithmetic. |
| +// |
| +// Memory is bounded by maxKeys tracked entries. Entries with no slots held |
| +// are evicted to make room, those whose buckets have refilled first, then the |
| +// least recently used. An entry holding slots is never evicted. When every |
| +// entry holds slots, a new client is let through untracked rather than |
| +// refused for want of memory. |
| +type Limiter struct { |
| + now func() time.Time |
| + epoch time.Time |
| + maxKeys int |
| + |
| + mu sync.Mutex |
| + entries map[key]*entry |
| + inflight map[string]int // per host; a host with none has no entry |
| + idle entry // sentinel of a circular list of zero-holder entries, least recently used first |
| +} |
| + |
| +// NewLimiter returns a Limiter that reads the time from now and tracks at |
| +// most maxKeys (host, rule, client) entries. |
| +func NewLimiter(now func() time.Time, maxKeys int) *Limiter { |
| + l := &Limiter{ |
| + now: now, |
| + epoch: now(), |
| + maxKeys: max(maxKeys, 1), |
| + entries: map[key]*entry{}, |
| + inflight: map[string]int{}, |
| + } |
| + l.idle.prev, l.idle.next = &l.idle, &l.idle |
| + return l |
| +} |
| + |
| +// Ticket is a slot an admitted request holds until it is answered. |
| +type Ticket struct { |
| + l *Limiter |
| + e *entry |
| + done bool // guarded by l.mu |
| +} |
| + |
| +// noSlot is the Ticket for a request let through without a slot: an upgrade |
| +// judged with holdSlot false, or a client admitted untracked. |
| +var noSlot = &Ticket{} |
| + |
| +// Release gives the slot back. Only the first call counts; calling it on a |
| +// nil Ticket or on one that holds no slot does nothing. |
| +func (t *Ticket) Release() { |
| + if t == nil || t.l == nil { |
| + return |
| + } |
| + l := t.l |
| + l.mu.Lock() |
| + if !t.done { |
| + t.done = true |
| + e := t.e |
| + e.holders-- |
| + if e.holders == 0 { |
| + l.pushIdle(e) |
| + } |
| + if n := l.inflight[e.k.host] - 1; n > 0 { |
| + l.inflight[e.k.host] = n |
| + } else { |
| + delete(l.inflight, e.k.host) |
| + } |
| + } |
| + l.mu.Unlock() |
| +} |
| + |
| +// Admit decides one request. holdSlot false (upgrades) judges without taking a slot. |
| +// On Shed/OverCeiling the returned Ticket is nil unless observe is true, in which |
| +// case the request is let through and the Ticket holds a slot like Allow. |
| +// |
| +// The order is fixed: a backstop token is taken first, so a request that is |
| +// then shed has still spent one. retryAfter is zero on Allow and at least a |
| +// second otherwise. A rule field of zero or less turns that check off, so a |
| +// zero Effective refuses nothing. |
| +func (l *Limiter) Admit(host string, eff Effective, client netip.Prefix, path string, holdSlot, observe bool) (t *Ticket, d Decision, retryAfter time.Duration) { |
| + idx, rule := eff.RuleFor(path) |
| + k := key{host: host, rule: idx, client: client} |
| + now := int64(l.now().Sub(l.epoch)) |
| + |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + e := l.entries[k] |
| + if e == nil { |
| + if len(l.entries) >= l.maxKeys && !l.evict(now) { |
| + return noSlot, Allow, 0 |
| + } |
| + e = &entry{k: k, tokens: int64(min(max(rule.Burst, 0), MaxValue)) * nano, last: now, fullAt: now} |
| + l.entries[k] = e |
| + l.pushIdle(e) |
| + } |
| + |
| + if ok, wait := e.spend(now, rule.Rate, rule.Burst); !ok { |
| + d, retryAfter = OverCeiling, roundUpSeconds(wait) |
| + } else if eff.BusyAt > 0 && rule.ClientShare > 0 && |
| + l.inflight[host] >= eff.BusyAt && e.holders >= rule.ClientShare { |
| + d, retryAfter = Shed, time.Second |
| + } |
| + if d != Allow && !observe { |
| + l.touch(e) |
| + return nil, d, retryAfter |
| + } |
| + if !holdSlot { |
| + l.touch(e) |
| + return noSlot, d, retryAfter |
| + } |
| + if e.holders == 0 { |
| + l.unlinkIdle(e) |
| + } |
| + e.holders++ |
| + l.inflight[host]++ |
| + return &Ticket{l: l, e: e}, d, retryAfter |
| +} |
| + |
| +// InFlight is the number of slots held for host. |
| +func (l *Limiter) InFlight(host string) int { |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + return l.inflight[host] |
| +} |
| + |
| +// spend refills the bucket to now and takes one token. With none to take it |
| +// reports how long until there is one. A rate or burst of zero or less means |
| +// no backstop. |
| +func (e *entry) spend(now int64, rate, burst int) (ok bool, wait time.Duration) { |
| + if rate <= 0 || burst <= 0 { |
| + e.fullAt = now |
| + return true, 0 |
| + } |
| + // Clamped to MaxValue as Validate does, so capacity and r*elapsed below |
| + // stay far inside int64 whatever a caller passes. |
| + r, capacity := int64(min(rate, MaxValue)), int64(min(burst, MaxValue))*nano |
| + e.tokens = min(e.tokens, capacity) // burst may have been lowered since |
| + // now was read before the lock, so it can trail e.last by a little; the |
| + // bucket then simply does not refill. |
| + if elapsed := now - e.last; elapsed > 0 { |
| + if elapsed >= ceilDiv(capacity-e.tokens, r) { |
| + e.tokens = capacity |
| + } else { |
| + e.tokens += r * elapsed |
| + } |
| + e.last = now |
| + } |
| + if e.tokens >= nano { |
| + e.tokens -= nano |
| + ok = true |
| + } else { |
| + wait = time.Duration(ceilDiv(nano-e.tokens, r)) |
| + } |
| + e.fullAt = e.last + ceilDiv(capacity-e.tokens, r) |
| + return ok, wait |
| +} |
| + |
| +// evict makes room for one more entry and reports whether it did. It drops |
| +// every idle entry whose bucket has refilled, which changes no answer, and |
| +// then the least recently used idle entries until an eighth of the map is |
| +// free. Freeing a batch spreads the cost of the sweep over many inserts. The |
| +// sweep walks only the idle list, so a map full of holders costs nothing. |
| +func (l *Limiter) evict(now int64) bool { |
| + for e := l.idle.next; e != &l.idle; { |
| + next := e.next |
| + if now >= e.fullAt { |
| + l.drop(e) |
| + } |
| + e = next |
| + } |
| + low := l.maxKeys - max(1, l.maxKeys/8) |
| + for len(l.entries) > low && l.idle.next != &l.idle { |
| + l.drop(l.idle.next) |
| + } |
| + return len(l.entries) < l.maxKeys |
| +} |
| + |
| +func (l *Limiter) drop(e *entry) { |
| + l.unlinkIdle(e) |
| + delete(l.entries, e.k) |
| +} |
| + |
| +// touch marks an idle entry as just used. |
| +func (l *Limiter) touch(e *entry) { |
| + if e.holders == 0 { |
| + l.unlinkIdle(e) |
| + l.pushIdle(e) |
| + } |
| +} |
| + |
| +func (l *Limiter) pushIdle(e *entry) { |
| + e.prev, e.next = l.idle.prev, &l.idle |
| + l.idle.prev.next = e |
| + l.idle.prev = e |
| +} |
| + |
| +func (l *Limiter) unlinkIdle(e *entry) { |
| + e.prev.next = e.next |
| + e.next.prev = e.prev |
| + e.prev, e.next = nil, nil |
| +} |
| + |
| +// ceilDiv is a/b rounded up, for a >= 0 and b > 0; zero for a <= 0. |
| +func ceilDiv(a, b int64) int64 { |
| + if a <= 0 { |
| + return 0 |
| + } |
| + return (a + b - 1) / b |
| +} |
| + |
| +// roundUpSeconds rounds a wait up to whole seconds, at least one, since |
| +// Retry-After carries whole seconds and a client told 0 retries at once. |
| +func roundUpSeconds(wait time.Duration) time.Duration { |
| + return max((wait+time.Second-1)/time.Second, 1) * time.Second |
| +} |
diff --git a/internal/ratelimit/limiter_test.go b/internal/ratelimit/limiter_test.go| new file mode 100644 |
| index 000000000..8767d9168 |
| --- /dev/null |
| +++ b/internal/ratelimit/limiter_test.go |
| @@ -0,0 +1,415 @@ |
| +package ratelimit |
| + |
| +import ( |
| + "net/netip" |
| + "sync" |
| + "testing" |
| + "time" |
| +) |
| + |
| +const host = "shop.example.com" |
| + |
| +// fakeClock is a settable clock, safe to read from many goroutines. |
| +type fakeClock struct { |
| + mu sync.Mutex |
| + t time.Time |
| +} |
| + |
| +func newClock() *fakeClock { return &fakeClock{t: time.Date(2026, 10, 24, 12, 0, 0, 0, time.UTC)} } |
| + |
| +func (c *fakeClock) now() time.Time { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + return c.t |
| +} |
| + |
| +func (c *fakeClock) add(d time.Duration) { |
| + c.mu.Lock() |
| + c.t = c.t.Add(d) |
| + c.mu.Unlock() |
| +} |
| + |
| +func eff(busyAt, share, rate, burst int) Effective { |
| + return Effective{BusyAt: busyAt, Base: Rule{ClientShare: share, Rate: rate, Burst: burst}} |
| +} |
| + |
| +func v4(s string) netip.Prefix { return netip.PrefixFrom(netip.MustParseAddr(s), 32) } |
| + |
| +// v6 returns the i-th /64 under 2001:db8::/32, for tests that need many clients. |
| +func v6(i int) netip.Prefix { |
| + var b [16]byte |
| + b[0], b[1], b[2], b[3] = 0x20, 0x01, 0x0d, 0xb8 |
| + b[4], b[5], b[6], b[7] = byte(i>>24), byte(i>>16), byte(i>>8), byte(i) |
| + return netip.PrefixFrom(netip.AddrFrom16(b), 64) |
| +} |
| + |
| +func (l *Limiter) tracked() int { |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + return len(l.entries) |
| +} |
| + |
| +// heldBy is the slot count of client's base-rule entry, -1 when it is not tracked. |
| +func (l *Limiter) heldBy(client netip.Prefix) int { |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + e := l.entries[key{host: host, rule: -1, client: client}] |
| + if e == nil { |
| + return -1 |
| + } |
| + return e.holders |
| +} |
| + |
| +func admit(t *testing.T, l *Limiter, e Effective, c netip.Prefix, path string) (*Ticket, Decision, time.Duration) { |
| + t.Helper() |
| + return l.Admit(host, e, c, path, true, false) |
| +} |
| + |
| +func TestAdmitQuietInstanceNeverSheds(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := Resolve(Settings{BusyAt: 64}, Ceiling{}) |
| + a := v4("203.0.113.7") |
| + for i := range 63 { |
| + if tk, d, _ := admit(t, l, e, a, "/"); d != Allow || tk == nil { |
| + t.Fatalf("admit %d: %v, ticket %v", i+1, d, tk) |
| + } |
| + } |
| + if tk, d, ra := admit(t, l, e, a, "/"); d != Allow || tk == nil || ra != 0 { |
| + t.Fatalf("64th admit on a quiet instance: %v retry %v, want Allow", d, ra) |
| + } |
| + if got := l.InFlight(host); got != 64 { |
| + t.Fatalf("InFlight = %d, want 64", got) |
| + } |
| +} |
| + |
| +func TestAdmitBusyShedsHeaviestOnly(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(4, 2, 300, 1000) |
| + a, b, c := v4("203.0.113.1"), v4("203.0.113.2"), v4("198.51.100.3") |
| + var aTickets []*Ticket |
| + for range 2 { |
| + tk, d, _ := admit(t, l, e, a, "/") |
| + if d != Allow { |
| + t.Fatalf("A: %v", d) |
| + } |
| + aTickets = append(aTickets, tk) |
| + if _, d, _ := admit(t, l, e, b, "/"); d != Allow { |
| + t.Fatalf("B: %v", d) |
| + } |
| + } |
| + if tk, d, ra := admit(t, l, e, a, "/"); d != Shed || tk != nil || ra != time.Second { |
| + t.Fatalf("A while busy: %v ticket %v retry %v, want Shed nil 1s", d, tk, ra) |
| + } |
| + if tk, d, _ := admit(t, l, e, c, "/"); d != Allow || tk == nil { |
| + t.Fatalf("fresh client C while busy: %v, want Allow", d) |
| + } |
| + aTickets[0].Release() |
| + if tk, d, _ := admit(t, l, e, a, "/"); d != Allow || tk == nil { |
| + t.Fatalf("A after releasing one: %v, want Allow", d) |
| + } |
| +} |
| + |
| +func TestBackstopBucket(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 10, 20) |
| + a := v4("203.0.113.9") |
| + for i := range 20 { |
| + tk, d, _ := admit(t, l, e, a, "/") |
| + if d != Allow { |
| + t.Fatalf("admit %d: %v", i+1, d) |
| + } |
| + tk.Release() |
| + } |
| + if tk, d, ra := admit(t, l, e, a, "/"); d != OverCeiling || tk != nil || ra != time.Second { |
| + t.Fatalf("21st: %v ticket %v retry %v, want OverCeiling nil 1s", d, tk, ra) |
| + } |
| + clk.add(100 * time.Millisecond) |
| + tk, d, _ := admit(t, l, e, a, "/") |
| + if d != Allow { |
| + t.Fatalf("after 100ms: %v, want Allow", d) |
| + } |
| + tk.Release() |
| + if _, d, _ := admit(t, l, e, a, "/"); d != OverCeiling { |
| + t.Fatalf("second after 100ms: %v, want OverCeiling", d) |
| + } |
| +} |
| + |
| +func TestBackstopRetryAfterRoundsUp(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 1, 1) |
| + a := v4("203.0.113.9") |
| + tk, d, _ := admit(t, l, e, a, "/") |
| + if d != Allow { |
| + t.Fatalf("first: %v", d) |
| + } |
| + tk.Release() |
| + if _, d, ra := admit(t, l, e, a, "/"); d != OverCeiling || ra != time.Second { |
| + t.Fatalf("second at t0: %v retry %v, want OverCeiling 1s", d, ra) |
| + } |
| + clk.add(200 * time.Millisecond) |
| + if _, d, ra := admit(t, l, e, a, "/"); d != OverCeiling || ra != time.Second { |
| + t.Fatalf("at t0+200ms: %v retry %v, want OverCeiling 1s", d, ra) |
| + } |
| + clk.add(799 * time.Millisecond) |
| + if _, d, ra := admit(t, l, e, a, "/"); d != OverCeiling || ra != time.Second { |
| + t.Fatalf("at t0+999ms: %v retry %v, want OverCeiling 1s (never below 1s)", d, ra) |
| + } |
| +} |
| + |
| +func TestPathRuleSeparateAllowance(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 1, 1) |
| + e.Paths = []PathRule{{Prefix: "/api/", Rule: Rule{ClientShare: 16, Rate: 100, Burst: 100}}} |
| + a := v4("203.0.113.4") |
| + tk, d, _ := admit(t, l, e, a, "/") |
| + if d != Allow { |
| + t.Fatalf("base first: %v", d) |
| + } |
| + tk.Release() |
| + if _, d, _ := admit(t, l, e, a, "/index.html"); d != OverCeiling { |
| + t.Fatalf("base exhausted: %v, want OverCeiling", d) |
| + } |
| + for i := range 100 { |
| + tk, d, _ := admit(t, l, e, a, "/api/x") |
| + if d != Allow { |
| + t.Fatalf("/api/x %d after base exhausted: %v, want Allow", i+1, d) |
| + } |
| + tk.Release() |
| + } |
| + // A second client spends only /api/ allowance; its base bucket is untouched. |
| + b := v4("203.0.113.5") |
| + for range 50 { |
| + tk, _, _ := admit(t, l, e, b, "/api/y") |
| + tk.Release() |
| + } |
| + if tk, d, _ := admit(t, l, e, b, "/"); d != Allow || tk == nil { |
| + t.Fatalf("base after /api/ use: %v, want Allow", d) |
| + } |
| +} |
| + |
| +func TestHoldSlotFalseTakesNoSlot(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 300, 1000) |
| + a := v4("203.0.113.4") |
| + held, _, _ := admit(t, l, e, a, "/") |
| + before := l.InFlight(host) |
| + tk, d, _ := l.Admit(host, e, a, "/ws", false, false) |
| + if d != Allow { |
| + t.Fatalf("upgrade: %v, want Allow", d) |
| + } |
| + if got := l.InFlight(host); got != before { |
| + t.Fatalf("InFlight after holdSlot=false: %d, want %d", got, before) |
| + } |
| + tk.Release() |
| + if got := l.InFlight(host); got != before { |
| + t.Fatalf("InFlight after releasing the no-slot ticket: %d, want %d", got, before) |
| + } |
| + held.Release() |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight = %d, want 0", got) |
| + } |
| +} |
| + |
| +func TestObserveLetsThroughAndHolds(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 1, 1) |
| + a := v4("203.0.113.4") |
| + first, _, _ := admit(t, l, e, a, "/") |
| + before := l.InFlight(host) |
| + tk, d, _ := l.Admit(host, e, a, "/", true, true) |
| + if d != OverCeiling || tk == nil { |
| + t.Fatalf("observe over ceiling: %v ticket %v, want OverCeiling with a ticket", d, tk) |
| + } |
| + if got := l.InFlight(host); got != before+1 { |
| + t.Fatalf("InFlight = %d, want %d", got, before+1) |
| + } |
| + tk.Release() |
| + first.Release() |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight = %d, want 0", got) |
| + } |
| + |
| + // Observe also lets a Shed through, holding a slot. |
| + e = eff(2, 1, 300, 1000) |
| + b := v4("203.0.113.5") |
| + t1, _, _ := admit(t, l, e, b, "/") |
| + t2, d, _ := l.Admit(host, e, b, "/", true, true) |
| + if d != Allow { |
| + t.Fatalf("second while quiet: %v", d) |
| + } |
| + t3, d, _ := l.Admit(host, e, b, "/", true, true) |
| + if d != Shed || t3 == nil || l.InFlight(host) != 3 { |
| + t.Fatalf("observe while busy: %v ticket %v InFlight %d, want Shed, a ticket, 3", d, t3, l.InFlight(host)) |
| + } |
| + t1.Release() |
| + t2.Release() |
| + t3.Release() |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight = %d, want 0", got) |
| + } |
| +} |
| + |
| +func TestReleaseIdempotent(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 300, 1000) |
| + a := v4("203.0.113.4") |
| + keep, _, _ := admit(t, l, e, a, "/") |
| + prior := l.InFlight(host) |
| + tk, _, _ := admit(t, l, e, a, "/") |
| + tk.Release() |
| + tk.Release() |
| + if got := l.InFlight(host); got != prior { |
| + t.Fatalf("InFlight after double Release = %d, want %d", got, prior) |
| + } |
| + var nilTicket *Ticket |
| + nilTicket.Release() |
| + keep.Release() |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight = %d, want 0", got) |
| + } |
| +} |
| + |
| +func TestMemoryBoundNeverEvictsHolders(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 100) |
| + e := eff(64, 16, 300, 1000) |
| + var holders []*Ticket |
| + for i := range 50 { |
| + tk, d, _ := admit(t, l, e, v6(i), "/") |
| + if d != Allow { |
| + t.Fatalf("holder %d: %v", i, d) |
| + } |
| + holders = append(holders, tk) |
| + } |
| + for i := range 10_000 { |
| + tk, d, _ := admit(t, l, e, v6(1000+i), "/") |
| + if d != Allow { |
| + t.Fatalf("sprayed client %d: %v", i, d) |
| + } |
| + tk.Release() |
| + if n := l.tracked(); n > 100 { |
| + t.Fatalf("after sprayed client %d: %d keys tracked, bound is 100", i, n) |
| + } |
| + if i%7 == 0 { |
| + clk.add(time.Millisecond) |
| + } |
| + } |
| + if got := l.InFlight(host); got != 50 { |
| + t.Fatalf("InFlight after spray = %d, want the 50 holders", got) |
| + } |
| + for i := range 50 { |
| + if got := l.heldBy(v6(i)); got != 1 { |
| + t.Fatalf("holder %d: tracked entry holds %d, want 1 (a holder was evicted)", i, got) |
| + } |
| + } |
| + for _, tk := range holders { |
| + tk.Release() |
| + } |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight after holders release = %d, want 0", got) |
| + } |
| + |
| + // A map full of holders admits new clients untracked rather than refusing. |
| + full := NewLimiter(clk.now, 100) |
| + holders = holders[:0] |
| + for i := range 100 { |
| + tk, _, _ := full.Admit(host, e, v6(i), "/", true, false) |
| + holders = append(holders, tk) |
| + } |
| + tk, d, _ := full.Admit(host, e, v6(5000), "/", true, false) |
| + if d != Allow || tk == nil { |
| + t.Fatalf("new client on a map full of holders: %v ticket %v, want Allow with a ticket", d, tk) |
| + } |
| + if n := full.tracked(); n != 100 { |
| + t.Fatalf("tracked = %d, want 100", n) |
| + } |
| + if got := full.InFlight(host); got != 100 { |
| + t.Fatalf("InFlight with an untracked admission = %d, want 100", got) |
| + } |
| + tk.Release() |
| + tk.Release() |
| + for _, h := range holders { |
| + h.Release() |
| + } |
| + if got := full.InFlight(host); got != 0 { |
| + t.Fatalf("InFlight = %d, want 0", got) |
| + } |
| +} |
| + |
| +func TestEvictionPrefersFullBuckets(t *testing.T) { |
| + clk := newClock() |
| + l := NewLimiter(clk.now, 3) |
| + e := eff(64, 16, 1, 1) |
| + e.Paths = []PathRule{{Prefix: "/fast/", Rule: Rule{ClientShare: 16, Rate: 1000, Burst: 1000}}} |
| + a, b, c, d := v4("203.0.113.1"), v4("203.0.113.2"), v4("203.0.113.3"), v4("203.0.113.4") |
| + use := func(cl netip.Prefix, path string) Decision { |
| + tk, dec, _ := l.Admit(host, e, cl, path, true, false) |
| + tk.Release() |
| + return dec |
| + } |
| + use(a, "/") // oldest, drained, refills in 1s |
| + clk.add(time.Millisecond) |
| + use(b, "/fast/") // full again 1ms later |
| + clk.add(time.Millisecond) |
| + use(c, "/") |
| + clk.add(10 * time.Millisecond) |
| + use(d, "/") // map full: B's bucket is full, so B goes, not the older A |
| + if got := use(a, "/"); got != OverCeiling { |
| + t.Fatalf("A after eviction: %v, want OverCeiling (A's drained bucket kept)", got) |
| + } |
| + |
| + // With no full bucket, the oldest idle entry goes. Here C is oldest. |
| + clk.add(time.Millisecond) |
| + use(b, "/") // new key, map full, nothing full: evicts C (A and D were used later) |
| + if got := use(c, "/"); got != Allow { |
| + t.Fatalf("C after eviction: %v, want Allow (evicted, so a fresh bucket)", got) |
| + } |
| +} |
| + |
| +func TestAdmitRace(t *testing.T) { |
| + clk := newClock() |
| + for _, maxKeys := range []int{100_000, 8} { |
| + l := NewLimiter(clk.now, maxKeys) |
| + e := eff(64, 16, MaxValue, MaxValue) |
| + e.Paths = []PathRule{{Prefix: "/api/", Rule: Rule{ClientShare: 4, Rate: MaxValue, Burst: MaxValue}}} |
| + var wg sync.WaitGroup |
| + for g := range 64 { |
| + wg.Go(func() { |
| + for i := range 1000 { |
| + c := v6(g % 16) |
| + if maxKeys < 100 { |
| + c = v6(g*1000 + i) |
| + } |
| + path := "/" |
| + if i%3 == 0 { |
| + path = "/api/x" |
| + } |
| + tk, _, _ := l.Admit(host, e, c, path, i%5 != 0, i%2 == 0) |
| + _ = l.InFlight(host) |
| + if i%11 == 0 { |
| + clk.add(time.Microsecond) |
| + } |
| + tk.Release() |
| + if i%13 == 0 { |
| + tk.Release() |
| + } |
| + } |
| + }) |
| + } |
| + wg.Wait() |
| + if got := l.InFlight(host); got != 0 { |
| + t.Fatalf("maxKeys %d: InFlight = %d after all releases, want 0", maxKeys, got) |
| + } |
| + if n := l.tracked(); n > maxKeys { |
| + t.Fatalf("maxKeys %d: tracked %d", maxKeys, n) |
| + } |
| + } |
| +} |
diff --git a/internal/ratelimit/shares.go b/internal/ratelimit/shares.go| new file mode 100644 |
| index 000000000..b067f388e |
| --- /dev/null |
| +++ b/internal/ratelimit/shares.go |
| @@ -0,0 +1,71 @@ |
| +package ratelimit |
| + |
| +import "sync" |
| + |
| +// Shares counts concurrent holders per key and, while the caller says it is |
| +// busy, refuses a key that already holds its share. It has no rate and no |
| +// clock. The backhaul uses it for the slots each sending edge holds. |
| +// |
| +// Memory is bounded by maxKeys. A key leaves the map when its last holder |
| +// releases, so only keys with holders are tracked; when maxKeys of them |
| +// already hold slots, a new key is let through untracked. |
| +type Shares struct { |
| + maxKeys int |
| + |
| + mu sync.Mutex |
| + held map[string]int |
| +} |
| + |
| +// NewShares returns a Shares that tracks at most maxKeys keys. |
| +func NewShares(maxKeys int) *Shares { |
| + return &Shares{maxKeys: max(maxKeys, 1), held: map[string]int{}} |
| +} |
| + |
| +func noRelease() {} |
| + |
| +// Acquire refuses when busy is true and key already holds >= share slots. |
| +// A share of zero or less never refuses. release is never nil, and only its |
| +// first call counts. |
| +func (s *Shares) Acquire(key string, busy bool, share int) (release func(), ok bool) { |
| + s.mu.Lock() |
| + n, tracked := s.held[key] |
| + if busy && share > 0 && n >= share { |
| + s.mu.Unlock() |
| + return noRelease, false |
| + } |
| + if !tracked && len(s.held) >= s.maxKeys { |
| + s.mu.Unlock() |
| + return noRelease, true |
| + } |
| + s.held[key] = n + 1 |
| + s.mu.Unlock() |
| + h := &shareHold{s: s, key: key} |
| + return h.release, true |
| +} |
| + |
| +// Held is the number of slots key holds. |
| +func (s *Shares) Held(key string) int { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return s.held[key] |
| +} |
| + |
| +type shareHold struct { |
| + s *Shares |
| + key string |
| + done bool // guarded by s.mu |
| +} |
| + |
| +func (h *shareHold) release() { |
| + s := h.s |
| + s.mu.Lock() |
| + if !h.done { |
| + h.done = true |
| + if n := s.held[h.key] - 1; n > 0 { |
| + s.held[h.key] = n |
| + } else { |
| + delete(s.held, h.key) |
| + } |
| + } |
| + s.mu.Unlock() |
| +} |
diff --git a/internal/ratelimit/shares_test.go b/internal/ratelimit/shares_test.go| new file mode 100644 |
| index 000000000..081423bee |
| --- /dev/null |
| +++ b/internal/ratelimit/shares_test.go |
| @@ -0,0 +1,104 @@ |
| +package ratelimit |
| + |
| +import ( |
| + "sync" |
| + "testing" |
| +) |
| + |
| +func TestSharesAcquire(t *testing.T) { |
| + s := NewShares(100) |
| + const k = "203.0.113.1" |
| + var rels []func() |
| + for i := range 5 { |
| + rel, ok := s.Acquire(k, false, 2) |
| + if !ok { |
| + t.Fatalf("not busy, acquire %d refused", i+1) |
| + } |
| + rels = append(rels, rel) |
| + } |
| + if got := s.Held(k); got != 5 { |
| + t.Fatalf("Held = %d, want 5", got) |
| + } |
| + if rel, ok := s.Acquire(k, true, 5); ok || rel == nil { |
| + t.Fatalf("busy at share: ok %v release %v, want refused with a non-nil no-op release", ok, rel != nil) |
| + } |
| + if got := s.Held(k); got != 5 { |
| + t.Fatalf("refusal changed Held to %d", got) |
| + } |
| + if _, ok := s.Acquire("203.0.113.2", true, 5); !ok { |
| + t.Fatal("busy, a key under its share was refused") |
| + } |
| + rels[0]() |
| + rels[0]() // idempotent |
| + if got := s.Held(k); got != 4 { |
| + t.Fatalf("Held after double release = %d, want 4", got) |
| + } |
| + rel, ok := s.Acquire(k, true, 5) |
| + if !ok { |
| + t.Fatal("busy, refused after a release freed a slot") |
| + } |
| + rel() |
| + for _, r := range rels[1:] { |
| + r() |
| + } |
| + if got := s.Held(k); got != 0 { |
| + t.Fatalf("Held = %d, want 0", got) |
| + } |
| +} |
| + |
| +func TestSharesMemoryBound(t *testing.T) { |
| + s := NewShares(3) |
| + var rels []func() |
| + for _, k := range []string{"a", "b", "c"} { |
| + rel, _ := s.Acquire(k, false, 1) |
| + rels = append(rels, rel) |
| + } |
| + // Full of holders: a new key goes through untracked. |
| + rel, ok := s.Acquire("d", true, 1) |
| + if !ok { |
| + t.Fatal("new key on a full Shares was refused") |
| + } |
| + if got := s.Held("d"); got != 0 { |
| + t.Fatalf("untracked key Held = %d, want 0", got) |
| + } |
| + rel() |
| + for _, r := range rels { |
| + r() |
| + } |
| + if n := s.tracked(); n != 0 { |
| + t.Fatalf("tracked = %d after every release, want 0", n) |
| + } |
| + // Released keys leave the map, so there is room again. |
| + rel, _ = s.Acquire("d", false, 1) |
| + if got := s.Held("d"); got != 1 { |
| + t.Fatalf("Held = %d, want 1", got) |
| + } |
| + rel() |
| +} |
| + |
| +func TestSharesRace(t *testing.T) { |
| + s := NewShares(8) |
| + keys := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"} |
| + var wg sync.WaitGroup |
| + for g := range 64 { |
| + wg.Go(func() { |
| + for i := range 1000 { |
| + k := keys[(g+i)%len(keys)] |
| + rel, _ := s.Acquire(k, i%2 == 0, 3) |
| + _ = s.Held(k) |
| + rel() |
| + rel() |
| + } |
| + }) |
| + } |
| + wg.Wait() |
| + if n := s.tracked(); n != 0 { |
| + t.Fatalf("tracked = %d after every release, want 0", n) |
| + } |
| +} |
| + |
| +func (s *Shares) tracked() int { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return len(s.held) |
| +} |