carlos / platform Public

alerts: a restarts rule for a platform unit that cannot stay up (#284)

Human

Prompt

Yeah, let's do it all, and then follow or same-time do suite debt and optimisation

From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.

Summary

The agent reads each platform unit's NRestarts from systemd every tick and fires a restarts alert when one unit restarts more than 3 times in 10 minutes; it clears after 10 minutes with none. RuleState and Event gain an optional detail naming the unit.

Agent notes

Alert sentences for the rule are not written yet; they go through copy review with the rest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 1 week ago 516ffbd25cdb048e60e9dcfc38a2a19159e6eede parent 940d59e

7 files changed, +362 −20

internal/host/alertsink_test.go +3 −3
internal/host/metrics_loop.go +27 −9
internal/host/metrics_loop_test.go +7 −7
internal/host/restartwatch.go +133 −0
internal/host/restartwatch_test.go +116 −0
internal/metrics/alert.go +34 −1
internal/metrics/alert_test.go +42 −0

CI failed — run details

CI log

=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	334.503s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.082s
ok  	github.com/carlosframework/platform/cmd/moveprobe	92.235s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.380s
ok  	github.com/carlosframework/platform/internal/activator	227.752s
ok  	github.com/carlosframework/platform/internal/agentsock	0.016s
ok  	github.com/carlosframework/platform/internal/alerting	0.003s
ok  	github.com/carlosframework/platform/internal/analytics	0.028s
ok  	github.com/carlosframework/platform/internal/appconfig	2.360s
ok  	github.com/carlosframework/platform/internal/apperr	0.082s
ok  	github.com/carlosframework/platform/internal/applog	0.134s
ok  	github.com/carlosframework/platform/internal/awsinv	0.004s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.775s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.008s
ok  	github.com/carlosframework/platform/internal/caps	0.011s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.713s
ok  	github.com/carlosframework/platform/internal/changelog	0.289s
ok  	github.com/carlosframework/platform/internal/clamd	0.055s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	363.049s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.499s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.009s
ok  	github.com/carlosframework/platform/internal/dbverify	0.004s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.005s
ok  	github.com/carlosframework/platform/internal/domains	2.210s
ok  	github.com/carlosframework/platform/internal/doorkey	0.006s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	26.044s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.120s
ok  	github.com/carlosframework/platform/internal/gate	1.655s
ok  	github.com/carlosframework/platform/internal/genselect	0.015s
ok  	github.com/carlosframework/platform/internal/geo	0.015s
ok  	github.com/carlosframework/platform/internal/geoip	0.030s
ok  	github.com/carlosframework/platform/internal/host	95.731s
ok  	github.com/carlosframework/platform/internal/hostactors	0.331s
ok  	github.com/carlosframework/platform/internal/hostbox	2.211s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.527s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.017s
ok  	github.com/carlosframework/platform/internal/hostexec	4.921s
ok  	github.com/carlosframework/platform/internal/hostfence	0.032s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.453s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.017s
ok  	github.com/carlosframework/platform/internal/hostguard	0.965s
ok  	github.com/carlosframework/platform/internal/hostops	1.473s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.017s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.017s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.450s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.135s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.017s
ok  	github.com/carlosframework/platform/internal/infraview	0.017s
ok  	github.com/carlosframework/platform/internal/instancectl	1.831s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.689s
ok  	github.com/carlosframework/platform/internal/instancefs	0.021s
ok  	github.com/carlosframework/platform/internal/instancekey	0.018s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.022s
ok  	github.com/carlosframework/platform/internal/layout	1.467s
ok  	github.com/carlosframework/platform/internal/lease	9.454s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.009s
ok  	github.com/carlosframework/platform/internal/ledger	0.013s
ok  	github.com/carlosframework/platform/internal/mail	0.169s
ok  	github.com/carlosframework/platform/internal/maintenance	0.083s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.566s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.080s
ok  	github.com/carlosframework/platform/internal/metrics	0.005s
ok  	github.com/carlosframework/platform/internal/moves	20.985s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.760s
ok  	github.com/carlosframework/platform/internal/objstore	0.158s
ok  	github.com/carlosframework/platform/internal/ops	0.162s
ok  	github.com/carlosframework/platform/internal/park	0.203s
ok  	github.com/carlosframework/platform/internal/privatefile	0.005s
2026/10/13 22:29:29 provision: listing apps/bab/blog/instances/: list refused — its records are unseen this pass
2026/10/13 22:29:29 provision: listing apps/bab/: listdirs refused — its records are unseen this pass
2026/10/13 22:29:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:29 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/13 22:29:29 provision: listing apps/bac/tito/instances/: list refused — its records are unseen this pass
2026/10/13 22:29:29 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:29 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/13 22:29:29 provision: remove retired instance route shop.example: provision: claim admission unavailable
2026/10/13 22:29:29 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/13 22:29:29 provision: remove retired sidecar route worker.sidecar.internal: provision: claim admission unavailable
2026/10/13 22:29:29 provision: claim install acme.go.tito.io: provision: claim admission unavailable
2026/10/13 22:29:29 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:29 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 22:29:29 provision: claim install bot.eleven.bac.sidecar.internal: provision: claim admission unavailable
2026/10/13 22:29:29 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 22:29:29 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/13 22:29:29 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 22:29:29 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: no sealed config
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 22:29:30 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 22:29:30 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 22:29:30 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 22:29:30 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 22:29:30 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: claim install acme.go.tito.io: registry: claim row ownership changed or unknown
2026/10/13 22:29:30 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 22:29:30 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["run"])
2026/10/13 22:29:30 provision: claim install bot.eleven.bac.sidecar.internal: registry: claim row ownership changed or unknown
2026/10/13 22:29:30 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:30 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:30 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 22:29:30 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/new.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 22:29:30 provision: minted instance route rl.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rg.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rh.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route ri.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rj.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rk.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rf.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route ra.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rb.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rc.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route rd.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/13 22:29:30 provision: skipping record apps/bac/tito/instances/rb.go.tito.io.json: read failed: read refused
2026/10/13 22:29:30 provision: minted instance route re.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appa.example (app "appa" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appb.example (app "appb" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appd.example (app "appd" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appg.example (app "appg" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appi.example (app "appi" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appc.example (app "appc" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appe.example (app "appe" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appf.example (app "appf" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.apph.example (app "apph" acct "bac")
2026/10/13 22:29:30 provision: minted instance route x.appj.example (app "appj" acct "bac")
2026/10/13 22:29:30 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:30 provision: minted instance route r0083.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0034.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0059.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0073.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0107.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0125.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0017.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0024.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0030.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0039.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0066.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0069.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0006.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0011.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0057.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0067.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0099.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0109.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0112.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0124.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0012.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0019.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0088.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0098.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0036.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0103.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0001.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0048.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0082.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0123.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0032.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0050.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0055.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0074.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0081.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0090.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0053.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0093.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0121.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0004.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0049.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0078.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0118.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0005.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0041.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0075.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0095.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0119.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0129.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0027.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0033.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0108.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0009.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0094.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0101.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0022.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0038.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0003.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0060.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0063.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0111.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0014.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0044.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0051.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0084.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0085.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0043.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0072.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0104.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0105.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0116.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0035.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0052.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0068.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0054.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0062.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0076.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0000.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0025.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0007.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0037.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0040.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0071.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0045.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0047.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0064.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0065.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0070.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0096.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0110.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0114.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0117.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0015.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0020.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0077.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0126.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0016.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0021.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0086.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0089.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0091.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0097.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0106.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0113.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0013.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0029.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0042.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0056.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0115.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0127.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0046.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0092.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0128.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0002.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0008.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0058.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0087.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0120.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0018.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0079.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0080.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0100.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0102.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0010.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0026.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0122.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0061.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0023.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0028.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:30 provision: minted instance route r0031.go.tito.io (app "tito" acct "bac")
--- FAIL: TestPlan14RootReadCensus (9.72s)
    plan14_root_reads_test.go:664: root read census: 10 discrepancies
        changed root read internal/host/units/carlos-litestream.service:external; review class, task, sites and digest
        changed root read internal/ops/ops.go:runLitestream; review class, task, sites and digest
        changed root source envelope internal/host
        changed root source envelope internal/host/units
        changed root source envelope internal/metrics
        changed root source envelope internal/ops
        unclassified root read internal/host/restartwatch.go:platformUnitRestarts file:Command@60
        unclassified root read internal/ops/ops.go:systemdDaemonReload file:Command@48
        changed legacy declaration internal/host/units/carlos-litestream.service:external
        changed legacy declaration internal/ops/ops.go:runLitestream
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: refusing to mint route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route svc.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route console.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route shared.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: converged acme.go.tito.io channel edge -> canary/rehearsal
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: empty channel
2026/10/13 22:29:41 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io environment "" -> "staging"
2026/10/13 22:29:41 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io steering "" -> "latency"
2026/10/13 22:29:41 provision: converged acme.go.tito.io steering "latency" -> ""
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io hibernate false -> true
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io hibernate true -> false (replication "daily" cleared: only a hibernating route's rung is read)
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged acme.go.tito.io status-named false -> true
2026/10/13 22:29:41 provision: converged acme.go.tito.io status-named true -> false
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route ha.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7002" -> "127.0.0.1:7001,127.0.0.1:7003" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 22:29:41 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7001,127.0.0.1:7003" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 22:29:41 provision: converged ha.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:7003,127.0.0.1:7001" -> "127.0.0.1:7003,127.0.0.1:7001" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "{\"path\":\"/healthz\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 22:29:41 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: converged whole.go.tito.io upstreams addr "127.0.0.1:9443" -> "", addrs "" -> "127.0.0.1:9443,127.0.0.1:9444" (probe "" -> "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}")
2026/10/13 22:29:41 provision: converged whole.go.tito.io upstreams addr "" -> "", addrs "127.0.0.1:9443,127.0.0.1:9444" -> "" (probe "{\"path\":\"/\",\"status\":200,\"interval_ms\":2000,\"fail_after\":3,\"rise_after\":2}" -> "")
2026/10/13 22:29:41 provision: whole.go.tito.io: record describes a different route SHAPE than the row (kind "instance"/"service" unit ""/"" socket "/run/carlos/whole.go.tito.io.sock"/"" db "/data/whole.go.tito.io.db"/"") — not converged in place; `carlos remove whole.go.tito.io` purges this route's capability grants (registry.Delete's contract, not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix
2026/10/13 22:29:41 provision: minted service route whole.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping accounts/bac/account.json: malformed account manifest: invalid character 'o' in literal null (expecting 'u')
2026/10/13 22:29:41 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: manifest bac/tito: read failed: provision: manifest bac/tito: invalid character 'o' in literal null (expecting 'u')
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: disagrees with its own path (host="wrong-host.go.tito.io" account="bac" app="tito")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/acme$evil.go.tito.io.json: host "acme$evil.go.tito.io" is not a hostname
2026/10/13 22:29:41 provision: not removing acme$evil.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/bad-service.go.tito.io.json: invalid shape: kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to
2026/10/13 22:29:41 provision: not removing bad-service.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/no-channel.go.tito.io.json: empty channel
2026/10/13 22:29:41 provision: not removing no-channel.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: skipping record apps/bac/tito/instances/acme.go.tito.io.json: malformed: provision: invalid maintenance projection or protected fields
2026/10/13 22:29:41 provision: not removing acme.go.tito.io: its record exists but could not be read cleanly this pass
2026/10/13 22:29:41 provision: removed retired instance route ghost.go.tito.io (record gone)
2026/10/13 22:29:41 provision: duplicate host dup.go.tito.io: keeping apps/bac/tito/instances/dup.go.tito.io.json, skipping apps/bad/tito/instances/dup.go.tito.io.json
2026/10/13 22:29:41 provision: minted instance route dup.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:41 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/13 22:29:41 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:41 provision: removed retired static route site.go.tito.io (app in the trash)
2026/10/13 22:29:41 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/13 22:29:41 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:41 provision: removed retired instance route acme.go.tito.io (app in the trash)
2026/10/13 22:29:41 provision: bac/tito is in the trash — not minting instance routes for it
2026/10/13 22:29:41 provision: trash marker for bac/tito unreadable: release: malformed trash marker for bac/tito: invalid character 'n' looking for beginning of object key string — not minting for this app
2026/10/13 22:29:41 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:41 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:41 provision: converged acme.go.tito.io channel canary/rehearsal -> edge
2026/10/13 22:29:41 provision: converged acme.go.tito.io environment "staging" -> ""
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (backing "unit" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (tunnel "pi-fleet" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (caps "port:25" (box-scoped, not expressible in a record)) — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (platform (owned by the domains reconcile pass)) — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (canary (owned by the domains reconcile pass)) — leaving it alone
2026/10/13 22:29:41 provision: acme.go.tito.io exists but is not the route this record describes (db "/data/other.db"/"/data/acme.go.tito.io.db") — leaving it alone
2026/10/13 22:29:41 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:41 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 22:29:41 provision: dry-run: would converge owned.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: dry-run: would converge owned.go.tito.io environment "" -> "staging"
2026/10/13 22:29:41 provision: dry-run: would converge owned.go.tito.io hibernate false -> true
2026/10/13 22:29:41 provision: dry-run: would converge owned.go.tito.io seed
2026/10/13 22:29:41 provision: dry-run: would mint instance route acme.go.tito.io (app "tito" acct "bac" channel "stable")
2026/10/13 22:29:41 provision: dry-run: would adopt existing route adopt.go.tito.io (app "tito" acct "bac") — this record would own its channel and environment
2026/10/13 22:29:41 provision: dry-run: would converge adopt.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: dry-run: would converge adopt.go.tito.io environment "" -> "staging"
2026/10/13 22:29:41 provision: dry-run: would remove retired instance route gone.go.tito.io (record gone)
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: adopted existing route adopt.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:41 provision: converged adopt.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: converged adopt.go.tito.io environment "" -> "staging"
2026/10/13 22:29:41 provision: converged owned.go.tito.io channel stable -> edge
2026/10/13 22:29:41 provision: converged owned.go.tito.io environment "" -> "staging"
2026/10/13 22:29:41 provision: converged owned.go.tito.io hibernate false -> true
2026/10/13 22:29:41 provision: removed retired instance route gone.go.tito.io (record gone)
2026/10/13 22:29:41 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: adopted existing route story.carlosframework.com (app "story" acct "bad") — this record now owns its channel and environment
2026/10/13 22:29:41 provision: story.carlosframework.com exists but is not the route this record describes (kind "instance"/"static") — leaving it alone
2026/10/13 22:29:41 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route pub.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route pubs.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route loop.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route named.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted service route priv.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route unit.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted instance route exec.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted static route pisite.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: minted static route site.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:41 provision: removed retired static route site.go.tito.io (record gone)
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: converged acme.go.tito.io replicas "ie,uk" -> "uk"
2026/10/13 22:29:42 provision: converged acme.go.tito.io replicas "uk" -> ""
2026/10/13 22:29:42 provision: minted instance route acme-uk.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: converged acme-uk.go.tito.io replica mark true -> false
2026/10/13 22:29:42 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: acme.go.tito.io: row is on the nested layout (db "/data/instances/acme.go.tito.io/acme.go.tito.io.db" socket "/run/carlos/instances/acme.go.tito.io/app.sock"); the record's shared paths are not drift
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route router.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: router.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/13 22:29:42 provision: acme.go.tito.io: not minting this pass: the box's layout intent could not be read: store down
2026/10/13 22:29:42 provision: adopted existing route acme.go.tito.io (app "tito" acct "bac") — this record now owns its channel and environment
2026/10/13 22:29:42 provision: converged acme.go.tito.io channel stable -> edge
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/13 22:29:42 provision: not removing acme.go.tito.io (record gone): participant of a running operation
2026/10/13 22:29:42 provision: removed retired instance route acme.go.tito.io (record gone)
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
2026/10/13 22:29:42 provision: refusing to mint held route acme.go.tito.io (app "tito" acct "bac"): this box declares CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic
2026/10/13 22:29:42 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/13 22:29:42 provision: acme.go.tito.io: held restoring by rb_other, not by mv_acme; leaving it
2026/10/13 22:29:42 provision: acme.go.tito.io: the row is not this pass's (app "other" acct ""); not holding it for mv_acme
2026/10/13 22:29:42 provision: acme.go.tito.io exists but is not the route this record describes (app "other"/"tito") — leaving it alone
2026/10/13 22:29:42 provision: dry-run: would mint instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac" channel "stable")
2026/10/13 22:29:42 provision: acme.go.tito.io: not held landing by mv_acme: the row's hold changed under this pass; the next pass judges it afresh
2026/10/13 22:29:42 provision: held acme.go.tito.io landing by mv_acme (was ready)
2026/10/13 22:29:42 provision: removed retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: converged sidecar bot.eleven.bac.sidecar.internal args ["sidecar","run"] -> ["worker","run"]
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: converged sidecar bot.eleven.bac.sidecar.internal ceilings 300/3600 -> 60/120
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: converged sidecar bot.eleven.bac.sidecar.internal channel stable -> canary/eleven
2026/10/13 22:29:42 provision: converged sidecar bot.eleven.bac.sidecar.internal environment "other" -> "relay"
2026/10/13 22:29:42 provision: converged sidecar bot.eleven.bac.sidecar.internal hibernate true -> false
2026/10/13 22:29:42 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: read failed: failStore: reads disabled under apps/bac/eleven/sidecars/
2026/10/13 22:29:42 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: malformed: invalid character 'n' looking for beginning of object key string
2026/10/13 22:29:42 provision: not removing sidecar bot.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/13 22:29:42 provision: skipping sidecar record apps/bac/eleven/sidecars/bot.json: empty channel
2026/10/13 22:29:42 provision: skipping sidecar record apps/bac/eleven/sidecars/held.json: empty channel
2026/10/13 22:29:42 provision: not removing sidecar held.eleven.bac.sidecar.internal: its record exists but could not be read cleanly this pass
2026/10/13 22:29:42 provision: removed retired sidecar route old.eleven.bac.sidecar.internal (record gone)
2026/10/13 22:29:42 provision: bac/eleven has sidecar records but no sidecars manifest — not minting
2026/10/13 22:29:42 provision: bac/eleven is in the trash — not minting sidecar routes for it
2026/10/13 22:29:42 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: sidecar hub.relay.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: minted sidecar route hub.relay.bac.sidecar.internal (app "relay" acct "bac" args ["relay"])
2026/10/13 22:29:42 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: removed retired sidecar route bot.eleven.bac.sidecar.internal (record gone)
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: sidecar bot.eleven.bac.sidecar.internal is not placeable here: it asks for "no-inbound" placement and this box accepts inbound work (CARLOS_INBOUND="")
2026/10/13 22:29:42 provision: skipping sidecar record apps/bac/relay/sidecars/hub.json: read failed: failStore: reads disabled under apps/bac/relay/sidecars/
2026/10/13 22:29:42 provision: REFUSING to mint sidecar bot.eleven.bac.sidecar.internal: bot.eleven.bac.sidecar.internal-sidecar.env is staged on this box — the legacy file co-spawn and a native sidecar row are two mechanisms for one host; remove the file to migrate this host onto sidecar records
2026/10/13 22:29:42 provision: minted sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" args ["sidecar","run"])
2026/10/13 22:29:42 provision: dry-run: would mint sidecar route bot.eleven.bac.sidecar.internal (app "eleven" acct "bac" channel "stable" args ["sidecar","run"])
2026/10/13 22:29:42 provision: dry-run: would remove retired sidecar route gone.eleven.bac.sidecar.internal (record gone)
2026/10/13 22:29:42 provision: bot.eleven.bac.sidecar.internal exists but is not a sidecar row this pass owns (kind "sidecar" provisioned false app "eleven" acct "bac") — leaving it alone
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: converged acme.go.tito.io update settings "{\"updates\":\"manual\",\"channel\":\"stable\"}" -> ""
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 22:29:42 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 22:29:42 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io (app "tito" acct "bac")
2026/10/13 22:29:42 provision: manifest bac/tito: read failed: provision: manifest bac/tito: unexpected end of JSON input
2026/10/13 22:29:42 provision: acme.go.tito.io: not minted this pass: the app's update policy could not be read
2026/10/13 22:29:42 provision: minted instance route acme.go.tito.io held landing by mv_acme (app "tito" acct "bac")
FAIL
FAIL	github.com/carlosframework/platform/internal/provision	13.064s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.007s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	4.032s
ok  	github.com/carlosframework/platform/internal/relay	7.221s
ok  	github.com/carlosframework/platform/internal/release	19.889s
ok  	github.com/carlosframework/platform/internal/rollout	3.739s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.003s
ok  	github.com/carlosframework/platform/internal/rootaws	0.007s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.006s
ok  	github.com/carlosframework/platform/internal/rootstate	0.024s
ok  	github.com/carlosframework/platform/internal/s3compat	0.004s
ok  	github.com/carlosframework/platform/internal/safefile	0.003s
ok  	github.com/carlosframework/platform/internal/schedule	2.308s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	9.398s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.224s
ok  	github.com/carlosframework/platform/internal/status	0.039s
ok  	github.com/carlosframework/platform/internal/steering	0.006s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.204s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.002s
ok  	github.com/carlosframework/platform/internal/updatewords	0.004s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.002s
ok  	github.com/carlosframework/platform/internal/vet	1.561s
FAIL
make: *** [Makefile:18: test] Error 1

amadan: step gate failed: exit status 2

amadan: job failed at step gate — its output is above, not at the tail of this log

diff --git a/internal/host/alertsink_test.go b/internal/host/alertsink_test.go

index bd162e674..d44bea351 100644

--- a/internal/host/alertsink_test.go

+++ b/internal/host/alertsink_test.go

@@ -241,10 +241,10 @@ func TestAlertTickDeliversThroughSinks(t *testing.T) {

disk := 0.95

latest := metrics.Sample{At: sinkTestAt, Host: metrics.HostSample{DiskFrac: &disk}}

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, sinks); err != nil {

t.Fatal(err)

}

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, sinks); err != nil {

t.Fatal(err)

}

if len(pub.in) != 1 {

@@ -257,7 +257,7 @@ func TestAlertTickDeliversThroughSinks(t *testing.T) {

// Clearing is a transition too.

low := 0.30

cleared := metrics.Sample{At: sinkTestAt.Add(time.Minute), Host: metrics.HostSample{DiskFrac: &low}}

- if err := alertTick(ctx, store, box, rules, st, cleared, statePath, sinks); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, cleared, alertReadings{}, statePath, sinks); err != nil {

t.Fatal(err)

}

if len(pub.in) != 2 || !strings.Contains(deref(pub.in[1].Subject), "cleared") {

diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go

index 2e0dad037..a52b6c7c4 100644

--- a/internal/host/metrics_loop.go

+++ b/internal/host/metrics_loop.go

@@ -33,6 +33,9 @@ type sampleProbes struct {

memFrac func() *float64

diskFrac func() *float64

units func() map[string]unitRaw

+ // restarts is each platform unit's systemd restart counter (see

+ // restartWatch). A nil func, or a nil map from it, is "could not ask".

+ restarts func() map[string]int64

now func() time.Time

}

@@ -481,16 +484,18 @@ func pruneStaleRaw(ctx context.Context, s release.Store, box string, now time.Ti

// rules (disk, lag; sustain 0) retry the same way, just starting from

// their very first over-threshold tick instead of a sustain deadline.

func alertTick(ctx context.Context, s release.Store, box string, rules metrics.Rules,

- st *metrics.AlertState, latest metrics.Sample, statePath string,

+ st *metrics.AlertState, latest metrics.Sample, extra alertReadings, statePath string,

sinks *alertSinks) error {

obs := metrics.Observation{

- At: latest.At,

- CPUFrac: latest.Host.CPUFrac,

- MemFrac: latest.Host.MemFrac,

- DiskFrac: latest.Host.DiskFrac,

- LagSecs: nil, // v1: no lag reading feeds the agent loop yet

- GeoIP: geoIPAlertReading(latest.At),

+ At: latest.At,

+ CPUFrac: latest.Host.CPUFrac,

+ MemFrac: latest.Host.MemFrac,

+ DiskFrac: latest.Host.DiskFrac,

+ LagSecs: nil, // v1: no lag reading feeds the agent loop yet

+ GeoIP: geoIPAlertReading(latest.At),

+ Restarts: extra.restarts,

+ RestartsUnit: extra.restartsUnit,

}

newState, events := metrics.Evaluate(rules, *st, obs)

@@ -524,6 +529,15 @@ func alertTick(ctx context.Context, s release.Store, box string, rules metrics.R

return publishErr

}

+// alertReadings are the readings alertTick takes that are not part of a

+// metrics.Sample: they describe the box's units rather than its resources,

+// and are not stored in the raw chunks. The zero value is "no reading", which

+// leaves every rule they feed exactly as it was.

+type alertReadings struct {

+ restarts *float64 // see restartWatch.tick

+ restartsUnit string

// persistAlertState writes st to path as indented JSON via the repo's

// standard temp+rename idiom (internal/appconfig/sync.go's renderEnv,

// internal/release/store.go's DirStore.Put): a crashed write must never

@@ -616,7 +630,8 @@ func realSampleProbes(execRoot string, stampedUnits func() []string) sampleProbe

}

return realUnitReadings(execRoot, stamped)

- now: time.Now,

+ restarts: platformUnitRestarts,

+ now: time.Now,

}

@@ -803,6 +818,7 @@ func metricsLoop(execRoot string, stampedUnits func() []string, sinks *alertSink

// alerted is what keeps a standing refusal from paging every minute.

// dataDir() is the bake dir the agent hands release.BakeDir.

var refusals refusalWatch

+ var restarts restartWatch

for {

time.Sleep(60 * time.Second)

tick++

@@ -812,7 +828,9 @@ func metricsLoop(execRoot string, stampedUnits func() []string, sinks *alertSink

sm.tick(probes)

latest := sm.buf[len(sm.buf)-1]

- if err := alertTick(ctx, s, box, rules, &st, latest, statePath, sinks); err != nil {

+ var extra alertReadings

+ extra.restarts, extra.restartsUnit = restarts.tick(latest.At, probes.restarts())

+ if err := alertTick(ctx, s, box, rules, &st, latest, extra, statePath, sinks); err != nil {

log.Printf("agent: alert tick: %v", err)

}

diff --git a/internal/host/metrics_loop_test.go b/internal/host/metrics_loop_test.go

index 6ce15716a..2c0a19c5a 100644

--- a/internal/host/metrics_loop_test.go

+++ b/internal/host/metrics_loop_test.go

@@ -143,7 +143,7 @@ func TestMetricsLoopStoreHonorsFleetPrefix(t *testing.T) {

firing := 0.95

latest := metrics.Sample{At: at, Host: metrics.HostSample{DiskFrac: &firing}}

statePath := filepath.Join(t.TempDir(), "monitor", "alerts.json")

- if err := alertTick(ctx, s, box, rules, st, latest, statePath, nil); err != nil {

+ if err := alertTick(ctx, s, box, rules, st, latest, alertReadings{}, statePath, nil); err != nil {

t.Fatal(err)

}

if _, _, err := under.Get(ctx, "fleets/pi/"+metrics.AlertKey(box)); err != nil {

@@ -222,7 +222,7 @@ func TestAlertTickPersistsPublishesAndPosts(t *testing.T) {

return nil

}

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

t.Fatal(err)

}

@@ -248,7 +248,7 @@ func TestAlertTickPersistsPublishesAndPosts(t *testing.T) {

}

// Second, identical tick: already firing, no new transition -> no post.

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

t.Fatal(err)

}

if len(posts) != 1 {

@@ -274,7 +274,7 @@ func TestAlertTickWebhookFailureDoesNotBlock(t *testing.T) {

}

post := func(url string, body []byte) error { return errors.New("webhook unreachable") }

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

t.Fatalf("alertTick must not fail on a webhook error: %v", err)

}

if _, err := os.Stat(statePath); err != nil {

@@ -316,7 +316,7 @@ func TestAlertTickPersistFailureDoesNotSuppressPublish(t *testing.T) {

return nil

}

- if err := alertTick(ctx, store, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

+ if err := alertTick(ctx, store, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

t.Fatalf("alertTick must not fail just because the local state write failed: %v", err)

}

if !posted {

@@ -550,7 +550,7 @@ func TestAlertTickRetriesTransitionAfterPublishFailure(t *testing.T) {

}

failStore := &failingPutStore{Store: release.NewDirStore(t.TempDir())}

- if err := alertTick(ctx, failStore, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err == nil {

+ if err := alertTick(ctx, failStore, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err == nil {

t.Fatal("expected alertTick to return the publish error against a Put-always-fails store")

}

if st.Rules["disk"].Firing {

@@ -563,7 +563,7 @@ func TestAlertTickRetriesTransitionAfterPublishFailure(t *testing.T) {

// Retry with the SAME observation against a working store: the pending

// transition must reproduce, publish, and post again.

workingStore := release.NewDirStore(t.TempDir())

- if err := alertTick(ctx, workingStore, box, rules, st, latest, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

+ if err := alertTick(ctx, workingStore, box, rules, st, latest, alertReadings{}, statePath, webhookSinks("http://example.test/webhook", post)); err != nil {

t.Fatalf("retry against a working store: %v", err)

}

if !st.Rules["disk"].Firing {

diff --git a/internal/host/restartwatch.go b/internal/host/restartwatch.go

new file mode 100644

index 000000000..14b89822d

--- /dev/null

+++ b/internal/host/restartwatch.go

@@ -0,0 +1,133 @@

+package host

+// A unit that cannot stay up looks healthy from every other angle the agent

+// has. systemd restarts it, the sampler finds its cgroup on most ticks, and

+// its memory and CPU read as modest because it never lives long enough to use

+// either. Discussion #284 is that case: carlos-litestream was OOM-killed four

+// seconds into every start for two hours, restarted over 900 times, and the

+// only thing that noticed was a person asking whether backups were running.

+//

+// systemd counts those restarts itself (NRestarts), so the agent asks it once

+// a tick and alerts on how fast the number is moving.

+import (

+ "os/exec"

+ "strconv"

+ "strings"

+ "time"

+// restartWindow is how far back a unit's restarts are counted. The restarts

+// rule fires on a count inside it (metrics.Rules.RestartCount) and clears

+// after the whole of it passes with none.

+const restartWindow = 10 * time.Minute

+// restartWatch remembers each watched unit's restart counter over

+// restartWindow. It lives for the life of the metrics loop; an agent restart

+// starts it empty, which is why a reading of zero waits for a full window.

+type restartWatch struct {

+ marks map[string][]restartMark

+ // quietSince is when the watch began, or last saw any unit restart or

+ // have its counter reset, whichever is later.

+ quietSince time.Time

+type restartMark struct {

+ at time.Time

+ n int64

+// tick records counts (unit name to systemd's NRestarts) as of now and

+// returns the most restarts any one unit made inside the window, with that

+// unit's name. nil counts means systemd could not be asked: the reading is

+// nil and the history is kept.

+//

+// A zero reading is only given after a whole window in which the watch saw

+// nothing restart. Zero is what clears a firing alert, and neither a watch

+// that has just started nor a unit an operator has just restarted by hand has

+// shown that the loop is over.

+func (w *restartWatch) tick(now time.Time, counts map[string]int64) (reading *float64, unit string) {

+ if counts == nil {

+ return nil, ""

+ }

+ if w.marks == nil {

+ w.marks = map[string][]restartMark{}

+ w.quietSince = now

+ }

+ for name := range w.marks {

+ if _, ok := counts[name]; !ok {

+ delete(w.marks, name) // unloaded: nothing left to watch

+ }

+ var worst int64

+ for name, n := range counts {

+ marks := w.marks[name]

+ if last := len(marks) - 1; last >= 0 && n != marks[last].n {

+ w.quietSince = now

+ if n < marks[last].n {

+ // The counter went backwards: systemd zeroes it when the unit

+ // is started by hand, so what came before is another run's

+ // history.

+ marks = nil

+ }

+ marks = append(marks, restartMark{at: now, n: n})

+ // Keep one mark at or beyond the window's edge as the baseline.

+ for len(marks) > 1 && now.Sub(marks[1].at) >= restartWindow {

+ marks = marks[1:]

+ }

+ w.marks[name] = marks

+ if d := n - marks[0].n; d > worst || (d == worst && d > 0 && name < unit) {

+ worst, unit = d, name

+ }

+ if worst == 0 && now.Sub(w.quietSince) < restartWindow {

+ return nil, ""

+ }

+ v := float64(worst)

+ return &v, unit

+// platformUnitRestarts asks systemd for the restart counter of every loaded

+// platform service. nil when systemd cannot be asked.

+//

+// Template instances (carlos-app@<host>, carlos-instance@<id>) are left out:

+// those are tenants' processes, and a tenant's crash loop is not the fleet

+// operator's page.

+func platformUnitRestarts() map[string]int64 {

+ out, err := exec.Command("systemctl", "show", "--property=Id", "--property=NRestarts", "carlos-*.service").Output()

+ if err != nil {

+ return nil

+ }

+ return parseUnitRestarts(string(out))

+// parseUnitRestarts reads `systemctl show --property=Id --property=NRestarts`

+// output: one blank-line-separated block per unit, the two properties in

+// whatever order systemd prints them. A block missing either is skipped.

+// Never nil: no loaded units is an answer, an empty one.

+func parseUnitRestarts(out string) map[string]int64 {

+ counts := map[string]int64{}

+ for _, block := range strings.Split(out, "\n\n") {

+ id, n, haveN := "", int64(0), false

+ for _, line := range strings.Split(block, "\n") {

+ k, v, ok := strings.Cut(strings.TrimSpace(line), "=")

+ if !ok {

+ continue

+ }

+ switch k {

+ case "Id":

+ id = v

+ case "NRestarts":

+ if parsed, err := strconv.ParseInt(v, 10, 64); err == nil {

+ n, haveN = parsed, true

+ }

+ if id == "" || !haveN || strings.Contains(id, "@") {

+ continue

+ }

+ counts[id] = n

+ }

+ return counts

diff --git a/internal/host/restartwatch_test.go b/internal/host/restartwatch_test.go

new file mode 100644

index 000000000..a58129801

--- /dev/null

+++ b/internal/host/restartwatch_test.go

@@ -0,0 +1,116 @@

+package host

+import (

+ "testing"

+ "time"

+// The #284 shape: one unit restarting every few seconds among quiet ones. The

+// reading is that unit's restarts inside the window, with its name.

+func TestRestartWatchCountsTheWorstUnitInsideTheWindow(t *testing.T) {

+ var w restartWatch

+ t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)

+ counts := func(ls, relay int64) map[string]int64 {

+ return map[string]int64{"carlos-litestream.service": ls, "carlos-relay.service": relay, "carlos-edge.service": 0}

+ }

+ // First sight of a counter is a baseline, not a restart: a unit that has

+ // restarted 900 times before the agent looked has not restarted "now".

+ if r, _ := w.tick(t0, counts(900, 2)); r != nil {

+ t.Fatalf("first tick gave a reading of %v, want none", *r)

+ }

+ r, unit := w.tick(t0.Add(time.Minute), counts(909, 2))

+ if r == nil || *r != 9 || unit != "carlos-litestream.service" {

+ t.Fatalf("after one minute: reading=%v unit=%q, want 9 restarts of carlos-litestream.service", r, unit)

+ }

+ r, unit = w.tick(t0.Add(2*time.Minute), counts(912, 3))

+ if r == nil || *r != 12 || unit != "carlos-litestream.service" {

+ t.Fatalf("after two minutes: reading=%v unit=%q, want 12", r, unit)

+ }

+ // The loop stops. Restarts age out of the window one tick at a time, and

+ // zero arrives only when the whole window has been quiet.

+ var last *float64

+ for m := 3; m <= 11; m++ {

+ last, _ = w.tick(t0.Add(time.Duration(m)*time.Minute), counts(912, 3))

+ if last == nil || *last == 0 {

+ t.Fatalf("minute %d: reading=%v, but minute 2's restarts are still inside the window", m, last)

+ }

+ last, _ = w.tick(t0.Add(12*time.Minute), counts(912, 3))

+ if last == nil || *last != 0 {

+ t.Fatalf("minute 12: reading=%v, want 0 after a quiet window", last)

+ }

+// Zero clears a firing alert, so a watch that has only just started (the

+// agent restarted mid-incident) must not say it.

+func TestRestartWatchGivesNoZeroBeforeAFullWindow(t *testing.T) {

+ var w restartWatch

+ t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)

+ quiet := map[string]int64{"carlos-litestream.service": 40}

+ for m := 0; m < 10; m++ {

+ if r, _ := w.tick(t0.Add(time.Duration(m)*time.Minute), quiet); r != nil {

+ t.Fatalf("minute %d: reading=%v before the watch has seen a whole window", m, *r)

+ }

+ if r, _ := w.tick(t0.Add(10*time.Minute), quiet); r == nil || *r != 0 {

+ t.Fatalf("minute 10: reading=%v, want 0", r)

+ }

+// systemd zeroes NRestarts when a unit is started by hand. That is neither a

+// negative number of restarts nor proof the loop is over.

+func TestRestartWatchTreatsACounterResetAsANewRun(t *testing.T) {

+ var w restartWatch

+ t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)

+ at := func(m int) time.Time { return t0.Add(time.Duration(m) * time.Minute) }

+ unit := func(n int64) map[string]int64 { return map[string]int64{"carlos-litestream.service": n} }

+ w.tick(at(0), unit(500))

+ if r, _ := w.tick(at(1), unit(510)); r == nil || *r != 10 {

+ t.Fatalf("reading=%v, want 10", r)

+ }

+ // An operator restarts it by hand: the counter is 0 again.

+ if r, _ := w.tick(at(2), unit(0)); r != nil {

+ t.Fatalf("a counter reset gave a reading of %v, want none", *r)

+ }

+ // Still broken: it loops again from zero, and is counted from zero.

+ if r, _ := w.tick(at(3), unit(5)); r == nil || *r != 5 {

+ t.Fatalf("after the reset: reading=%v, want 5", r)

+ }

+// systemd unavailable is "could not ask": no reading, and what the watch

+// already knew is kept for when it can ask again.

+func TestRestartWatchKeepsItsHistoryWhenSystemdCannotBeAsked(t *testing.T) {

+ var w restartWatch

+ t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)

+ w.tick(t0, map[string]int64{"carlos-litestream.service": 1})

+ if r, _ := w.tick(t0.Add(time.Minute), nil); r != nil {

+ t.Fatalf("reading=%v with no answer from systemd", *r)

+ }

+ if r, _ := w.tick(t0.Add(2*time.Minute), map[string]int64{"carlos-litestream.service": 7}); r == nil || *r != 6 {

+ t.Fatalf("reading=%v, want 6 against the baseline from before the gap", r)

+ }

+func TestParseUnitRestarts(t *testing.T) {

+ // Verbatim shape from systemd 257: NRestarts before Id, a blank line

+ // between units.

+ out := "NRestarts=0\nId=carlos-edge.service\n\nNRestarts=912\nId=carlos-litestream.service\n\n" +

+ "NRestarts=3\nId=carlos-app@shop.example.com.service\n\nId=carlos-adopt.service\n\nNRestarts=x\nId=carlos-relay.service\n"

+ got := parseUnitRestarts(out)

+ want := map[string]int64{"carlos-edge.service": 0, "carlos-litestream.service": 912}

+ if len(got) != len(want) {

+ t.Fatalf("got %v, want %v (a tenant's unit, a block with no counter and one with a bad counter are all skipped)", got, want)

+ }

+ for k, v := range want {

+ if got[k] != v {

+ t.Errorf("%s = %d, want %d", k, got[k], v)

+ }

+ if got := parseUnitRestarts(""); got == nil || len(got) != 0 {

+ t.Fatalf("no loaded units = %v, want an empty map, not nil", got)

+ }

diff --git a/internal/metrics/alert.go b/internal/metrics/alert.go

index 24a404138..489952dc4 100644

--- a/internal/metrics/alert.go

+++ b/internal/metrics/alert.go

@@ -19,6 +19,10 @@ type Rules struct {

// GeoIPSustain is how long a geoip reason must hold before the rule

// fires (geoip spec §10.4); it clears on the first reading of 0.

GeoIPSustain time.Duration // default 30 * time.Minute

+ // RestartCount is how many restarts one unit may make inside the

+ // observation's window before the restarts rule fires. It is a count, not

+ // a fraction, and the rule clears only on a reading of 0.

+ RestartCount float64 // fire >, default 3

}

// DefaultRules are the thresholds shipped with the platform.

@@ -33,6 +37,7 @@ func DefaultRules() Rules {

ClearBy: 0.05,

// GeoIPSustain: geoip spec §10.4.

GeoIPSustain: 30 * time.Minute,

+ RestartCount: 3,

}

@@ -86,6 +91,12 @@ type Observation struct {

// 1 stale, 2 expiring, 3 expired, 4 not loaded, 5 feed failing. nil on

// a box whose edge has not published one.

GeoIP *float64

+ // Restarts is the most restarts any one watched unit made inside the

+ // reporter's window, and RestartsUnit names that unit. A reading of 0

+ // says a whole window passed with none; nil says the reporter could not

+ // tell yet.

+ Restarts *float64

+ RestartsUnit string

}

// RuleState is one rule's sustain/hysteresis bookkeeping. Since and FiredAt

@@ -103,6 +114,9 @@ type RuleState struct {

Since time.Time `json:"since,omitempty"` // continuously-above-threshold since (sustain tracking)

FiredAt time.Time `json:"fired_at,omitempty"` // set on fire, cleared on clear

Value float64 `json:"value,omitempty"` // reading at last transition

+ // Detail names what the reading was about, for the rules whose value

+ // alone does not say: the unit for "restarts". Empty for the rest.

+ Detail string `json:"detail,omitempty"`

}

// AlertState is the JSON persisted to disk by the agent loop and published

@@ -112,7 +126,7 @@ type AlertState struct {

V int `json:"v"` // 1

Box string `json:"box"`

UpdatedAt time.Time `json:"updated_at"`

- Rules map[string]RuleState `json:"rules"` // "disk","mem","cpu","lag"

+ Rules map[string]RuleState `json:"rules"` // "disk","mem","cpu","lag","geoip","restarts"

}

// Event is emitted on a firing or clearing transition — never on every

@@ -122,6 +136,7 @@ type Event struct {

Firing bool `json:"firing"` // true=fired, false=cleared

Value float64 `json:"value"`

Box string `json:"box"`

+ Detail string `json:"detail,omitempty"` // see RuleState.Detail

}

// AlertKey is the bucket key an AlertState for box is stored/published

@@ -274,5 +289,23 @@ func Evaluate(r Rules, st AlertState, obs Observation) (AlertState, []Event) {

}

+ // restarts: a count inside a window the reporter keeps, so there is no

+ // sustain here. It fires above RestartCount and clears only at 0, which is

+ // a whole window without a restart: a unit that has merely slowed down

+ // (systemd backs its restarts off) has not recovered.

+ if obs.Restarts != nil {

+ val := *obs.Restarts

+ cur := out.Rules["restarts"]

+ switch {

+ case cur.Firing && val == 0:

+ unit := cur.Detail

+ out.Rules["restarts"] = RuleState{}

+ events = append(events, Event{Rule: "restarts", Firing: false, Value: val, Box: st.Box, Detail: unit})

+ case !cur.Firing && val > r.RestartCount:

+ out.Rules["restarts"] = RuleState{Firing: true, Since: obs.At, FiredAt: obs.At, Value: val, Detail: obs.RestartsUnit}

+ events = append(events, Event{Rule: "restarts", Firing: true, Value: val, Box: st.Box, Detail: obs.RestartsUnit})

+ }

return out, events

}

diff --git a/internal/metrics/alert_test.go b/internal/metrics/alert_test.go

index e4f55fa2a..92e0a6d4c 100644

--- a/internal/metrics/alert_test.go

+++ b/internal/metrics/alert_test.go

@@ -247,3 +247,45 @@ func TestGeoIPReasonChangeWhileFiring(t *testing.T) {

t.Fatalf("an unchanged reason re-fired: %v", ev)

}

+// The restarts rule fires above the count, names the unit, holds while the

+// unit is still restarting (however slowly), and clears only on a reading of

+// zero. A nil reading changes nothing in either direction.

+func TestRestartsRuleFiresAboveTheCountAndClearsOnlyAtZero(t *testing.T) {

+ r := DefaultRules()

+ t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)

+ f := func(v float64) *float64 { return &v }

+ st := AlertState{V: 1, Box: "b"}

+ st, ev := Evaluate(r, st, Observation{At: t0, Restarts: f(3), RestartsUnit: "carlos-litestream.service"})

+ if len(ev) != 0 || st.Rules["restarts"].Firing {

+ t.Fatalf("fired at the count, not above it: %v", ev)

+ }

+ st, ev = Evaluate(r, st, Observation{At: t0.Add(time.Minute), Restarts: f(4), RestartsUnit: "carlos-litestream.service"})

+ if len(ev) != 1 || !ev[0].Firing || ev[0].Rule != "restarts" || ev[0].Value != 4 || ev[0].Detail != "carlos-litestream.service" {

+ t.Fatalf("above the count: %+v", ev)

+ }

+ if rs := st.Rules["restarts"]; !rs.Firing || rs.Detail != "carlos-litestream.service" || !rs.FiredAt.Equal(t0.Add(time.Minute)) {

+ t.Fatalf("state after firing: %+v", rs)

+ }

+ // Still restarting, now once a minute, and another unit is the worst.

+ st, ev = Evaluate(r, st, Observation{At: t0.Add(2 * time.Minute), Restarts: f(1), RestartsUnit: "carlos-relay.service"})

+ if len(ev) != 0 || !st.Rules["restarts"].Firing || st.Rules["restarts"].Detail != "carlos-litestream.service" {

+ t.Fatalf("a slower loop cleared or renamed the alert: %v %+v", ev, st.Rules["restarts"])

+ }

+ st, ev = Evaluate(r, st, Observation{At: t0.Add(3 * time.Minute)})

+ if len(ev) != 0 || !st.Rules["restarts"].Firing {

+ t.Fatalf("a nil reading changed the rule: %v", ev)

+ }

+ st, ev = Evaluate(r, st, Observation{At: t0.Add(13 * time.Minute), Restarts: f(0)})

+ if len(ev) != 1 || ev[0].Firing || ev[0].Detail != "carlos-litestream.service" {

+ t.Fatalf("a zero reading must clear and name the unit that had fired: %+v", ev)

+ }

+ if rs := st.Rules["restarts"]; rs.Firing || rs.Detail != "" || !rs.FiredAt.IsZero() {

+ t.Fatalf("state after clearing: %+v", rs)

+ }

+ // Quiet stays quiet.

+ if _, ev = Evaluate(r, st, Observation{At: t0.Add(14 * time.Minute), Restarts: f(0)}); len(ev) != 0 {

+ t.Fatalf("a quiet rule emitted: %v", ev)

+ }