amadan

carlos / platform Public

Merge pull request #57 from carlosframework/worktree-remote-host-agent

Remote host-agent: tunnel route backing, heartbeat metrics, off-cloud bootstrap (#53)
Paul Campbell d6e7d08b09ad5cd03c0e3578eb700468d4f82cd9 parent 28874d5parent 2181507
33 files changed, +4860 −60
  • CLAUDE.md +1 −1
  • cli.go +207 −16
  • docs/superpowers/plans/2026-08-04-pi-enablement.md +247 −0
  • docs/superpowers/plans/2026-08-04-remote-host-agent.md +1261 −0
  • docs/superpowers/specs/2026-08-04-remote-host-pi-design.md +267 −0
  • go.mod +1 −1
  • infra/remote-host.md +95 −0
  • internal/console/server.go +28 −1
  • internal/console/server_test.go +127 −0
  • internal/console/templates/fleet.html +36 −0
  • internal/console/view.go +84 −0
  • internal/edge/edge.go +158 −1
  • internal/edge/tunnel.go +224 −0
  • internal/edge/tunnel_e2e_test.go +218 −0
  • internal/edge/tunnel_test.go +316 −0
  • internal/host/agent.go +37 −7
  • internal/host/bootstrap.go +39 −0
  • internal/host/host_test.go +47 −0
  • internal/host/probes.go +84 −0
  • internal/host/tunnel_client.go +59 −0
  • internal/host/tunnel_client_test.go +100 −0
  • internal/lease/lease.go +23 −2
  • internal/lease/lease_test.go +95 −0
  • internal/registry/registry.go +77 −17
  • internal/registry/registry_test.go +120 −0
  • internal/status/contract_test.go +40 −8
  • internal/status/dryrun.go +64 −1
  • internal/status/fleet.go +46 −3
  • internal/status/snapshot.go +79 −2
  • internal/status/status_test.go +132 −0
  • internal/tunnelclient/tunnelclient.go +267 −0
  • internal/tunnelclient/tunnelclient_test.go +154 −0
  • statuslive_test.go +127 −0
diff --git a/CLAUDE.md b/CLAUDE.md
index 363f011c8..fe492be45 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -23,7 +23,7 @@ capable model — exact values, file paths and interfaces go in the brief.
## Build & test
- `go build ./...`, `go vet ./...`, `go test ./... -count=1` — all clean, all
- 14 packages, before any commit lands.
+ 18 packages, before any commit lands.
- `gofmt -l` clean for files you touch (three pre-existing unformatted files
elsewhere are left alone).
- Sandbox note: use `GOCACHE=/tmp/claude-1001/gocache GOFLAGS=-mod=mod
diff --git a/cli.go b/cli.go
index a95989041..b4a56b284 100644
--- a/cli.go
+++ b/cli.go
@@ -19,12 +19,14 @@ import (
"os"
"path/filepath"
"regexp"
+ "sort"
"strings"
"text/tabwriter"
"time"
"github.com/carlosframework/platform/internal/appconfig"
"github.com/carlosframework/platform/internal/edge"
+ "github.com/carlosframework/platform/internal/host"
"github.com/carlosframework/platform/internal/ledger"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
@@ -217,7 +219,22 @@ func runInstances(args []string) {
// back to the app's default bundle and needs no bucket check — there is
// nothing to prove exists.
//
-// When BOTH flags are given, both rails must resolve cleanly before EITHER
+// A third, independent axis, -tunnel, names the reverse channel a route is
+// bound to at the edge (registry.Route.Tunnel — backing for later work in
+// the remote-host-agent feature, issue #53). It is simpler than -environment
+// in one way and matches it in another: shape is checked the same way
+// (registry.ValidTunnelName, same [a-z0-9-]{1,32} pattern as
+// ValidEnvironmentName), but there is no bucket to check EXISTENCE against
+// (a tunnel only becomes real once a host-agent dials in under that name) —
+// and it is -host only, since there is no bulk sibling to SetTunnel, so -app
+// + -tunnel is refused outright rather than silently doing nothing. Same
+// "-tunnel -" clear sentinel and the same platformCanaryOwnedRow refusal as
+// -channel/-environment, for the same structural reason. It writes through
+// its own SetTunnel call, independent of the -channel/-environment switch
+// below — with only ever one candidate, there is no combined transaction for
+// it to join.
+//
+// When BOTH -channel and -environment are given, both rails must resolve cleanly before EITHER
// axis writes: a single combined transaction (SetChannelsAndEnvironments)
// commits both columns for every candidate together, so a combined move can
// never leave a route repointed on one axis but not the other.
@@ -270,23 +287,28 @@ func runRoute(args []string) {
channel := fs.String("channel", "", "release channel to subscribe the route(s) to")
environment := fs.String("environment", "",
`named config environment to bind the route(s) to (must already exist in the bucket); "-" clears to default`)
+ tunnel := fs.String("tunnel", "",
+ `name of the reverse channel backing this route at the edge; "-" clears to a Socket/Addr-routed upstream (host only, no -app)`)
allowVersionChange := fs.Bool("allow-version-change", false,
"allow repointing to a channel that RESOLVES but holds a different version than the route currently serves "+
"(never overrides an unresolvable target, a bad signature, or a missing pubkey)")
dryRun := fs.Bool("dry-run", false, "run the full safety-rail check and print the plan, but write nothing")
fs.Parse(args)
- const usage = "usage: carlos route -host <host> [-channel <channel>] [-environment <name>] [-registry <db>] [-allow-version-change] [-dry-run]\n" +
+ const usage = "usage: carlos route -host <host> [-channel <channel>] [-environment <name>] [-tunnel <name>] [-registry <db>] [-allow-version-change] [-dry-run]\n" +
" or: carlos route -app <app> [-channel <channel>] [-environment <name>] [-account <sqid|name|all>] [-account-id <sqid>] [-registry <db>] [-allow-version-change] [-dry-run]\n" +
- " (-account-id names ONE account by sqid and supersedes -account, \"all\" included)"
+ " (-account-id names ONE account by sqid and supersedes -account, \"all\" included; -tunnel is -host only)"
if *host == "" && *app == "" {
log.Fatal(usage)
}
if *host != "" && *app != "" {
log.Fatal("route: -host and -app are mutually exclusive — name the one route, or the app whose routes to move\n\n" + usage)
}
- if *channel == "" && *environment == "" {
- log.Fatal(`route: at least one of -channel or -environment is required. "" only means stable-by-default at route CREATION (carlos add) — ` +
+ if *tunnel != "" && *app != "" {
+ log.Fatal("route: -tunnel is only supported with -host — bulk -app tunnel repointing is not implemented\n\n" + usage)
+ }
+ if *channel == "" && *environment == "" && *tunnel == "" {
+ log.Fatal(`route: at least one of -channel, -environment, or -tunnel is required. "" only means stable-by-default at route CREATION (carlos add) — ` +
"this verb sets an explicit binding, and needs an explicit target to set it to.")
}
if *channel != "" && !release.ValidChannel(*channel) {
@@ -321,6 +343,23 @@ func runRoute(args []string) {
if envClear {
resolvedEnv = ""
}
+ // tunnelClear is the same "-" clear sentinel as -environment's, checked
+ // before ValidTunnelName for the same reason envClear precedes
+ // ValidEnvironmentName above: a bare hyphen fits [a-z0-9-]{1,32} and
+ // would otherwise be treated as a real tunnel name. Unlike -environment
+ // there is no reserved name and no bucket to validate EXISTENCE against
+ // here — a tunnel name only becomes real once the host-agent dials in
+ // under it (later work in this feature) — but the shape itself is
+ // checked, same as -environment's, since this name is later matched
+ // against a dialed-in identity and may build keys.
+ tunnelClear := *tunnel == "-"
+ if *tunnel != "" && !tunnelClear && !registry.ValidTunnelName(*tunnel) {
+ log.Fatalf("route: %q is not a valid tunnel name — want [a-z0-9-]{1,32}", *tunnel)
+ }
+ resolvedTunnel := *tunnel
+ if tunnelClear {
+ resolvedTunnel = ""
+ }
r, err := registry.Open(*regPath)
if err != nil {
@@ -594,6 +633,30 @@ func runRoute(args []string) {
"%s: %s — no flag overrides a row this verb does not own", v.rt.Host, v.note))
}
}
+ // The tunnel rail: -host only (the -app refusal above guarantees exactly
+ // one candidate whenever -tunnel is set), and unlike -channel/-environment
+ // there is no existence check — a tunnel only becomes real once the
+ // host-agent dials in under that name, so there is no bucket store to
+ // consult yet. Same Platform+Canary refusal as the other two axes, for
+ // the same structural reason (platformCanaryOwnedRow's doc comment covers
+ // all three columns it guards).
+ var tunnelFrom string
+ tunnelNoop, tunnelWillWrite := false, false
+ if *tunnel != "" {
+ rt := candidates[0]
+ tunnelFrom = rt.Tunnel
+ switch {
+ case tunnelFrom == resolvedTunnel:
+ tunnelNoop = true
+ case platformCanaryOwnedRow(rt):
+ refusals = append(refusals, fmt.Sprintf(
+ "%s: Platform+Canary route — its whole row is rebuilt by the domains reconciler's canary-minting "+
+ "pass (internal/domains/reconcile.go) — this verb does not own this row, no flag overrides that",
+ rt.Host))
+ default:
+ tunnelWillWrite = true
+ }
+ }
if len(refusals) > 0 {
for _, msg := range refusals {
fmt.Fprintln(os.Stderr, msg)
@@ -631,6 +694,15 @@ func runRoute(args []string) {
}
}
}
+ // The tunnel axis writes independently of the channel/environment switch
+ // above — it has no bulk sibling to combine into their transactions (it
+ // is always exactly one candidate; see the -app refusal above), so there
+ // is nothing for it to be atomic WITH.
+ if tunnelWillWrite && !*dryRun {
+ if err := r.SetTunnel(candidates[0].Host, resolvedTunnel); err != nil {
+ log.Fatalf("route: write: %v", err)
+ }
+ }
prefix := ""
if *dryRun {
@@ -642,6 +714,12 @@ func runRoute(args []string) {
}
return e
}
+ tunnelLabel := func(t string) string {
+ if t == "" {
+ return "none"
+ }
+ return t
+ }
for i := range candidates {
if chanVerdicts != nil {
v := chanVerdicts[i]
@@ -659,6 +737,13 @@ func runRoute(args []string) {
fmt.Printf("%s%s: environment %s -> %s\n", prefix, v.rt.Host, envLabel(v.from), envLabel(resolvedEnv))
}
}
+ if *tunnel != "" && i == 0 {
+ if tunnelNoop {
+ fmt.Printf("%s: tunnel already %s — nothing to do\n", candidates[0].Host, tunnelLabel(resolvedTunnel))
+ } else {
+ fmt.Printf("%s%s: tunnel %s -> %s\n", prefix, candidates[0].Host, tunnelLabel(tunnelFrom), tunnelLabel(resolvedTunnel))
+ }
+ }
}
}
@@ -1560,12 +1645,13 @@ func runKeygen(args []string) {
}
// statusUsage is shared between "no flag given" and "-h/--help" so an
-// operator gets the same explanation either way: this verb only has a
-// dry-run mode. The fleet status page's LIVE publish loop (CollectBox ->
-// WriteBox on every box, BuildFleet -> PublishFleet on the lease holder)
-// runs inside the box agent's own 5-minute tick, not from a one-shot CLI
-// invocation — there is deliberately no "carlos status publish" here.
+// operator gets the same explanation either way: this verb reads and checks,
+// it never publishes. The fleet status page's PUBLISH loop (CollectBox ->
+// WriteBox on every box, BuildFleet -> PublishFleet on the lease holder) runs
+// inside the box agent's own 5-minute tick — there is deliberately no
+// "carlos status publish" here; -live only reads what those ticks wrote.
const statusUsage = `usage: carlos status -dry-run <dir>
+ carlos status -live
-dry-run <dir> build a throwaway fleet stuffed with fixture secrets
(hostnames, a machine id, mid-band fractions), run it
@@ -1576,19 +1662,32 @@ const statusUsage = `usage: carlos status -dry-run <dir>
scratch directory before ever pointing this binary at a
real bucket.
-carlos status has no live mode: the fleet status page is published every 5
-minutes by the box agent's own status lease holder (see "carlos agent"),
-never by a one-shot CLI invocation.
+ -live read status/boxes/*.json out of the status bucket and print
+ one line per host: release, tunnel state, memory, load and
+ free disk — raw figures, never published on the public
+ status page. A host that has stopped reporting prints as
+ UNREACHABLE with how long it has been dark and none of its
+ last readings. Needs bucket credentials and
+ CARLOS_STATUS_BUCKET (or CARLOS_DEPLOYMENT_BUCKET, the same
+ defaulting the agent's own status loop applies).
`
-// RunStatus is the "carlos status" verb: dry-run only. Modeled on ops.Run's
-// FlagSet shape (main.go's "ops" case), but flatter — status has exactly one
-// mode today, so there is no sub-verb dispatch to build out yet.
+// RunStatus is the "carlos status" verb: a fixture dry-run and a live read of
+// what the fleet's boxes last reported. Modeled on ops.Run's FlagSet shape
+// (main.go's "ops" case), but flatter — two independent modes, no sub-verb
+// dispatch.
func RunStatus(args []string) {
fs := flag.NewFlagSet("status", flag.ExitOnError)
dryRunDir := fs.String("dry-run", "", "build a fixture fleet, publish it into this directory, and verify no leaks")
+ liveFlag := fs.Bool("live", false, "read status/boxes/*.json from the status bucket and print per-host metrics")
fs.Parse(args)
+ if *liveFlag {
+ if err := statusLive(os.Stdout); err != nil {
+ log.Fatal(err)
+ }
+ return
+ }
if *dryRunDir == "" {
fmt.Fprint(os.Stderr, statusUsage)
os.Exit(1)
@@ -1597,3 +1696,95 @@ func RunStatus(args []string) {
log.Fatal(err)
}
}
+
+// statusLive prints what every box in the fleet last reported about itself,
+// read straight from status/boxes/ — the private snapshots, raw numbers and
+// all, which is why this needs the operator's own bucket credentials and why
+// nothing here is ever served publicly (see internal/status's package
+// comment). Bucket resolution is host.ResolveStatusBucket, the identical
+// CARLOS_STATUS_BUCKET-then-CARLOS_DEPLOYMENT_BUCKET rule the agent's status
+// loop uses, so this reads exactly the bucket the agents write.
+func statusLive(w io.Writer) error {
+ ctx := context.Background()
+ bucket := host.ResolveStatusBucket(os.Getenv)
+ if bucket == "" {
+ return fmt.Errorf("set CARLOS_STATUS_BUCKET or CARLOS_DEPLOYMENT_BUCKET")
+ }
+ s, err := release.NewS3Store(ctx, bucket)
+ if err != nil {
+ return err
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ return err
+ }
+ printLiveBoxes(w, boxes, time.Now())
+ return nil
+}
+
+// printLiveBoxes is statusLive's rendering half, split out so it can be
+// asserted on without a bucket. One line per box, label-sorted so two runs
+// diff on the fleet rather than on list order.
+//
+// A stale box (status.BoxStale — the same rule the console and the published
+// boxes table use) prints UNREACHABLE and how long it has been dark, and
+// deliberately NOT its last readings: those numbers described a machine that
+// has since stopped answering, and printing them in the same columns as a
+// live host's is how an outage gets read as fine. A fresh box that carries no
+// metrics block at all is a host running code from before the block existed —
+// which is a fact worth stating, not a row of zeros.
+func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) {
+ sorted := append([]status.Box(nil), boxes...)
+ sort.Slice(sorted, func(i, j int) bool { return sorted[i].Label < sorted[j].Label })
+ for _, b := range sorted {
+ if status.BoxStale(b, now, status.SnapshotPeriod) {
+ fmt.Fprintf(w, "%-24s %-12s UNREACHABLE (last heartbeat %s ago)\n",
+ b.Label, orDash(b.Region), now.Sub(b.GeneratedAt).Round(time.Minute))
+ continue
+ }
+ m := b.Metrics
+ if m == nil {
+ fmt.Fprintf(w, "%-24s %-12s ok (no metrics block — pre-upgrade host)\n", b.Label, orDash(b.Region))
+ continue
+ }
+ fmt.Fprintf(w, "%-24s %-12s release=%s tunnel=%s mem=%s/%s load=%s disk-free=%s\n",
+ b.Label, orDash(b.Region), orDash(m.Release), orDash(m.Tunnel),
+ humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes),
+ humanFloat(m.Load1), humanBytes(m.DiskFreeBytes))
+ }
+}
+
+// orDash, humanBytes and humanFloat render one field of a metrics block.
+// Unknown is a dash in all three — never a zero, never a blank column: the
+// status package carries "I couldn't measure this" as a nil pointer all the
+// way to here precisely so the reader can tell it apart from a measured zero.
+func orDash(s string) string {
+ if s == "" {
+ return "-"
+ }
+ return s
+}
+
+func humanBytes(v *int64) string {
+ if v == nil {
+ return "-"
+ }
+ b := float64(*v)
+ switch {
+ case b >= 1<<30:
+ return fmt.Sprintf("%.1fGB", b/(1<<30))
+ case b >= 1<<20:
+ return fmt.Sprintf("%.1fMB", b/(1<<20))
+ case b >= 1<<10:
+ return fmt.Sprintf("%.1fKB", b/(1<<10))
+ default:
+ return fmt.Sprintf("%dB", *v)
+ }
+}
+
+func humanFloat(v *float64) string {
+ if v == nil {
+ return "-"
+ }
+ return fmt.Sprintf("%.2f", *v)
+}
diff --git a/docs/superpowers/plans/2026-08-04-pi-enablement.md b/docs/superpowers/plans/2026-08-04-pi-enablement.md
new file mode 100644
index 000000000..ab51d1113
--- /dev/null
+++ b/docs/superpowers/plans/2026-08-04-pi-enablement.md
@@ -0,0 +1,247 @@
+# Raspberry Pi Instance Host (dev platform) Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. **Depends on the remote-host-agent plan (`2026-08-04-remote-host-agent.md`) being merged first** — everything here consumes its machinery.
+
+**Goal:** A Raspberry Pi 4 (4 GB) joins the dev platform (Kass account, 640012953448) as the instance host for region `pi`, serving a demo fleet of ≥5 resident + ≥50 parked instances through the dev edge over the reverse channel.
+
+**Architecture:** No new placement machinery — the Pi is a region with its own data bucket (the hq pattern from Plan C Task C6); its activator is the only one pointed at that bucket, so residency IS placement. No live fallback: Pi instances are Pi-resident, and a dark Pi renders unreachable. Spec: `docs/superpowers/specs/2026-08-04-remote-host-pi-design.md`.
+
+**Tech Stack:** OpenTofu (module in this repo, root in the private platform-infrastructure repo), bash + `carlos` CLI (v0.4.0+ console-API verbs), 64-bit Raspberry Pi OS Lite.
+
+## Global Constraints
+
+- The off-cloud IAM principal is scoped to exactly the deployment bucket + the `pi` region bucket — wrong-region-fails-at-IAM must hold (spec; C6 decision 6).
+- The module must be reusable as-is for tito's hq Step 1 (same shape, different account/names).
+- 64-bit OS is mandatory: hosts resolve artifacts by `runtime.GOOS + "-" + runtime.GOARCH` (`internal/release/adopt.go:22`); 32-bit Raspberry Pi OS resolves nothing.
+- Demo instances are created via the console-API path (`carlos apps create`, PR #52) — no hand-rolled signup.
+- No live fallback wiring for region `pi` (approved design decision — a Wi-Fi blip must never migrate the demo).
+- Tofu roots live in the private `carlosframework/platform-infrastructure` repo (per `infra/README.md`); this repo carries only the module + documentation of the root's instantiation.
+- Website-feature-sync rule applies at merge: "run CARLOS on a Raspberry Pi" is website material — tag/tap + website PR in the same batch.
+
+---
+
+### Task 1: Tofu module `infra/modules/region-remote-host`
+
+**Files:**
+- Create: `infra/modules/region-remote-host/main.tf`, `variables.tf`, `outputs.tf`, `README.md`
+
+**Interfaces:**
+- Consumes: nothing in-repo (standalone module; mirrors `infra/modules/region-host`'s conventions — read its `main.tf`/`variables.tf` first and match style, provider constraints, and tagging).
+- Produces: a region data bucket + an IAM user with a scoped inline policy + an access key, for a host with no instance profile. Outputs: `bucket_name`, `access_key_id`, `secret_access_key` (sensitive).
+
+- [ ] **Step 1: Write the module.** `variables.tf`:
+
+```hcl
+variable "region_code" {
+ description = "Short region code this host serves (e.g. pi, hq)."
+ type = string
+}
+
+variable "data_bucket_name" {
+ description = "Name of this region's data bucket (created here)."
+ type = string
+}
+
+variable "deployment_bucket_arn" {
+ description = "ARN of the existing deployment bucket the host reads releases/status from and writes heartbeats to."
+ type = string
+}
+```
+
+`main.tf` (adapt provider/tag boilerplate from `infra/modules/region-host/main.tf` so the two modules read as siblings):
+
+```hcl
+resource "aws_s3_bucket" "data" {
+ bucket = var.data_bucket_name
+}
+
+resource "aws_s3_bucket_public_access_block" "data" {
+ bucket = aws_s3_bucket.data.id
+ block_public_acls = true
+ block_public_policy = true
+ ignore_public_acls = true
+ restrict_public_buckets = true
+}
+
+# The off-cloud host principal. No instance profile exists off EC2; this
+# user + access key is the C6 decision-6 shape (IAM Roles Anywhere is the
+# later hardening — the policy below is what either principal would carry).
+resource "aws_iam_user" "host" {
+ name = "carlos-remote-host-${var.region_code}"
+}
+
+resource "aws_iam_access_key" "host" {
+ user = aws_iam_user.host.name
+}
+
+# Scope: the deployment bucket + THIS region's data bucket and nothing
+# else — wrong-region-fails-at-IAM, same property as every cloud host.
+resource "aws_iam_user_policy" "host" {
+ name = "carlos-remote-host-${var.region_code}"
+ user = aws_iam_user.host.name
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Sid = "DeploymentBucket"
+ Effect = "Allow"
+ Action = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket"]
+ Resource = [var.deployment_bucket_arn, "${var.deployment_bucket_arn}/*"]
+ },
+ {
+ Sid = "OwnRegionData"
+ Effect = "Allow"
+ Action = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket"]
+ Resource = [aws_s3_bucket.data.arn, "${aws_s3_bucket.data.arn}/*"]
+ },
+ ]
+ })
+}
+```
+
+`outputs.tf`:
+
+```hcl
+output "bucket_name" { value = aws_s3_bucket.data.bucket }
+output "access_key_id" { value = aws_iam_access_key.host.id }
+output "secret_access_key" {
+ value = aws_iam_access_key.host.secret
+ sensitive = true
+}
+```
+
+`README.md`: one paragraph — what it is, the two consumers (dev `pi`, tito `hq` Step 1), and that S3 actions in the policy must be reconciled against what `region-host`'s policy grants (if region-host's policy names a different/narrower action set, MATCH IT — read `infra/modules/region-host/main.tf:60-116` while writing and copy its action list verbatim; the list above is a starting point, region-host is the authority).
+- [ ] **Step 2: Validate** — `tofu -chdir=infra/modules/region-remote-host init -backend=false && tofu -chdir=infra/modules/region-remote-host validate` → valid. (Sandbox may block the provider download; if `init` cannot fetch providers offline, record that validation ran only `tofu fmt -check` and flag the full validate for the operator.)
+- [ ] **Step 3: Commit** — `infra: region-remote-host module — data bucket + scoped off-cloud principal (pi now, hq later)`.
+
+---
+
+### Task 2: Demo seeding script
+
+**Files:**
+- Create: `scripts/seed-pi-demo.sh`
+
+**Interfaces:**
+- Consumes: `carlos apps create` (console-API verb, PR #52; rate limit is 10 apps/address/day — the script must respect it), `carlos route -tunnel` (remote-host-agent plan Task 1).
+- Produces: an idempotent script an operator runs from a dev-credentialed machine, in batches across days (55+ apps vs. the 10/day limiter), plus edge-side tunnel route backing.
+
+- [ ] **Step 1: Write the script:**
+
+```bash
+#!/usr/bin/env bash
+# Seed the Pi demo fleet on the dev platform: N resident + M parked demo
+# instances, all resident in region "pi" (the Pi's own data bucket — no
+# placement machinery, residency IS placement), edge routes backed by the
+# Pi's reverse channel.
+#
+# The console API's app-create limiter is 10/address/day, so run this in
+# daily batches: seed-pi-demo.sh <start-index> <count>. State is the dev
+# platform itself — re-running an already-created index is a no-op.
+set -euo pipefail
+
+START=${1:?usage: seed-pi-demo.sh <start-index> <count>}
+COUNT=${2:?usage: seed-pi-demo.sh <start-index> <count>}
+TUNNEL_NAME=${CARLOS_PI_TUNNEL_NAME:?set CARLOS_PI_TUNNEL_NAME (the Pi's channel name)}
+if (( COUNT > 10 )); then
+ echo "warning: console API limits app creation to 10/address/day" >&2
+fi
+
+for i in $(seq "$START" $((START + COUNT - 1))); do
+ app=$(printf 'pi-demo-%03d' "$i")
+ if carlos apps ls | grep -q "^${app}\b"; then
+ echo "exists: $app"
+ else
+ carlos apps create "$app"
+ echo "created: $app"
+ fi
+done
+
+echo
+echo "Next, on the DEV EDGE box, back each new hostname with the Pi channel:"
+echo " carlos route -host <hostname-of-app> -tunnel ${TUNNEL_NAME}"
+echo "and on the PI, add the matching local routes (carlos add / adopt as usual)."
+echo "First 5 apps stay resident (no hibernate flag); the rest get the"
+echo "hibernate flag on their Pi-local routes so they park."
+```
+
+ Before committing, verify the verb names against the CLI actually on main (`carlos apps -h` / cli.go): if the listing verb is `carlos apps list` rather than `ls`, or create takes flags (account, region), match reality — the PR #52 branch is the authority. Adjust the grep accordingly.
+- [ ] **Step 2:** `chmod +x scripts/seed-pi-demo.sh && bash -n scripts/seed-pi-demo.sh` → syntax-clean.
+- [ ] **Step 3: Commit** — `scripts: pi demo fleet seeding (batched under the 10/day app-create limit)`.
+
+---
+
+### Task 3: Region-pi runbook (concrete env, dev values)
+
+**Files:**
+- Create: `infra/dev/region-pi.md` (create the `infra/dev/` directory; it mirrors where the hq physical runbook will live for tito)
+
+**Interfaces:**
+- Consumes: `infra/remote-host.md` (the generic software half, remote-host-agent plan Task 8); this file is the dev-Pi-specific instantiation.
+
+- [ ] **Step 1: Write the runbook:**
+
+```markdown
+# Region pi — Raspberry Pi 4 host on the dev platform
+
+Hardware: Raspberry Pi 4, 4 GB. Storage: SSD over USB3 strongly preferred;
+high-endurance SD acceptable for the demo (SQLite + WAL streaming is the
+I/O load, and SD I/O is the honest constraint on this board).
+
+OS: 64-bit Raspberry Pi OS Lite — MANDATORY. Hosts pick release artifacts
+by GOOS-GOARCH; this Pi resolves linux-arm64, the same artifact the
+Graviton fleet runs. 32-bit resolves nothing.
+
+Follow infra/remote-host.md with these dev-platform values:
+
+- Tofu: instantiate infra/modules/region-remote-host in the dev root
+ (platform-infrastructure repo) as:
+ module "region_pi" {
+ source = "git::https://github.com/carlosframework/platform.git//infra/modules/region-remote-host"
+ region_code = "pi"
+ data_bucket_name = "carlos-dev-pi-640012953448"
+ deployment_bucket_arn = <dev deployment bucket ARN>
+ }
+ Apply from the platform-infrastructure checkout (operator step — the
+ root module is not in this repo).
+- /etc/carlos/credentials: the module's access key output (0600).
+- /etc/carlos/host.env additions beyond the standard agent env:
+ AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials
+ CARLOS_HIBERNATE_BUCKET=carlos-dev-pi-640012953448
+ STATUS_BOX_REGION="Pi · demo"
+ CARLOS_TUNNEL_URL=https://<dev tunnel hostname>
+ CARLOS_TUNNEL_TOKEN=<from the edge's CARLOS_TUNNEL_CLIENTS entry>
+ CARLOS_TUNNEL_NAME=pi-box
+- Dev edge host.env additions:
+ CARLOS_TUNNEL_HOST=<dev tunnel hostname, DNS → dev edge>
+ CARLOS_TUNNEL_CLIENTS=pi-box=<token>
+ (Restart the dev edge to pick these up. Mint the token with
+ `openssl rand -hex 32`; it lives only in these two env files.)
+- No live fallback: no other host gets IAM on the pi bucket. A dark Pi
+ shows UNREACHABLE in the fleet view and `carlos status -live`; instances
+ resume when it re-dials. This is the approved demo posture.
+
+Seeding: scripts/seed-pi-demo.sh (batches of ≤10/day — console limiter).
+Target: ≥5 resident + ≥50 parked.
+
+## Demo acceptance (run these, tick them)
+
+- [ ] Pi appears in `carlos status -live` with metrics + tunnel=connected.
+- [ ] A resident instance serves through the dev edge over the channel.
+- [ ] A parked instance wakes on first request through the channel.
+- [ ] Pull Wi-Fi, restore it: next request succeeds after re-dial
+ (seconds — the bound the e2e test asserts in-repo).
+- [ ] Power off: fleet shows UNREACHABLE (age, no stale numbers).
+ Power on: serving resumes; `carlos status -live` confirms nothing
+ migrated (instances still pi-resident).
+```
+
+- [ ] **Step 2: Commit** — `infra: region-pi runbook — Pi 4 as dev-platform instance host`.
+
+---
+
+### Task 4: Final verification + PR + website batch
+
+- [ ] **Step 1:** `go build ./... && go vet ./... && go test ./... -count=1` (unchanged code, but the gate is unconditional) + `tofu fmt -check infra/modules/region-remote-host` → clean.
+- [ ] **Step 2:** Open the PR: "Raspberry Pi instance host: region pi on the dev platform". Body links the spec, the remote-host-agent PR it stacks on, and lists the operator steps that remain manual (tofu apply in platform-infrastructure, token mint, Pi hands-on, seeding batches). Draft until the remote-host-agent PR merges.
+- [ ] **Step 3:** At merge: website-feature-sync batch — the "CARLOS on a Raspberry Pi" story (tag/tap + website PR together; apex deploy stays manual).
+- [ ] **Step 4:** After the live demo acceptance boxes in `infra/dev/region-pi.md` are ticked by the operator, record actual measured numbers (resident count achieved, wake latency, reconnect time) back into that file — measured, not asserted.
diff --git a/docs/superpowers/plans/2026-08-04-remote-host-agent.md b/docs/superpowers/plans/2026-08-04-remote-host-agent.md
new file mode 100644
index 000000000..9719a7cb7
--- /dev/null
+++ b/docs/superpowers/plans/2026-08-04-remote-host-agent.md
@@ -0,0 +1,1261 @@
+# Remote Host-Agent (issue #53, C6 Step 3) Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** The existing `carlos` binary gains a remote host-agent role: tunnel-backed routes on the edge (reverse HTTP/2 over a host-dialed connection), raw host metrics in the heartbeat Box snapshot (banded before anything public), verified dark-then-returns lease semantics, and an off-cloud bootstrap path.
+
+**Architecture:** The host dials the edge's existing 443 listener on a reserved hostname, the connection inverts (edge = h2 client, host = h2 server), and tunnel-backed routes proxy over it via the same `httputil.ReverseProxy` discipline as addr-backed routes. Spec: `docs/superpowers/specs/2026-08-04-remote-host-pi-design.md`. Charter: issue #53; settled architecture is NOT relitigated (outbound-only, no new binary).
+
+**Tech Stack:** Go 1.26, `golang.org/x/net/http2` (already in module graph as indirect — promoting to direct needs no network), stdlib `httputil`, existing S3/lease/status machinery.
+
+## Global Constraints
+
+- Wire-shape changes are ADDITIVE: Box snapshots, registry rows, and route JSON written before this change must still parse (repo rule, learned three times).
+- Consumer list for any shape change = everything that PARSES the shape, not everything that calls the API.
+- All 14 packages: `go build ./...`, `go vet ./...`, `go test ./... -count=1`, `gofmt -l` clean on touched files, before any commit lands.
+- Sandbox: `GOCACHE=/tmp/claude-1001/gocache GOFLAGS=-mod=mod GOPROXY=off`; disable the tool sandbox if the toolchain reports "read-only file system".
+- Production channel names pinned in `internal/release/laddercheck_test.go` must keep passing.
+- A tunnel-backed host is exactly as untrusted as an addr-backed one: `SetXForwarded` + `scrubInboundPurgeHeader` inbound, `applyPurgeHeader`/`purgeModifier` outbound, unchanged.
+- Raw metrics NEVER reach the publicly served `status/fleet.json` — the privacy banding in `CollectBox`/`BuildFleet` is a public commitment.
+- Per CLAUDE.md delegation policy: Opus implements/reviews the edge listener, auth, and lease/failover tasks (1, 2, 3, 4, 9, 10); Sonnet fits tasks 5–8; the reviewer for the security-adjacent tasks is Opus.
+- Work happens on the existing worktree branch `worktree-remote-host-agent`; draft PR when Task 1 lands.
+
+**Recorded divergences from issue #53 wording** (carry into the PR description and the C6 Step 3 tick):
+1. "current release + channel" per box: the Box heartbeat carries the running binary's release (`version.Build`). There is no box-level platform channel (channels are per-route; verified by grep — no `CARLOS_PLATFORM_CHANNEL` exists), so no channel field is invented.
+2. Resident/parked instance counts already exist on the Box (`RouteCounts.Resident`/`Hibernating`) — not duplicated into the new metrics block.
+3. Transport is reverse HTTP/2, not a websocket port of titogo's tunnel (the in-repo judgment call the issue delegates; rationale in the spec).
+
+---
+
+### Task 1: Registry `Tunnel` column + `carlos route -tunnel` setter
+
+**Files:**
+- Modify: `internal/registry/registry.go` (Route struct ~line 30–103, migrate ALTER list ~line 163, scan ~line 353, Upsert ~line 460, narrow setters near `SetEnvironment`)
+- Modify: `cli.go` (runRoute — follow the existing `-environment` flag shape)
+- Test: `internal/registry/registry_test.go`
+
+**Interfaces:**
+- Consumes: existing `Route`, `Registry.Upsert`, `SetEnvironment` precedent.
+- Produces: `Route.Tunnel string` (channel name backing this route; `""` = not tunnel-backed), `(*Registry).SetTunnel(host, tunnel string) error`, CLI `carlos route -host <h> -tunnel <name|->`. Upsert PRESERVES a non-empty existing Tunnel (same rationale as Environment: full-row re-Upserts must not silently unbind).
+
+- [ ] **Step 1: Write the failing tests** (model on the existing Environment preservation tests in `registry_test.go` — read them first, mirror their harness):
+
+```go
+func TestTunnelColumnRoundTrips(t *testing.T) {
+ r := testRegistry(t) // whatever helper the file already uses to open a temp registry
+ must(t, r.Upsert(registry.Route{Host: "demo.example", Kind: "instance", Status: "ready"}))
+ must(t, r.SetTunnel("demo.example", "pi-box"))
+ rt := mustGet(t, r, "demo.example")
+ if rt.Tunnel != "pi-box" {
+ t.Fatalf("Tunnel = %q, want pi-box", rt.Tunnel)
+ }
+}
+
+func TestUpsertPreservesTunnel(t *testing.T) {
+ r := testRegistry(t)
+ must(t, r.Upsert(registry.Route{Host: "demo.example", Kind: "instance", Status: "ready"}))
+ must(t, r.SetTunnel("demo.example", "pi-box"))
+ // A full-row re-Upsert with zero-value Tunnel must not unbind.
+ must(t, r.Upsert(registry.Route{Host: "demo.example", Kind: "instance", Status: "ready"}))
+ if rt := mustGet(t, r, "demo.example"); rt.Tunnel != "pi-box" {
+ t.Fatalf("Upsert clobbered Tunnel: %q", rt.Tunnel)
+ }
+ // Clearing has exactly one door: SetTunnel(host, "").
+ must(t, r.SetTunnel("demo.example", ""))
+ if rt := mustGet(t, r, "demo.example"); rt.Tunnel != "" {
+ t.Fatalf("SetTunnel clear failed: %q", rt.Tunnel)
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/registry/ -run Tunnel -count=1` → FAIL (`rt.Tunnel undefined`).
+- [ ] **Step 3: Implement.** Add `Tunnel string` to `Route` (doc comment: "name of the reverse channel backing this route at the edge; written only by SetTunnel / carlos route -tunnel; Upsert preserves non-empty values — see Environment's comment for why"). Add `` `ALTER TABLE routes ADD COLUMN tunnel TEXT NOT NULL DEFAULT ''` `` to the migrate ALTER list. Add `tunnel` to the SELECT column list + Scan targets + Upsert (INSERT column, and the preserve-non-empty UPDATE clause exactly as Environment does — copy its SQL shape). Add `SetTunnel` modeled byte-for-byte on `SetEnvironment`.
+- [ ] **Step 4: Run tests** — `go test ./internal/registry/ -count=1` → PASS (all, not just new).
+- [ ] **Step 5: CLI flag.** In `cli.go` runRoute, add `-tunnel` string flag following `-environment`'s pattern exactly (including `-` meaning clear and the platform/canary-owned-row refusal if `-environment` has one — mirror it). Manual check: `go run . route -h` shows the flag.
+- [ ] **Step 6: Full gate + commit** — `go build ./... && go vet ./... && go test ./... -count=1 && gofmt -l internal/registry/registry.go cli.go` (expect no output from gofmt). Commit: `git add -A && git commit -m "registry: additive Tunnel column, preserved on Upsert, set via carlos route -tunnel"`.
+
+---
+
+### Task 2: Edge `TunnelHub` — authenticated reverse-channel registration
+
+**Files:**
+- Create: `internal/edge/tunnel.go`
+- Test: `internal/edge/tunnel_test.go`
+- Modify: `go.mod` (promote `golang.org/x/net` to direct require — `-mod=mod` does this automatically on build; no network needed, v0.55.0 is in the module cache)
+
+**Interfaces:**
+- Produces (consumed by Tasks 3, 4, 10):
+ - `type TunnelHub struct{...}` with `NewTunnelHub(host string, clients map[string]string) *TunnelHub` (host = reserved hostname; clients = channel name → bearer token)
+ - `(*TunnelHub).ServeHTTP(w, r)` — the registration endpoint (only ever invoked for the reserved hostname)
+ - `(*TunnelHub).transportFor(name string) http.RoundTripper` — errors when the channel is down (ReverseProxy turns that into 502)
+ - `(*TunnelHub).State(name string) string` — `"connected"` or `""`
+ - `(*TunnelHub).Host() string`
+ - Wire protocol: `GET /register`, headers `Authorization: Bearer <token>`, `Carlos-Tunnel-Name: <name>`, `Connection: Upgrade`, `Upgrade: carlos-tunnel/1` → `101 Switching Protocols` → connection inverts (edge h2 client / host h2 server).
+
+- [ ] **Step 1: Write the failing tests:**
+
+```go
+package edge
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestTunnelRegisterRejectsBadToken(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ for name, hdr := range map[string]http.Header{
+ "no auth": {},
+ "wrong token": {"Authorization": {"Bearer nope"}, "Carlos-Tunnel-Name": {"pi-box"}},
+ "unknown name": {"Authorization": {"Bearer sekrit"}, "Carlos-Tunnel-Name": {"other"}},
+ // A valid token may only claim ITS OWN channel name.
+ "cross-name": {"Authorization": {"Bearer sekrit"}, "Carlos-Tunnel-Name": {""}},
+ } {
+ req := httptest.NewRequest("GET", "/register", nil)
+ req.Header = hdr
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("%s: got %d, want 401", name, rec.Code)
+ }
+ }
+}
+
+func TestTunnelRegisterRejectsUnknownPath(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ req := httptest.NewRequest("GET", "/anything-else", nil)
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("got %d, want 404", rec.Code)
+ }
+}
+
+func TestParseTunnelClients(t *testing.T) {
+ got, err := parseTunnelClients("pi-box=sekrit,hq=other")
+ if err != nil || got["pi-box"] != "sekrit" || got["hq"] != "other" {
+ t.Fatalf("parse: %v %v", got, err)
+ }
+ if _, err := parseTunnelClients("malformed"); err == nil {
+ t.Fatal("want error on entry without '='")
+ }
+}
+```
+
+(The happy-path 101 + round-trip is Task 10's e2e — it needs a real hijackable conn, which httptest.NewRecorder can't provide.)
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/edge/ -run Tunnel -count=1` → FAIL (undefined: NewTunnelHub).
+- [ ] **Step 3: Implement `internal/edge/tunnel.go`:**
+
+```go
+// Package-internal reverse-channel hub: remote hosts dial OUT to the edge's
+// existing 443 listener on a reserved hostname, authenticate with a bearer
+// token bound to their channel name, and the connection inverts — the edge
+// becomes an HTTP/2 client over the hijacked conn, the host an HTTP/2 server.
+// Nothing ever connects TO a remote host (issue #53's locked-down property);
+// this file is the edge half of that bargain. Transport judgment call
+// (spec 2026-08-04): reverse h2 rather than a websocket port of titogo's
+// tunnel — x/net/http2 already supplies mux, flow control, and pings.
+package edge
+
+import (
+ "crypto/subtle"
+ "fmt"
+ "log"
+ "net"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/net/http2"
+)
+
+const (
+ tunnelRegisterPath = "/register"
+ tunnelUpgradeProto = "carlos-tunnel/1"
+ // tunnelPingEvery / tunnelPingTimeout bound how long a half-dead channel
+ // can look connected: a failed or timed-out ping tears it down, the host
+ // notices its conn die and re-dials (seconds — the bound Task 10 asserts).
+ tunnelPingEvery = 15 * time.Second
+ tunnelPingTimeout = 10 * time.Second
+)
+
+type tunnelConn struct {
+ cc *http2.ClientConn
+ raw net.Conn
+}
+
+type TunnelHub struct {
+ host string // reserved hostname this hub answers on
+ clients map[string]string // channel name -> bearer token
+ mu sync.Mutex
+ conns map[string]*tunnelConn
+}
+
+func NewTunnelHub(host string, clients map[string]string) *TunnelHub {
+ return &TunnelHub{host: host, clients: clients, conns: make(map[string]*tunnelConn)}
+}
+
+func (h *TunnelHub) Host() string { return h.host }
+
+// parseTunnelClients parses CARLOS_TUNNEL_CLIENTS ("name=token[,name=token]").
+func parseTunnelClients(s string) (map[string]string, error) {
+ out := make(map[string]string)
+ for _, pair := range strings.Split(s, ",") {
+ pair = strings.TrimSpace(pair)
+ if pair == "" {
+ continue
+ }
+ name, token, ok := strings.Cut(pair, "=")
+ if !ok || name == "" || token == "" {
+ return nil, fmt.Errorf("tunnel clients: malformed entry %q (want name=token)", pair)
+ }
+ out[name] = token
+ }
+ return out, nil
+}
+
+// ServeHTTP handles ONLY the registration handshake; the Edge routes requests
+// here when r.Host equals the reserved hostname, so ordinary app traffic can
+// never reach it and this handler never sees ordinary traffic.
+func (h *TunnelHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != tunnelRegisterPath {
+ http.NotFound(w, r)
+ return
+ }
+ name := r.Header.Get("Carlos-Tunnel-Name")
+ token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+ want, ok := h.clients[name]
+ // Constant-time compare; the name lookup happening first is fine — the
+ // name is not the secret, the token is. A token is bound to exactly one
+ // channel name, so a leaked token cannot impersonate a different host.
+ if name == "" || !ok || subtle.ConstantTimeCompare([]byte(token), []byte(want)) != 1 {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ http.Error(w, "hijack unsupported", http.StatusInternalServerError)
+ return
+ }
+ conn, bufrw, err := hj.Hijack()
+ if err != nil {
+ log.Printf("edge: tunnel %s: hijack: %v", name, err)
+ return
+ }
+ // Manual 101 on the hijacked conn (the ResponseWriter is spent once hijacked).
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: %s\r\nConnection: Upgrade\r\n\r\n", tunnelUpgradeProto)
+ if err := bufrw.Flush(); err != nil {
+ conn.Close()
+ return
+ }
+ tr := &http2.Transport{}
+ cc, err := tr.NewClientConn(conn)
+ if err != nil {
+ log.Printf("edge: tunnel %s: h2 client conn: %v", name, err)
+ conn.Close()
+ return
+ }
+ h.mu.Lock()
+ if old := h.conns[name]; old != nil {
+ old.raw.Close() // a re-register replaces the previous channel
+ }
+ h.conns[name] = &tunnelConn{cc: cc, raw: conn}
+ h.mu.Unlock()
+ log.Printf("edge: tunnel %s: connected from %s", name, conn.RemoteAddr())
+ go h.pingLoop(name, cc, conn)
+}
+
+// pingLoop is the edge-side liveness check: a dead ping closes the raw conn
+// and unregisters the channel, so requests fail fast to 502 instead of
+// hanging on a black-holed TCP session (the venue-Wi-Fi failure mode).
+func (h *TunnelHub) pingLoop(name string, cc *http2.ClientConn, raw net.Conn) {
+ for {
+ time.Sleep(tunnelPingEvery)
+ ctx, cancel := context.WithTimeout(context.Background(), tunnelPingTimeout)
+ err := cc.Ping(ctx)
+ cancel()
+ if err != nil {
+ log.Printf("edge: tunnel %s: ping failed (%v) — dropping channel", name, err)
+ raw.Close()
+ h.mu.Lock()
+ if cur := h.conns[name]; cur != nil && cur.raw == raw {
+ delete(h.conns, name)
+ }
+ h.mu.Unlock()
+ return
+ }
+ }
+}
+
+func (h *TunnelHub) conn(name string) *http2.ClientConn {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if c := h.conns[name]; c != nil {
+ return c.cc
+ }
+ return nil
+}
+
+// State reports "connected" or "" — consumed by fleet status rendering.
+func (h *TunnelHub) State(name string) string {
+ if h.conn(name) != nil {
+ return "connected"
+ }
+ return ""
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// transportFor resolves the channel PER REQUEST, so a proxy built before a
+// reconnect uses the fresh channel afterwards with no reload needed.
+func (h *TunnelHub) transportFor(name string) http.RoundTripper {
+ return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ cc := h.conn(name)
+ if cc == nil {
+ return nil, fmt.Errorf("tunnel %q not connected", name)
+ }
+ return cc.RoundTrip(req)
+ })
+}
+```
+
+(Add `"context"` to imports. `go build ./internal/edge/` promotes `golang.org/x/net` to a direct require in go.mod under `-mod=mod` — commit the go.mod/go.sum change with this task.)
+
+- [ ] **Step 4: Run tests** — `go test ./internal/edge/ -run Tunnel -count=1` → PASS.
+- [ ] **Step 5: Full gate + commit** — all four commands clean. Commit: `edge: TunnelHub — authenticated reverse-channel registration (reverse h2 over hijacked 443)`.
+
+---
+
+### Task 3: Edge dispatch — tunnel-backed routes + hub wiring
+
+**Files:**
+- Modify: `internal/edge/edge.go` (Edge struct ~line 77, `Allowed` ~line 282, `ServeHTTP` ~line 287, `Reload` switch ~line 234–248, proxy constructors ~line 760–803, and `RunWithActivator` for env wiring)
+- Test: `internal/edge/tunnel_test.go` (extend)
+
+**Interfaces:**
+- Consumes: `TunnelHub` (Task 2), `Route.Tunnel` (Task 1), `purgeModifier`, `scrubInboundPurgeHeader`.
+- Produces: `(*Edge).EnableTunnels(hub *TunnelHub)`; env contract `CARLOS_TUNNEL_HOST` (reserved hostname) + `CARLOS_TUNNEL_CLIENTS` (name=token list) parsed in `RunWithActivator`; ready-hook seam `RunWithActivatorReady(args, act, cache, ready func(*Edge))` (consumed by Task 4).
+
+- [ ] **Step 1: Write the failing test** (extend tunnel_test.go):
+
+```go
+// testRegistryWithRoute opens a temp registry and upserts one route. If
+// edge_test.go already has an equivalent helper, use that instead — one
+// harness, not two.
+func testRegistryWithRoute(t *testing.T, rt registry.Route) *registry.Registry {
+ t.Helper()
+ r, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { r.Close() })
+ if err := r.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ return r
+}
+
+func TestReloadBuildsTunnelBackedRoute(t *testing.T) {
+ reg := testRegistryWithRoute(t, registry.Route{
+ Host: "demo.example", Kind: "instance", Status: "ready", Tunnel: "pi-box",
+ })
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ e.EnableTunnels(hub)
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ // Without EnableTunnels the route must be skipped (default: continue);
+ // with it, the host must be served and Allowed, and the reserved
+ // hostname must be Allowed for cert minting.
+ if !e.Allowed("demo.example") {
+ t.Fatal("tunnel-backed route not served after EnableTunnels+Reload")
+ }
+ if !e.Allowed("tunnel.test") {
+ t.Fatal("reserved tunnel hostname must be Allowed (autocert HostPolicy)")
+ }
+ // Channel down -> 502, not a hang.
+ req := httptest.NewRequest("GET", "http://demo.example/", nil)
+ req.Host = "demo.example"
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+ if rec.Code != http.StatusBadGateway {
+ t.Fatalf("disconnected tunnel: got %d, want 502", rec.Code)
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/edge/ -run TestReloadBuildsTunnelBackedRoute -count=1` → FAIL (undefined: EnableTunnels).
+- [ ] **Step 3: Implement.**
+ - Edge struct: add `tunnels *TunnelHub` field with doc comment ("nil = tunnel serving not configured; routes with Tunnel set are skipped, same nil-means-disabled convention as act/cache").
+ - `func (e *Edge) EnableTunnels(hub *TunnelHub) { e.tunnels = hub }` — call before Reload, same contract as EnableStatic.
+ - `Allowed`: `if e.tunnels != nil && host == e.tunnels.Host() { return true }` before the map lookup.
+ - `ServeHTTP`: immediately after the host split (~line 291), before the upstream lookup: `if e.tunnels != nil && host == e.tunnels.Host() { e.tunnels.ServeHTTP(w, r); return }`.
+ - `Reload` switch — new case after `t.Addr != ""` (a tunnel route has no socket/addr, so order vs. those is moot, but keep it before the static/status kinds so an instance-kind tunnel route can't fall through):
+
+```go
+case t.Tunnel != "":
+ if e.tunnels == nil {
+ continue // tunnel serving not configured on this box
+ }
+ p := tunnelProxy(t.Tunnel, e.tunnels)
+ p.ModifyResponse = e.purgeModifier(t)
+ proxy = p
+```
+
+ - Proxy constructor beside tcpProxy:
+
+```go
+// tunnelProxy reverse-proxies over a named reverse channel (TunnelHub).
+// Same Host preservation and purge-header discipline as tcpProxy — a
+// tunnel-backed host is exactly as untrusted as an addr-backed one. The
+// scheme is https because the h2 ClientConn requires it; no dial happens
+// (the channel already exists), so the value is otherwise inert.
+func tunnelProxy(name string, hub *TunnelHub) *httputil.ReverseProxy {
+ return &httputil.ReverseProxy{
+ Rewrite: func(pr *httputil.ProxyRequest) {
+ pr.SetXForwarded()
+ pr.Out.URL.Scheme = "https"
+ pr.Out.URL.Host = pr.In.Host // h2 :authority = the real hostname
+ pr.Out.Host = pr.In.Host
+ scrubInboundPurgeHeader(pr)
+ },
+ Transport: hub.transportFor(name),
+ }
+}
+```
+
+ - `RunWithActivator` wiring: rename the body to `RunWithActivatorReady(args []string, act Activator, cache *Cache, ready func(*Edge))`; keep `RunWithActivator` as a wrapper passing `nil`. Inside, after the Edge is fully configured (after EnableCache/EnableStatic and before the blocking serve/listen), add:
+
+```go
+if th := os.Getenv("CARLOS_TUNNEL_HOST"); th != "" {
+ clients, err := parseTunnelClients(os.Getenv("CARLOS_TUNNEL_CLIENTS"))
+ if err != nil {
+ log.Fatalf("edge: %v", err)
+ }
+ if len(clients) == 0 {
+ log.Fatalf("edge: CARLOS_TUNNEL_HOST set but CARLOS_TUNNEL_CLIENTS empty")
+ }
+ e.EnableTunnels(NewTunnelHub(th, clients))
+ if err := e.Reload(); err != nil {
+ log.Fatalf("edge: reload after EnableTunnels: %v", err)
+ }
+ log.Printf("edge: tunnel hub armed on %s (%d client(s))", th, len(clients))
+}
+if ready != nil {
+ ready(e)
+}
+```
+
+- [ ] **Step 4: Run tests** — `go test ./internal/edge/ -count=1` → PASS (whole package — the Reload/ServeHTTP changes must not break socket/addr/static/status tests).
+- [ ] **Step 5: Full gate + commit** — `edge: tunnel-backed route dispatch + hub wiring on the existing 443 listener`.
+
+---
+
+### Task 4: Host dial-out client (`tunnelLoop`)
+
+**Files:**
+- Create: `internal/host/tunnel_client.go`
+- Test: `internal/host/tunnel_client_test.go`
+- Modify: `internal/host/agent.go` (RunAgent ~line 146–157)
+
+**Interfaces:**
+- Consumes: Task 3's wire protocol and `RunWithActivatorReady` seam.
+- Produces: `tunnelLoop(handler http.Handler)` goroutine; `tunnelState() string` (`""` disabled / `"connected"` / `"reconnecting"`) consumed by Task 6's heartbeat probe; env contract `CARLOS_TUNNEL_URL`, `CARLOS_TUNNEL_TOKEN`, `CARLOS_TUNNEL_NAME`.
+
+- [ ] **Step 1: Write the failing test** — dial + serve against a hand-rolled upgrade server (no Edge involvement; the full loop is Task 10):
+
+```go
+package host
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "golang.org/x/net/http2"
+)
+
+// fakeTunnelEdge accepts one registration, answers 101, then drives ONE h2
+// request over the inverted conn and returns the response body it read.
+func fakeTunnelEdge(t *testing.T, wantToken, wantName string) (*httptest.Server, chan string) {
+ t.Helper()
+ got := make(chan string, 1)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer "+wantToken ||
+ r.Header.Get("Carlos-Tunnel-Name") != wantName {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ conn, bufrw, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ t.Errorf("hijack: %v", err)
+ return
+ }
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: carlos-tunnel/1\r\nConnection: Upgrade\r\n\r\n")
+ bufrw.Flush()
+ cc, err := (&http2.Transport{}).NewClientConn(conn)
+ if err != nil {
+ t.Errorf("h2 client: %v", err)
+ return
+ }
+ req, _ := http.NewRequest("GET", "https://demo.example/hello", nil)
+ resp, err := cc.RoundTrip(req)
+ if err != nil {
+ t.Errorf("roundtrip over tunnel: %v", err)
+ got <- ""
+ return
+ }
+ body, _ := io.ReadAll(resp.Body)
+ got <- resp.Header.Get("X-From-Host") + ":" + string(body)
+ }))
+ return srv, got
+}
+
+func TestDialAndServeInvertsConnection(t *testing.T) {
+ srv, got := fakeTunnelEdge(t, "sekrit", "pi-box")
+ defer srv.Close()
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-From-Host", r.Host)
+ fmt.Fprint(w, "pong")
+ })
+ conn, err := dialTunnel(srv.URL, "pi-box", "sekrit")
+ if err != nil {
+ t.Fatalf("dialTunnel: %v", err)
+ }
+ go serveTunnel(conn, handler)
+ if v := <-got; v != "demo.example:pong" {
+ t.Fatalf("over-tunnel response = %q, want demo.example:pong", v)
+ }
+}
+
+func TestDialTunnelRejectsNon101(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ }))
+ defer srv.Close()
+ if _, err := dialTunnel(srv.URL, "pi-box", "wrong"); err == nil {
+ t.Fatal("want error on 401 handshake")
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/host/ -run Tunnel -count=1` → FAIL (undefined: dialTunnel).
+- [ ] **Step 3: Implement `internal/host/tunnel_client.go`:**
+
+```go
+// The host half of the reverse channel (issue #53 / C6 Step 3): dial OUT to
+// the edge, invert the connection, and serve our own local edge routing over
+// HTTP/2. Reconnect fast — hq's dual-WAN failover and a Pi's Wi-Fi blip are
+// both "the conn died, dial again", and that latency is user-visible.
+package host
+
+import (
+ "bufio"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "log"
+ "math/rand/v2"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "sync/atomic"
+ "time"
+
+ "golang.org/x/net/http2"
+)
+
+// tunnelStateVal is "" (disabled/never started), "connected", or
+// "reconnecting" — read by statusLoop's heartbeat probe (probes must fail
+// toward honest values; "" means the loop isn't running at all).
+var tunnelStateVal atomic.Value
+
+func tunnelState() string {
+ if v, ok := tunnelStateVal.Load().(string); ok {
+ return v
+ }
+ return ""
+}
+
+const (
+ tunnelBackoffFloor = 250 * time.Millisecond
+ tunnelBackoffCap = 2 * time.Second
+)
+
+// tunnelLoop dials CARLOS_TUNNEL_URL and serves handler over the inverted
+// conn, forever, with jittered capped backoff. Inert unless all three env
+// vars are set — the same convention as every other optional agent loop.
+func tunnelLoop(handler http.Handler) {
+ rawURL := os.Getenv("CARLOS_TUNNEL_URL")
+ token := os.Getenv("CARLOS_TUNNEL_TOKEN")
+ name := os.Getenv("CARLOS_TUNNEL_NAME")
+ if rawURL == "" || token == "" || name == "" {
+ if rawURL != "" || token != "" || name != "" {
+ log.Printf("agent: tunnel loop disabled: need all of CARLOS_TUNNEL_URL, CARLOS_TUNNEL_TOKEN, CARLOS_TUNNEL_NAME")
+ }
+ return
+ }
+ log.Printf("agent: tunnel loop armed (%s as %q)", rawURL, name)
+ backoff := tunnelBackoffFloor
+ for {
+ conn, err := dialTunnel(rawURL, name, token)
+ if err != nil {
+ tunnelStateVal.Store("reconnecting")
+ log.Printf("agent: tunnel dial: %v (retry in %s)", err, backoff)
+ time.Sleep(backoff + rand.N(backoff/2))
+ if backoff *= 2; backoff > tunnelBackoffCap {
+ backoff = tunnelBackoffCap
+ }
+ continue
+ }
+ backoff = tunnelBackoffFloor
+ tunnelStateVal.Store("connected")
+ log.Printf("agent: tunnel connected as %q", name)
+ serveTunnel(conn, handler) // blocks until the conn dies
+ tunnelStateVal.Store("reconnecting")
+ log.Printf("agent: tunnel lost — re-dialing")
+ }
+}
+
+// bufferedConn replays bytes the handshake's bufio.Reader over-read (the h2
+// preface can arrive in the same packet as the 101) before the raw conn.
+type bufferedConn struct {
+ r io.Reader
+ net.Conn
+}
+
+func (b bufferedConn) Read(p []byte) (int, error) { return b.r.Read(p) }
+
+// dialTunnel performs the registration handshake and returns the raw conn,
+// ready to serve h2 on. https URLs use real TLS (the edge's public cert —
+// server auth by PKI, client auth by bearer token); http is for tests.
+func dialTunnel(rawURL, name, token string) (net.Conn, error) {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return nil, fmt.Errorf("tunnel url: %w", err)
+ }
+ addr := u.Host
+ var conn net.Conn
+ switch u.Scheme {
+ case "https":
+ if u.Port() == "" {
+ addr = net.JoinHostPort(u.Hostname(), "443")
+ }
+ conn, err = tls.Dial("tcp", addr, &tls.Config{
+ ServerName: u.Hostname(),
+ NextProtos: []string{"http/1.1"}, // the handshake is 1.1; h2 starts after the 101
+ })
+ case "http":
+ conn, err = net.Dial("tcp", addr)
+ default:
+ return nil, fmt.Errorf("tunnel url: unsupported scheme %q", u.Scheme)
+ }
+ if err != nil {
+ return nil, err
+ }
+ req := &http.Request{
+ Method: "GET",
+ URL: &url.URL{Path: "/register"},
+ Host: u.Hostname(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + token},
+ "Carlos-Tunnel-Name": {name},
+ "Connection": {"Upgrade"},
+ "Upgrade": {"carlos-tunnel/1"},
+ },
+ }
+ if err := req.Write(conn); err != nil {
+ conn.Close()
+ return nil, err
+ }
+ br := bufio.NewReader(conn)
+ resp, err := http.ReadResponse(br, req)
+ if err != nil {
+ conn.Close()
+ return nil, err
+ }
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusSwitchingProtocols {
+ conn.Close()
+ return nil, fmt.Errorf("tunnel register: %s", resp.Status)
+ }
+ if br.Buffered() > 0 {
+ return bufferedConn{r: io.MultiReader(br, conn), Conn: conn}, nil
+ }
+ return conn, nil
+}
+
+// serveTunnel serves handler over the inverted conn until it dies.
+func serveTunnel(conn net.Conn, handler http.Handler) {
+ (&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: handler})
+}
+```
+
+- [ ] **Step 4: Run tests** — `go test ./internal/host/ -run Tunnel -count=1` → PASS.
+- [ ] **Step 5: Wire into RunAgent.** In `internal/host/agent.go` replace the final `edge.RunWithActivator(args, act, cache)` call with:
+
+```go
+edge.RunWithActivatorReady(args, act, cache, func(e *edge.Edge) {
+ go tunnelLoop(e)
+})
+```
+
+(`*edge.Edge` implements `http.Handler`; the tunnel serves the host's own local routing, so a request arriving over the channel takes exactly the path a local request would — activator wake included.)
+
+- [ ] **Step 6: Full gate + commit** — `host: tunnel dial-out loop — reverse channel client with capped jittered backoff`.
+
+---
+
+### Task 5: Status shape — additive `Metrics` block on Box + back-compat + dry-run leak scope
+
+**Files:**
+- Modify: `internal/status/snapshot.go` (package doc lines 1–7, `Probes` ~line 22, `Box` ~line 72, `CollectBox` ~line 184)
+- Modify: `internal/status/dryrun.go` (read it FIRST — it publishes a fixture fleet and asserts no fixture leaks into published objects)
+- Test: `internal/status/snapshot_test.go` (or the file the package's existing CollectBox tests live in — read and extend, don't create a parallel file)
+
+**Interfaces:**
+- Consumes: nothing new.
+- Produces (consumed by Tasks 6, 7):
+
+```go
+// BoxMetrics is the raw, PRIVATE block: real numbers, never banded. It lives
+// in status/boxes/<label>.json — which only fleet-status machinery and
+// operators read — and is deliberately NOT copied into FleetBox, so the
+// publicly served fleet.json keeps the banded-words-only commitment.
+type BoxMetrics struct {
+ MemTotalBytes *int64 `json:"mem_total_bytes,omitempty"`
+ MemAvailBytes *int64 `json:"mem_avail_bytes,omitempty"`
+ Load1 *float64 `json:"load1,omitempty"`
+ DiskFreeBytes *int64 `json:"disk_free_bytes,omitempty"`
+ Release string `json:"release,omitempty"` // running binary's version.Build
+ Tunnel string `json:"tunnel,omitempty"` // ""|"connected"|"reconnecting"
+}
+```
+
+ plus `Box.Metrics *BoxMetrics` with `json:"metrics,omitempty"`, and `Probes` gains the same six fields (`MemTotalBytes`, `MemAvailBytes`, `Load1`, `DiskFreeBytes *int64/*float64`, `Release`, `Tunnel string`). Resident/parked counts are NOT duplicated — `RouteCounts` already carries them (recorded divergence 2).
+
+- [ ] **Step 1: Write the failing tests:**
+
+```go
+func TestCollectBoxCarriesRawMetrics(t *testing.T) {
+ mt, ma, df := int64(4056547328), int64(2028273664), int64(9999999)
+ l1 := 0.42
+ b := CollectBox(nil, Probes{
+ MemTotalBytes: &mt, MemAvailBytes: &ma, Load1: &l1, DiskFreeBytes: &df,
+ Release: "abc1234", Tunnel: "connected",
+ }, time.Now())
+ m := b.Metrics
+ if m == nil || *m.MemTotalBytes != mt || *m.Load1 != l1 || m.Release != "abc1234" || m.Tunnel != "connected" {
+ t.Fatalf("metrics block wrong: %+v", m)
+ }
+}
+
+// Yesterday's Box shape (no metrics key) must parse today, and a FUTURE
+// box's unknown keys must not break today's parse either.
+func TestBoxBackCompatParse(t *testing.T) {
+ yesterday := []byte(`{"generated_at":"2026-08-03T10:00:00Z","label":"ada-stargazing",
+ "routes":{"static":1,"resident":2,"hibernating":3,"awake":null,"asleep":null},
+ "load":"comfortable","capacity":"plenty",
+ "replication":{"healthy":true,"lag_seconds":5},
+ "adoption_running":true,"expired_canaries":0}`)
+ var b Box
+ if err := json.Unmarshal(yesterday, &b); err != nil {
+ t.Fatalf("yesterday's shape failed to parse: %v", err)
+ }
+ if b.Metrics != nil {
+ t.Fatal("absent metrics must decode as nil, not zero-value")
+ }
+ tomorrow := []byte(`{"generated_at":"2026-08-05T10:00:00Z","label":"ada-stargazing",
+ "routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},
+ "load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},
+ "adoption_running":null,"expired_canaries":null,
+ "metrics":{"mem_total_bytes":1,"some_future_field":true}}`)
+ if err := json.Unmarshal(tomorrow, &b); err != nil {
+ t.Fatalf("tomorrow's shape failed to parse: %v", err)
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/status/ -run 'CollectBoxCarriesRaw|BackCompat' -count=1` → FAIL.
+- [ ] **Step 3: Implement.** Add the fields; in `CollectBox`, after the existing Box literal is built, attach the block only when something is known (all-unknown publishes no block — additive for old parsers, quiet for boxes that can't answer):
+
+```go
+if p.MemTotalBytes != nil || p.MemAvailBytes != nil || p.Load1 != nil ||
+ p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" {
+ box.Metrics = &BoxMetrics{
+ MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,
+ Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes,
+ Release: p.Release, Tunnel: p.Tunnel,
+ }
+}
+```
+
+ Update the package doc comment (lines 1–7) and Box's doc: banding is still enforced here for every PUBLIC field; the Metrics block is the deliberate raw/private exception, excluded from fleet.json by construction (BuildFleet copies fields explicitly into FleetBox and does not copy Metrics — verify by reading `fleet.go:116-124`, and say so in the comment).
+- [ ] **Step 4: Dry-run leak scope.** Read `internal/status/dryrun.go` end to end. Extend its fixture Probes with distinctive raw values (e.g. `MemTotalBytes = 4242424242`). Keep/extend its leak assertions so that: (a) `status/fleet.json` and `status/index.html` must NOT contain `4242424242` (raw never public); (b) the box object MAY contain it. If the existing leak check sweeps ALL published objects for fraction-like fixtures, scope that sweep to fleet.json + index.html for the new raw fixtures only — do not weaken the existing hostname/machine-id assertions for box objects.
+- [ ] **Step 5: Run tests** — `go test ./internal/status/ -count=1` → PASS (including DryRun tests).
+- [ ] **Step 6: Full gate + commit** — `status: additive raw Metrics block on Box — private to the bucket, never published to fleet.json`.
+
+---
+
+### Task 6: Host probes for the metrics block + heartbeat wiring
+
+**Files:**
+- Modify: `internal/host/probes.go`, `internal/host/agent.go` (statusLoop's Probes literal ~line 622)
+- Test: `internal/host/probes_test.go` (or wherever `parseIsActive`-style probe tests live — read first, extend in place)
+
+**Interfaces:**
+- Consumes: `Probes` fields (Task 5), `tunnelState()` (Task 4), `version.Build` (`internal/version`).
+- Produces: `readMemBytes() (total, avail *int64)`, `readLoad1() *float64`, `readDiskFreeBytes() *int64` — all fail toward nil, never a guessed zero (the file's contract).
+
+- [ ] **Step 1: Write the failing tests** — pure parsers, no /proc in tests (follow the file's design note):
+
+```go
+func TestParseMeminfoBytes(t *testing.T) {
+ total, avail := parseMeminfoBytes(strings.NewReader(
+ "MemTotal: 3961472 kB\nMemFree: 100 kB\nMemAvailable: 1980736 kB\n"))
+ if total == nil || *total != 3961472*1024 || avail == nil || *avail != 1980736*1024 {
+ t.Fatalf("got %v %v", total, avail)
+ }
+ if tt, aa := parseMeminfoBytes(strings.NewReader("garbage")); tt != nil || aa != nil {
+ t.Fatal("unparseable meminfo must yield nil, not zero")
+ }
+}
+
+func TestParseLoad1(t *testing.T) {
+ if l := parseLoad1("0.42 0.36 0.30 1/234 5678\n"); l == nil || *l != 0.42 {
+ t.Fatalf("got %v", l)
+ }
+ if l := parseLoad1(""); l != nil {
+ t.Fatal("empty loadavg must yield nil")
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** — `go test ./internal/host/ -run 'Meminfo|Load1' -count=1` → FAIL.
+- [ ] **Step 3: Implement** in probes.go: `parseMeminfoBytes(r io.Reader)` (same scanner shape as readMemFrac, returning bytes = kB×1024), `readMemBytes()` opening `/proc/meminfo` and delegating; `parseLoad1(s string)` (first field of `/proc/loadavg` via `strconv.ParseFloat`), `readLoad1()` reading the file; `readDiskFreeBytes()` via `syscall.Statfs(dataDir(), &st)` returning `int64(st.Bavail) * st.Bsize` (nil on error). Every failure → nil.
+- [ ] **Step 4: Wire into statusLoop's Probes literal** (agent.go ~line 622), after the existing fields:
+
+```go
+memTotal, memAvail := readMemBytes()
+p := status.Probes{
+ // ... existing fields unchanged ...
+ MemTotalBytes: memTotal,
+ MemAvailBytes: memAvail,
+ Load1: readLoad1(),
+ DiskFreeBytes: readDiskFreeBytes(),
+ Release: version.Build,
+ Tunnel: tunnelState(),
+}
+```
+
+(Import `"github.com/carlosframework/platform/internal/version"`.)
+- [ ] **Step 5: Run tests** — `go test ./internal/host/ -count=1` → PASS.
+- [ ] **Step 6: Full gate + commit** — `host: raw metric probes (mem/load/disk/release/tunnel) into the heartbeat Box`.
+
+---
+
+### Task 7: Render metrics + staleness — console fleet view and `carlos status -live`
+
+**Files:**
+- Modify: `internal/status/fleet.go` (export a box loader), `internal/console/view.go` + `internal/console/server.go` (fleet view, anchor `view.go:344` FleetView / `server.go:961` handleFleet — read both first), `cli.go` (RunStatus ~line 1587, statusUsage ~line 1568)
+- Test: `internal/status/fleet_test.go`, plus the console package's existing fleet-view test file (read and extend in place)
+
+**Interfaces:**
+- Consumes: `Box.Metrics` (Task 5), existing `loadBoxes` (`fleet.go:145`), staleness rule (`now - GeneratedAt > 3×period`, `fleet.go:115`).
+- Produces: `status.LoadBoxes(ctx, s) ([]Box, error)` (exported wrapper over loadBoxes — one loader, one skip-corrupt rule, no second parser); `status.BoxStale(b Box, now time.Time) bool` using the same 3× constant; CLI `carlos status -live` reading `CARLOS_STATUS_BUCKET`/`CARLOS_DEPLOYMENT_BUCKET` via the same `resolveStatusBucket` helper agent.go's statusLoop uses.
+
+- [ ] **Step 1: Write the failing test** (fleet_test.go):
+
+```go
+func TestLoadBoxesAndStaleness(t *testing.T) {
+ s := testDirStore(t) // the DirStore helper the package's tests already use
+ fresh := Box{GeneratedAt: time.Now(), Label: "ada-stargazing",
+ Metrics: &BoxMetrics{Release: "abc1234", Tunnel: "connected"}}
+ dark := Box{GeneratedAt: time.Now().Add(-time.Hour), Label: "grace-kayaking"}
+ must(t, WriteBox(context.Background(), s, fresh))
+ must(t, WriteBox(context.Background(), s, dark))
+ boxes, err := LoadBoxes(context.Background(), s)
+ if err != nil || len(boxes) != 2 {
+ t.Fatalf("LoadBoxes: %v, %d boxes", err, len(boxes))
+ }
+ for _, b := range boxes {
+ wantStale := b.Label == "grace-kayaking"
+ if BoxStale(b, time.Now(), 5*time.Minute) != wantStale {
+ t.Fatalf("%s: stale = %v, want %v", b.Label, !wantStale, wantStale)
+ }
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** → FAIL (undefined: LoadBoxes).
+- [ ] **Step 3: Implement** in fleet.go: `func LoadBoxes(ctx context.Context, s release.Store) ([]Box, error) { return loadBoxes(ctx, s) }`; hoist the 3× rule into `const staleFactor = 3` + `func BoxStale(b Box, now time.Time, period time.Duration) bool { return now.Sub(b.GeneratedAt) > staleFactor*period }`. BuildFleet's inline computation at `fleet.go:115` switches to calling it so the two can never drift.
+- [ ] **Step 4: Console.** In the fleet view (FleetView/handleFleet), add a per-box "Hosts" detail: call `status.LoadBoxes` against the store handle the console already holds for fleet.json, render per box — label, region, release, tunnel state, `mem avail/total` (humanized), load1, disk free — and when `BoxStale`: render the box as **unreachable** (distinct style + the age of the last heartbeat) and suppress the metric numbers entirely (an unreachable host must never show its last happy numbers — spec). Follow the surrounding template/view conventions exactly (vendored Lucide icons only, no CDN). Extend the existing console fleet-view test with: a stale box renders "unreachable" and does NOT render its stale metric values; a fresh box renders them.
+- [ ] **Step 5: CLI.** Replace RunStatus's dry-run-only shape with two modes (keep `-dry-run` byte-identical; update statusUsage — it currently *documents* having no live mode, and that sentence must go):
+
+```go
+liveFlag := fs.Bool("live", false, "read status/boxes/*.json from the status bucket and print per-host metrics")
+// ...
+if *liveFlag {
+ if err := statusLive(os.Stdout); err != nil {
+ log.Fatal(err)
+ }
+ return
+}
+```
+
+```go
+// statusLive prints one line per box: label, region, release, tunnel state,
+// memory, load, disk — or UNREACHABLE (+heartbeat age) for a stale box,
+// never its last happy numbers. Operator credentials; same bucket
+// resolution as the agent's own status loop.
+func statusLive(w io.Writer) error {
+ ctx := context.Background()
+ bucket := host.ResolveStatusBucket(os.Getenv) // export the existing helper from internal/host
+ if bucket == "" {
+ return fmt.Errorf("set CARLOS_STATUS_BUCKET or CARLOS_DEPLOYMENT_BUCKET")
+ }
+ s, err := release.NewS3Store(ctx, bucket)
+ if err != nil {
+ return err
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ return err
+ }
+ now := time.Now()
+ for _, b := range boxes {
+ if status.BoxStale(b, now, 5*time.Minute) {
+ fmt.Fprintf(w, "%-24s %-12s UNREACHABLE (last heartbeat %s ago)\n",
+ b.Label, b.Region, now.Sub(b.GeneratedAt).Round(time.Minute))
+ continue
+ }
+ m := b.Metrics
+ if m == nil {
+ fmt.Fprintf(w, "%-24s %-12s ok (no metrics block — pre-upgrade host)\n", b.Label, b.Region)
+ continue
+ }
+ fmt.Fprintf(w, "%-24s %-12s release=%s tunnel=%s mem=%s/%s load=%s disk-free=%s\n",
+ b.Label, b.Region, orDash(m.Release), orDash(m.Tunnel),
+ humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes),
+ humanFloat(m.Load1), humanBytes(m.DiskFreeBytes))
+ }
+ return nil
+}
+```
+
+(Write the tiny `orDash`/`humanBytes`/`humanFloat` helpers beside it — nil renders as `-`. Rename/export `resolveStatusBucket` → `host.ResolveStatusBucket` with a wrapper keeping the internal callsite compiling.)
+- [ ] **Step 6: Run everything** — `go test ./internal/status/ ./internal/console/ ./... -count=1` → PASS.
+- [ ] **Step 7: Full gate + commit** — `status/console/cli: render raw host metrics privately, stale hosts as unreachable`.
+
+---
+
+### Task 8: Off-cloud bootstrap (`carlos bootstrap -offcloud`) + runbook
+
+**Files:**
+- Modify: `internal/host/bootstrap.go` (RunBootstrap ~line 89, bootstrap ~line 104)
+- Create: `infra/remote-host.md`
+- Test: `internal/host/bootstrap_test.go` (read the existing staging-root tests first, extend in place)
+
+**Interfaces:**
+- Consumes: existing `bootstrap(root)` idempotent flow.
+- Produces: `-offcloud` flag; contract: credentials file at `<root>/etc/carlos/credentials` (mode 0600, owned by the service user on a live run) must exist BEFORE bootstrap proceeds; host.env must carry `AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials` (checked and printed, not silently assumed).
+
+- [ ] **Step 1: Write the failing test** (staging-root path — no root privileges in tests):
+
+```go
+func TestBootstrapOffcloudRequiresCredentialsFile(t *testing.T) {
+ root := t.TempDir()
+ if err := bootstrapOffcloud(root); err == nil {
+ t.Fatal("want error when <root>/etc/carlos/credentials is absent")
+ }
+ credDir := filepath.Join(root, "etc", "carlos")
+ os.MkdirAll(credDir, 0o755)
+ credFile := filepath.Join(credDir, "credentials")
+ os.WriteFile(credFile, []byte("[default]\naws_access_key_id=X\naws_secret_access_key=Y\n"), 0o644)
+ if err := bootstrapOffcloud(root); err == nil {
+ t.Fatal("want error on 0644 credentials (must be 0600)")
+ }
+ os.Chmod(credFile, 0o600)
+ if err := bootstrapOffcloud(root); err != nil {
+ t.Fatalf("offcloud bootstrap with staged 0600 credentials: %v", err)
+ }
+ // The normal bootstrap artifacts must exist too — offcloud is a superset.
+ if _, err := os.Stat(filepath.Join(root, "etc", "systemd", "system", "carlos-edge.service")); err != nil {
+ t.Fatalf("units not installed: %v", err)
+ }
+}
+```
+
+- [ ] **Step 2: Run to verify failure** → FAIL (undefined: bootstrapOffcloud).
+- [ ] **Step 3: Implement.**
+
+```go
+// bootstrapOffcloud is bootstrap for a box with no EC2 instance profile
+// (issue #53 point 4): identical structural work, plus the credential
+// preconditions — a staged 0600 credentials file scoped (at IAM) to the
+// deployment bucket + this host's own region bucket and nothing else, so
+// wrong-region-fails-at-IAM holds off-cloud too. The flag is explicit; no
+// IMDS sniffing (the agent's one IMDS probe is already gated for non-EC2).
+func bootstrapOffcloud(root string) error {
+ cred := filepath.Join(root, "etc", "carlos", "credentials")
+ fi, err := os.Stat(cred)
+ if err != nil {
+ return fmt.Errorf("offcloud: stage the scoped credentials file first: %s (0600, see infra/remote-host.md): %w", cred, err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ return fmt.Errorf("offcloud: %s must be mode 0600, is %o", cred, fi.Mode().Perm())
+ }
+ if err := bootstrap(root); err != nil {
+ return err
+ }
+ if filepath.Clean(root) == "/" {
+ if err := chownToService(cred); err != nil { // same helper the data dirs use
+ return err
+ }
+ }
+ fmt.Printf("offcloud bootstrap complete.\n ensure /etc/carlos/host.env sets:\n AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials\n then stage /opt/carlos/carlos and start carlos-edge.service (see infra/remote-host.md)\n")
+ return nil
+}
+```
+
+ Wire the flag in RunBootstrap: `offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile")`, dispatching to `bootstrapOffcloud(*root)` when set. If `chownToService`'s signature differs (read it at ~line 230), adapt the call — the intent is: service user owns the file, still 0600.
+- [ ] **Step 4: Run tests** — `go test ./internal/host/ -run Bootstrap -count=1` → PASS (and the pre-existing bootstrap tests still pass).
+- [ ] **Step 5: Write `infra/remote-host.md`** — the one-page software-half runbook (issue #53's deliverable; the hq PHYSICAL half lands later in `infra/tito/region-hq.md`):
+
+```markdown
+# Remote host runbook (software half)
+
+A CARLOS host is anything that can reach the deployment bucket. A remote
+host is OUTBOUND-ONLY: its whole network surface is outbound 443 to S3 and
+the edge. No inbound port, no SSH path required by the platform.
+
+Prereqs: systemd Debian-family OS, 64-bit (the artifact key is
+GOOS-GOARCH — linux-arm64 or linux-amd64; a 32-bit OS resolves nothing).
+
+1. Provision the IAM principal + region data bucket
+ (infra/modules/region-remote-host). Scope: deployment bucket + this
+ host's own region bucket, NOTHING else. IAM Roles Anywhere is the
+ long-term option; a scoped access key is acceptable to start.
+2. Stage /etc/carlos/credentials (0600):
+ [default]
+ aws_access_key_id=...
+ aws_secret_access_key=...
+3. Stage /etc/carlos/host.env — the usual agent env plus:
+ AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials
+ CARLOS_TUNNEL_URL=https://<reserved-tunnel-hostname>
+ CARLOS_TUNNEL_TOKEN=<this host's token>
+ CARLOS_TUNNEL_NAME=<this host's channel name>
+4. Stage the binary at /opt/carlos/carlos (matching GOOS-GOARCH).
+5. Run: carlos bootstrap -offcloud
+6. systemctl start carlos-edge.service
+7. Verify: journalctl -u carlos-edge shows "tunnel connected"; the box
+ appears in `carlos status -live` within one 5-minute heartbeat.
+
+Edge side: set CARLOS_TUNNEL_HOST + CARLOS_TUNNEL_CLIENTS
+(name=token,...) in the edge box's host.env; back routes with
+`carlos route -host <h> -tunnel <name>`.
+```
+
+- [ ] **Step 6: Full gate + commit** — `host: carlos bootstrap -offcloud + remote host runbook (software half)`.
+
+---
+
+### Task 9: Failover semantics — the dark-then-returns lease test
+
+**Files:**
+- Modify: `internal/lease/lease.go` (Acquire ~line 177 — extract a TTL seam)
+- Test: `internal/lease/lease_test.go` (live-gated harness: `liveConfig(t)` skips without `CARLOS_TEST_S3_BUCKET`)
+
+**Interfaces:**
+- Consumes: existing `Acquire`, `HeldError`, `liveConfig` harness.
+- Produces: unexported `acquireTTL(ctx, bucket, prefix, region string, ttl time.Duration) (*Held, error)`; `Acquire` becomes `return acquireTTL(ctx, bucket, prefix, region, LeaseTTL)`. No exported surface changes.
+
+Scope note: the wake-elsewhere half of failover (a fallback host's activator waking the instance from the regional bucket on request) is the EXISTING activator wake path, already exercised by the activator's own tests — nothing new to build or test there (issue #53: "verify it rather than build it"). What is genuinely new to prove is the returning-dark-host-must-lose property below.
+
+- [ ] **Step 1: Extract the seam** — mechanical: move Acquire's body into `acquireTTL` with `leaser.TTL = ttl`; Acquire delegates with `LeaseTTL`. Run `go test ./internal/lease/ -count=1` → PASS (everything skips or passes as before).
+- [ ] **Step 2: Write the dark-then-returns test** (proves issue #53 point 3: a returning host must NOT reclaim a lease now held elsewhere — the conditional-write race guarantees it; this test is the proof for the dark-then-returns sequence specifically):
+
+```go
+// TestDarkHostCannotReclaimAfterFailover: host A holds the lease and goes
+// dark (stops renewing, never releases). After TTL expiry, host B claims it
+// (the failover wake). A returns and tries to re-acquire: it MUST lose with
+// *HeldError naming B's ownership — never silently reclaim.
+func TestDarkHostCannotReclaimAfterFailover(t *testing.T) {
+ bucket, prefix, region := liveConfig(t)
+ ctx := context.Background()
+ const shortTTL = 15 * time.Second
+
+ a, err := acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ if err != nil {
+ t.Fatalf("A initial acquire: %v", err)
+ }
+ // A goes dark: stop renewal WITHOUT releasing (the crash/power-cut shape).
+ // rn.stop() halts the renewer but does not touch the lock object in S3.
+ a.rn.stop()
+
+ // While A's lease is still live, B must be refused — dark ≠ expired.
+ if _, err := acquireTTL(ctx, bucket, prefix, region, shortTTL); err == nil {
+ t.Fatal("B acquired while A's un-expired lease still held")
+ }
+
+ time.Sleep(shortTTL + 5*time.Second) // let A's lease expire
+
+ b, err := acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ if err != nil {
+ t.Fatalf("B acquire after expiry (the failover wake): %v", err)
+ }
+ defer b.Release(ctx)
+
+ // A returns from the dead and tries to pick up where it left off.
+ _, err = acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ var held *HeldError
+ if !errors.As(err, &held) {
+ t.Fatalf("returning dark host got %v, want *HeldError (must lose the race)", err)
+ }
+}
+```
+
+- [ ] **Step 3: Run gated** — without `CARLOS_TEST_S3_BUCKET`: `go test ./internal/lease/ -count=1` → SKIP cleanly. With it set (if operator credentials are available in the session): the test passes in ~30s. Record in the PR description whether the live run was executed.
+- [ ] **Step 4: Full gate + commit** — `lease: prove the dark-then-returns failover race (returning host must lose)`.
+
+---
+
+### Task 10: E2E — edge + fake remote host over a real reverse channel, reconnect bound
+
+**Files:**
+- Create: `internal/edge/tunnel_e2e_test.go`
+
+**Interfaces:**
+- Consumes: everything from Tasks 1–4. This is issue #53's acceptance test, in-repo, no network beyond loopback.
+
+- [ ] **Step 1: Write the e2e test:**
+
+```go
+package edge
+
+// The acceptance test issue #53 names: edge + fake remote host over a REAL
+// reverse channel on loopback. Round-trip, header survival, kill-mid-flight,
+// re-dial, and the reconnect bound (asserted, not narrated).
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+const e2eReconnectBound = 5 * time.Second
+
+func TestTunnelEndToEnd(t *testing.T) {
+ reg := testRegistryWithRoute(t, registry.Route{
+ Host: "demo.example", Kind: "instance", Status: "ready", Tunnel: "pi-box",
+ })
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ e.EnableTunnels(hub)
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ // The edge listens on loopback; both app traffic and the tunnel
+ // registration ride this one listener — the no-new-port property.
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ // Fake remote host: its "local edge" answers with a header + body, and
+ // echoes X-Forwarded-For so the test can prove header discipline held.
+ appHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-App-Header", "survived")
+ w.Header().Set("Carlos-Purge", "/should-be-stripped") // edge must strip
+ fmt.Fprintf(w, "host=%s xff=%s", r.Host, r.Header.Get("X-Forwarded-For"))
+ })
+ stop := startFakeRemoteHost(t, srv.URL, "pi-box", "sekrit", appHandler)
+ defer stop()
+
+ waitForState(t, hub, "pi-box", "connected", e2eReconnectBound)
+
+ do := func() *http.Response {
+ req, _ := http.NewRequest("GET", srv.URL+"/", nil)
+ req.Host = "demo.example"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request through tunnel: %v", err)
+ }
+ return resp
+ }
+ resp := do()
+ body, _ := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ if resp.StatusCode != 200 || resp.Header.Get("X-App-Header") != "survived" {
+ t.Fatalf("round-trip: %d %v", resp.StatusCode, resp.Header)
+ }
+ if resp.Header.Get("Carlos-Purge") != "" {
+ t.Fatal("Carlos-Purge leaked to the client — purge discipline broken over tunnel")
+ }
+ if !strings.Contains(string(body), "host=demo.example") || !strings.Contains(string(body), "xff=127.") {
+ t.Fatalf("host/XFF discipline broken: %q", body)
+ }
+
+ // Kill the channel out from under the host (edge side drops it), then
+ // assert the host re-dials and the NEXT request succeeds within bound.
+ hub.mu.Lock()
+ hub.conns["pi-box"].raw.Close()
+ delete(hub.conns, "pi-box")
+ hub.mu.Unlock()
+
+ start := time.Now()
+ waitForState(t, hub, "pi-box", "connected", e2eReconnectBound)
+ if resp := do(); resp.StatusCode != 200 {
+ t.Fatalf("post-reconnect request: %d", resp.StatusCode)
+ }
+ if d := time.Since(start); d > e2eReconnectBound {
+ t.Fatalf("reconnect took %s, bound is %s", d, e2eReconnectBound)
+ }
+}
+
+func waitForState(t *testing.T, hub *TunnelHub, name, want string, within time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(within)
+ for time.Now().Before(deadline) {
+ if hub.State(name) == want {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatalf("tunnel %s never reached state %q within %s", name, want, within)
+}
+```
+
+`startFakeRemoteHost` reuses the HOST-side client for real: it must call `internal/host`'s actual dial+serve+backoff loop, not a reimplementation — otherwise the reconnect bound proves nothing. `internal/host` already imports `internal/edge`, so the edge test can't import host (cycle). Resolve it the repo-idiomatic way: extract the host's dial/serve/backoff core (`dialTunnel`, `serveTunnel`, `bufferedConn`, the retry loop, backoff constants) into a tiny new leaf package `internal/tunnelclient` (no imports beyond stdlib + x/net), with `internal/host/tunnel_client.go` becoming a thin wrapper that reads env, stores state via a `func(string)` callback, and calls it. Then `startFakeRemoteHost` runs `tunnelclient.Loop(ctx, url, name, token, handler, onState)` in a goroutine and returns a cancel func. Update Task 4's files accordingly when executing this task (the tests written there keep passing — they exercise the same functions through the new package).
+- [ ] **Step 2: Run it** — `go test ./internal/edge/ -run TestTunnelEndToEnd -count=1 -v` → PASS, reconnect measured under 5s.
+- [ ] **Step 3: Race check** — `go test ./internal/edge/ ./internal/tunnelclient/ -count=1 -race` → PASS.
+- [ ] **Step 4: Full gate + commit** — `edge: e2e reverse-channel acceptance — round-trip, header discipline, reconnect bound`.
+
+---
+
+### Task 11: Final verification + draft PR
+
+- [ ] **Step 1:** `go build ./... && go vet ./... && go test ./... -count=1` — all packages (now 15+ with tunnelclient; CLAUDE.md's "14 packages" line is stale the moment tunnelclient lands — update CLAUDE.md's count in this commit). `gofmt -l` on every touched file → empty.
+- [ ] **Step 2:** `go test ./internal/release/ -run Ladder -count=1` → the pinned production channel names still pass.
+- [ ] **Step 3:** Push branch, open DRAFT PR titled "Remote host-agent: tunnel route backing, heartbeat metrics, off-cloud bootstrap (#53)". Body: link issue #53 + the spec; list the three recorded divergences (top of this plan); state which acceptance tests ran live vs gated (Task 9). End with the Claude Code attribution footer.
+- [ ] **Step 4:** Request Opus review (per CLAUDE.md) on: `internal/edge/tunnel.go` (auth, hijack lifecycle), the Reload/ServeHTTP changes, `internal/tunnelclient`, and the lease test.
+- [ ] **Step 5:** Once PR #49 (plan-c update) is merged to main: tick C6 Step 3 in `docs/superpowers/plans/2026-08-01-plan-c-tito-selfhosted.md` and record the divergences there. If #49 is still open at merge time, leave a comment on #49 instead so the tick isn't lost.
diff --git a/docs/superpowers/specs/2026-08-04-remote-host-pi-design.md b/docs/superpowers/specs/2026-08-04-remote-host-pi-design.md
new file mode 100644
index 000000000..8bcf53651
--- /dev/null
+++ b/docs/superpowers/specs/2026-08-04-remote-host-pi-design.md
@@ -0,0 +1,267 @@
+# Remote host-agent + Raspberry Pi instance host — design
+
+**Date:** 2026-08-04
+**Charter:** [platform#53](https://github.com/carlosframework/platform/issues/53)
+(C6 Step 3 of `docs/superpowers/plans/2026-08-01-plan-c-tito-selfhosted.md`,
+PR #49) plus Raspberry Pi enablement on the dev platform.
+**Settled upstream, not relitigated here:** the control plane is the
+deployment bucket; a remote host is outbound-only (443 to S3 and the edge —
+no inbound port, no SSH path, no push channel); this is the existing
+`carlos` binary in host-agent role, not a new binary.
+
+## Goal
+
+Two deliverables, two stacked PRs:
+
+1. **PR 1 — remote host-agent machinery (#53 verbatim):** tunnel route
+ backing on the edge, host metrics in heartbeats, placement/failover
+ verification, off-cloud bootstrap. Deployment-agnostic.
+2. **PR 2 — Pi enablement:** a Raspberry Pi 4 (4 GB) joins the **dev
+ platform** (Kass account, 640012953448) as instance host for region
+ `pi`, hosting a demo fleet — proving "a CARLOS host is anything that
+ can reach the deployment bucket" on a $60 board, and rehearsing
+ exactly the machinery tito's `hq` office box needs.
+
+Demo blast radius stays off production; the pocket-Pi demo only needs
+outbound 443, which survives venue Wi-Fi.
+
+## PR 1 — remote host-agent machinery
+
+### Tunnel route backing
+
+- `registry.Route` gains an additive `Tunnel` column: the name of the
+ reverse channel backing the route. Channel name = the owning host's
+ box label. Wire/DB shape additive; routes written before this change
+ parse unchanged.
+- The dispatch switch in `internal/edge/edge.go` `Reload()` (today:
+ socket → addr → static → status) gains a fourth backing:
+ `t.Tunnel != ""` → `tunnelProxy`, the same `httputil.ReverseProxy`
+ shape as `tcpProxy` but with a RoundTripper bound to the named
+ channel. Header discipline is inherited because it is the same proxy
+ path: `SetXForwarded`, `scrubInboundPurgeHeader` inbound,
+ `applyPurgeHeader` outbound. **A tunnel-backed host is exactly as
+ untrusted as an addr-backed one.**
+
+ **Known limitations (final review, 2026-08-04):**
+ - **Second-hop header fidelity.** That inherited-discipline claim holds
+ for the first hop (client → edge) but not the second: a tunnel-backed
+ request also crosses the remote host's OWN `*edge.Edge` — the same
+ `ReverseProxy` code, but a second, independent hop with its own
+ `SetXForwarded` call. Stdlib `Rewrite`-mode proxying deletes any
+ inbound `X-Forwarded-For`/`X-Forwarded-Proto` and re-stamps them from
+ the immediate peer, so the app on the remote host sees the edge's own
+ IP as client (not the real one) and `X-Forwarded-Proto: http` (h2
+ `ServeConn` over the hijacked tunnel conn has `r.TLS == nil`, even
+ though the original client-to-edge hop was TLS). This fails in the
+ safe direction — client identity is DROPPED, never spoofed; a
+ tunnel-backed app cannot be tricked into trusting a forged
+ `X-Forwarded-*` from outside. Future fix: the remote host's edge could
+ special-case its own tunnel listener and trust/forward the inbound
+ `X-Forwarded-*` it already received over that hijacked conn, since
+ that hop terminates the same `TunnelHub` handshake this file
+ implements — not general inbound trust.
+ - **No WebSockets over a tunnel-backed route.** The edge's upgrade path
+ hands `Connection: Upgrade` requests straight to the route's
+ `ReverseProxy`; a tunnel's transport is an `http2.ClientConn`
+ (`RoundTrip`), which rejects connection-specific headers, so every
+ WebSocket handshake to a tunnel-backed host fails closed — a loud 502,
+ never a silent hang or a downgraded connection. Future door: RFC 8441
+ extended CONNECT, which carries a bidirectional stream inside h2 and
+ is the standards-track way to tunnel a WebSocket over an HTTP/2
+ connection.
+
+### Transport: reverse HTTP/2 over a host-dialed TLS connection
+
+Decision (the in-repo judgment call #53 asks for, recorded): we port the
+*idea* of titogo's `internal/tunnel` — persistent authenticated dial-out
+— but not its websocket transport.
+
+- The host dials the edge's **existing 443 listener** — no new port —
+ with an upgrade request to a reserved path carrying
+ `Authorization: Bearer <token>` and its channel name.
+- The edge verifies the token (constant-time; the token is bound to the
+ channel name it may claim — dev configures one `name=token` pair in
+ the edge environment, hq can list several), hijacks the connection,
+ and **inverts** it: the edge runs an `http2.Transport` client over
+ the hijacked conn; the host runs an `http2.Server` whose handler is
+ its own local edge routing.
+- HTTP/2 supplies stream multiplexing, flow control, and pings.
+ A failed ping tears the channel down and flips its state to
+ `reconnecting`. `golang.org/x/net` is already in the module graph;
+ no websocket dependency is added.
+- New `internal/edge/tunnel.go`: the channel hub (name → live channel),
+ registration handler, token check, lifecycle.
+
+Rejected: porting the websocket transport (vendors a dependency and
+hand-rolls multiplexing h2 gives us); a dedicated listener port (venue
+and office networks may only pass 443).
+
+### Host dial-out
+
+- New `tunnelLoop` goroutine in `RunAgent` (`internal/host/agent.go`),
+ beside `statusLoop`/`configSyncLoop`; active only when
+ `CARLOS_TUNNEL_URL` + `CARLOS_TUNNEL_TOKEN` are configured.
+- Jittered exponential backoff **capped low**: the acceptance test
+ asserts a reconnect bound in seconds. hq's dual-WAN failover is a
+ re-dial over the surviving link; the Pi's Wi-Fi blip is the same
+ event. Reconnect latency is user-visible.
+- While a channel is down the edge answers its routes 502; the live
+ edge cache keeps public pages serving through a blip.
+
+### Host metrics in heartbeats
+
+The heartbeat record is the existing fleet status Box snapshot
+(`status/boxes/<label>.json`, writer `internal/status/snapshot.go`
+`WriteBox`, driven by `statusLoop`). Decision: **extend Box with an
+optional additive `metrics` block; band or strip before anything
+public.**
+
+- New fields: memory total/available, load, disk free, resident and
+ parked instance counts, current release + channel, tunnel/channel
+ state (connected/reconnecting; egress link hint if knowable).
+- The public fleet merger (`internal/status/fleet.go` `BuildFleet` →
+ `PublishFleet`) continues to publish **only the existing banded
+ words**. Raw numbers never reach the publicly served `fleet.json` —
+ the privacy banding in `CollectBox` was deliberate and survives.
+- Consumers of the raw block (consumer list = everything that PARSES
+ the shape): the private, session-gated console fleet view renders raw
+ metrics from an in-process unbanded merge; `carlos status` gains a
+ live mode reading box snapshots from the bucket (operator
+ credentials). Nothing else parses Box today (verified: edge status
+ serving is byte-passthrough; activator and ops do not parse it).
+- **Additive:** Boxes written by old hosts must still parse — pinned by
+ a back-compat test fixture (yesterday's shape parses today).
+- Staleness is the one existing rule: a Box older than 3× the publish
+ period renders **unreachable** — never its last happy numbers — in
+ both console fleet view and CLI live status.
+
+### Placement + failover: verify, don't build
+
+- No pool machinery. `Route.Pool`/`Route.Region` remain dormant
+ columns. Placement stays IAM + topology: one region, one data
+ bucket, one host that can reach it.
+- New tests prove the dark-then-returns sequence end to end: host goes
+ dark (stale heartbeat + lease expiry) → instance wakes on a host
+ with fallback eligibility from the regional bucket → the returning
+ host **loses** the conditional-write race and does not reclaim.
+- No redirect verb exists or is added.
+
+### Off-cloud bootstrap
+
+- `carlos bootstrap -offcloud`: identical structural work (units, dirs,
+ service user, timers — Raspberry Pi OS and the office box are both
+ systemd Debian), but expects a **staged credentials file** (0600, in
+ the credentials dir, owned by the service user) instead of an EC2
+ instance profile. The flag is explicit; no IMDS sniffing.
+- Credential scope: the deployment bucket + the host's own region data
+ bucket and **nothing else** — wrong-region-fails-at-IAM holds
+ off-cloud. IAM Roles Anywhere is documented as the hq-grade
+ alternative to a long-lived key.
+- The existing IMDS public-IP probe is already gated off for non-EC2.
+- Deliverable includes the one-page **software-half host runbook**
+ shape (the hq physical runbook lands later in
+ `infra/tito/region-hq.md`).
+
+### Acceptance (tests, not narration)
+
+- E2E in-repo: edge + fake remote host over a real reverse channel;
+ route resolves, request round-trips, response headers survive.
+- Kill the channel mid-flight → host re-dials → next request succeeds;
+ measure and assert a reconnect bound.
+- Stale-heartbeat host renders unreachable; its instance wakes on a
+ fallback host; the returning host loses the lease race.
+- Heartbeat parse test: yesterday's shape still parses today.
+- All 14 packages: `go build`, `go vet`, `go test -count=1`, `gofmt`
+ clean on touched files; pinned production channel names in
+ `internal/release/laddercheck_test.go` keep passing.
+
+Per CLAUDE.md delegation policy this is production-path work: **Opus
+reviews** the edge listener, auth, and lease/failover pieces.
+
+## PR 2 — Raspberry Pi enablement (dev platform)
+
+### Infra
+
+- New reusable tofu module `infra/modules/region-remote-host`: the
+ region's data bucket + an off-cloud IAM principal scoped to exactly
+ the deployment bucket and that region's bucket. Pi uses an IAM user
+ + access key; the module shape leaves room for an IAM Roles Anywhere
+ trust later. This is deliberately the module tito's hq Step 1 reuses.
+- The dev root module (platform-infrastructure checkout, like other
+ roots) instantiates it as region `pi`.
+
+### Region + placement
+
+- Dev gets a `pi` region entry. Demo instances are created with
+ residency `pi`; the Pi's activator — the only one pointed at that
+ bucket — wakes them.
+- **No live fallback** (decision): Pi instances are Pi-resident,
+ period. A dark Pi renders unreachable; it never silently migrates the
+ demo to the cloud host mid-blip. Failover semantics are proven by
+ PR 1's tests, not wired live here.
+- No tier field, no provisioner changes — that is tito's C6 Step 2.
+
+### The Pi itself
+
+Hardware: Raspberry Pi 4, 4 GB. Runbook `infra/dev/region-pi.md`
+(software half, mirroring the hq runbook shape):
+
+- **64-bit Raspberry Pi OS Lite is mandatory** — hosts pick artifacts
+ by `runtime.GOOS + "-" + runtime.GOARCH`; a 32-bit OS resolves no
+ artifact.
+- SSD over USB3 recommended; high-endurance SD acceptable for the demo.
+ SQLite + WAL streaming is the I/O load and SD I/O is the honest
+ constraint on a Pi 4.
+- Stage the `linux-arm64` binary + `host.env` + credentials file, run
+ `carlos bootstrap -offcloud`, verify the dial-out.
+- **Zero build-pipeline work:** ship's default target is already
+ `linux-arm64` and the fleet is arm64 Graviton; the Pi eats the same
+ artifact.
+
+### Demo seeding + success criteria
+
+A small script creates the demo fleet via `carlos apps create` (the
+PR #52 console-API path) against dev: **≥5 resident + ≥50 parked**
+instances (hibernation makes parked ~free; resident count is bounded by
+per-app footprint in 4 GB). The demo proves:
+
+1. The Pi appears in the fleet view with real metrics and its channel
+ state.
+2. Instances serve through the dev edge over the reverse channel with
+ correct headers.
+3. A parked instance wakes on first request through the tunnel.
+4. Pull the Wi-Fi → the next request after re-dial succeeds within the
+ asserted reconnect bound.
+5. Power the Pi off → it renders unreachable; power it on → it resumes
+ serving. Nothing migrated.
+
+That is the viability claim demonstrated rather than asserted: *a
+CARLOS host is anything that can reach the deployment bucket.*
+
+## Error handling (both PRs)
+
+- Token mismatch → 401; the host backs off with a cap (no thundering
+ re-dial) and logs.
+- Half-dead channel → h2 ping timeout closes it; the host re-dials.
+- Bucket unreachable from the host → heartbeats stop → unreachable in
+ fleet after 3× period.
+- Channel down → 502 from the edge; edge cache covers public pages.
+
+## Sequencing
+
+1. PR 1 in its own worktree/branch/draft PR; Opus review on edge
+ listener, auth, lease/failover. On merge: tick C6 Step 3 in Plan C
+ (once PR #49's plan text is on main) and record divergences.
+2. PR 2 stacks on PR 1: tofu module, dev `pi` region, runbook, demo
+ seeding.
+3. Website-feature-sync rule applies at merge: "run CARLOS on a
+ Raspberry Pi" is website material.
+
+## Out of scope
+
+- C6 Steps 1, 2, 4–7 (tito hq bucket/IAM tofu, tier/provisioner,
+ hardware, relocation verb, canary, default flip).
+- Pool/eligibility machinery (dormant columns stay dormant).
+- Live fallback for the Pi region.
+- Multi-arch CI (manual `make build GOARCH=...` remains the build
+ path).
diff --git a/go.mod b/go.mod
index 642039770..b1bfa10f6 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
github.com/aws/smithy-go v1.27.3
github.com/benbjohnson/litestream v0.5.14
golang.org/x/crypto v0.52.0
+ golang.org/x/net v0.55.0
modernc.org/sqlite v1.55.0
)
@@ -53,7 +54,6 @@ require (
github.com/superfly/ltx v0.5.1 // indirect
github.com/tetratelabs/wazero v1.2.1 // indirect
github.com/wasilibs/go-re2 v1.3.0 // indirect
- golang.org/x/net v0.55.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.37.0 // indirect
diff --git a/infra/remote-host.md b/infra/remote-host.md
new file mode 100644
index 000000000..9782cfdb4
--- /dev/null
+++ b/infra/remote-host.md
@@ -0,0 +1,95 @@
+# Remote host runbook (software half)
+
+A CARLOS host is anything that can reach the deployment bucket. A remote
+host is OUTBOUND-ONLY: its whole network surface is outbound 443 to S3 and
+the edge tunnel. No inbound port, no SSH path required by the platform.
+
+Prereqs: systemd Debian-family OS, 64-bit (the artifact key is
+GOOS-GOARCH — linux-arm64 or linux-amd64; a 32-bit OS resolves nothing).
+
+## Steps
+
+1. Provision the IAM principal + region data bucket
+ (`infra/modules/region-remote-host`). Scope: deployment bucket + this
+ host's own region bucket, and NOTHING else — wrong-region-fails-at-IAM
+ holds off-cloud too. A scoped access key is acceptable to start; IAM
+ Roles Anywhere is the long-term, hq-grade alternative (no long-lived
+ key on the box at all).
+
+2. Stage `/etc/carlos/credentials` (mode 0600):
+
+ [default]
+ aws_access_key_id=...
+ aws_secret_access_key=...
+
+ `carlos bootstrap -offcloud` refuses to proceed if this file is
+ missing or not exactly 0600 — it will name this path in the error.
+
+3. Stage `/etc/carlos/host.env` — the usual agent env, plus the
+ credentials pointer and the three tunnel variables. All three tunnel
+ variables are required together; if any is missing the dial-out loop
+ stays inert (no crash, no retry storm — it just never dials):
+
+ AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials
+ CARLOS_TUNNEL_URL=https://<edge's reserved tunnel hostname>
+ CARLOS_TUNNEL_TOKEN=<this host's token>
+ CARLOS_TUNNEL_NAME=<this host's channel name>
+
+4. Stage the binary at `/opt/carlos/carlos` (matching GOOS-GOARCH).
+
+5. Run: `carlos bootstrap -offcloud`
+
+ This does everything the normal `carlos bootstrap` does (units, dirs,
+ service user, timers) and, before any of it, checks the credentials
+ precondition from step 2 — a failed check leaves the box untouched. On
+ a live root it also chowns the credentials file to the service user
+ (still mode 0600).
+
+6. `systemctl start carlos-edge.service`
+
+7. Verify:
+ - `journalctl -u carlos-edge` shows "tunnel connected".
+ - The box appears in `carlos status -live` within one heartbeat
+ (5 minutes + jitter).
+
+## Edge side (once per edge box, not per remote host)
+
+Set in the edge box's `host.env`:
+
+ CARLOS_TUNNEL_HOST=<reserved hostname>
+ CARLOS_TUNNEL_CLIENTS=name=token,name2=token2,...
+
+`CARLOS_TUNNEL_HOST` must be a bare hostname — no port, no scheme. Each
+name in `CARLOS_TUNNEL_CLIENTS` must match `^[a-z0-9-]{1,32}$` or the edge
+refuses to boot.
+
+Back routes to a tunneled host with:
+
+ carlos route -host <h> -tunnel <name>
+
+## Security note
+
+In `-dev` (HTTP) edge mode the registration bearer token travels in clear
+text. Tunnel hosts must only ever dial a real-TLS edge in production —
+never point `CARLOS_TUNNEL_URL` at an HTTP edge outside a dev sandbox.
+
+## Known limitations
+
+- **Apps see the edge's IP, not the real client's.** A tunnel-backed request
+ crosses two reverse-proxy hops: client → edge (over the internet), then
+ edge → app (over the tunnel, on the remote host's own local edge). The
+ second hop re-stamps `X-Forwarded-For`/`X-Forwarded-Proto` from its own
+ immediate peer, so an app on a tunnel-backed host sees the edge's IP as
+ client, and `X-Forwarded-Proto: http` even though the original request was
+ TLS. This fails safe — client identity is dropped, never spoofed — but any
+ app on a tunnel-backed host that logs or gates on client IP will be wrong
+ today.
+- **No WebSockets to a tunnel-backed host.** The tunnel transport can't carry
+ a `Connection: Upgrade` handshake; every WebSocket attempt to an app on a
+ tunnel-backed host fails with a 502 (loud, not a hang). Plain HTTP request/
+ response is unaffected. RFC 8441 extended CONNECT is the likely future fix.
+
+## Related
+
+The hq PHYSICAL half (racking, power, network uplink for a region-hq box)
+lands separately in `infra/tito/region-hq.md`.
diff --git a/internal/console/server.go b/internal/console/server.go
index a82ddacd2..a24650380 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -97,6 +97,16 @@ var templates = template.Must(template.New("").Funcs(template.FuncMap{
return fmt.Sprintf("%d B", *v)
}
},
+ // loadOrDash renders a 1-minute load average: an absolute number of
+ // runnable tasks, not a fraction, so pct (which multiplies by 100 and
+ // stamps a %) would be a lie about the same *float64. nil is the same
+ // em-dash every unknown on this console gets.
+ "loadOrDash": func(v *float64) string {
+ if v == nil {
+ return "—"
+ }
+ return fmt.Sprintf("%.2f", *v)
+ },
// mapGet reads one named field out of Fleet.Instances — a map so a
// template range would iterate it key-sorted, alphabetically, which
// scrambles the static/resident/hibernating/awake/asleep order the
@@ -325,6 +335,14 @@ type page struct {
Fleet *status.Fleet // /fleet: nil means no snapshot published yet
+ // /fleet, below the published boxes table: the per-host PRIVATE detail
+ // read straight out of status/boxes/ (HostRows, view.go). Raw numbers
+ // render here and nowhere public — this page is session-gated to
+ // operators, fleet.json is not. Nil renders no section at all: no boxes
+ // yet, no bucket handle, or a boxes read that failed are all "no detail
+ // to show", never a reason to fail the page.
+ Hosts []HostRow
+
// /fleet/metrics: the operator dashboard (Task 11), both from LoadMetrics
// (metricsview.go) and both nil-safe to render — no boxes and no firing
// alerts are the normal, healthy, empty state, not an error.
@@ -1037,7 +1055,16 @@ func (s *Server) handleFleet(w http.ResponseWriter, r *http.Request) {
s.renderMessage(w, r, http.StatusInternalServerError, "Error", "Could not read fleet status. Please try again.")
return
}
- p := page{Title: "Fleet status", Address: addr, CSRF: raw, Fleet: f}
+ // The private host detail is an EXTRA read on a page that already has its
+ // answer: a bucket that won't list the boxes costs the operator the
+ // per-host block, not the whole fleet page, so this degrades with a log
+ // line rather than turning a readable page into a 500.
+ hosts, err := HostRows(r.Context(), s.cfg.Releases, time.Now())
+ if err != nil {
+ log.Printf("console: fleet hosts: %v", err)
+ hosts = nil
+ }
+ p := page{Title: "Fleet status", Address: addr, CSRF: raw, Fleet: f, Hosts: hosts}
s.render(w, http.StatusOK, "fleet", p)
}
diff --git a/internal/console/server_test.go b/internal/console/server_test.go
index 99a94cb4d..9c5d054a9 100644
--- a/internal/console/server_test.go
+++ b/internal/console/server_test.go
@@ -1161,6 +1161,133 @@ func TestFleetKeylessReadable(t *testing.T) {
}
}
+// seedHostBoxes writes one fresh and one long-dark box snapshot, both
+// carrying a private Metrics block, and publishes the merged feed over them —
+// the exact pair /fleet's Hosts detail has to tell apart.
+func seedHostBoxes(t *testing.T, fx *consoleFixture) {
+ t.Helper()
+ ctx := context.Background()
+ now := time.Now()
+ for _, b := range []status.Box{
+ {
+ GeneratedAt: now, Label: "ada-stargazing", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: pi64(2 << 30), MemAvailBytes: pi64(1 << 30),
+ Load1: p(0.42), DiskFreeBytes: pi64(5 << 30),
+ Release: "abc1234", Tunnel: "connected",
+ },
+ },
+ {
+ GeneratedAt: now.Add(-time.Hour), Label: "grace-kayaking", Region: "eu-west-2",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: pi64(9 << 30), MemAvailBytes: pi64(7 << 30),
+ Load1: p(8.25), DiskFreeBytes: pi64(3 << 30),
+ Release: "beefcaf", Tunnel: "connected",
+ },
+ },
+ } {
+ if err := status.WriteBox(ctx, fx.rel, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ f, err := status.BuildFleet(ctx, fx.rel, nil, now, status.SnapshotPeriod)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := status.PublishFleet(ctx, fx.rel, f); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestFleetHostsDetailSuppressesStaleMetrics is Task 7's core rule, on the one
+// surface allowed to render raw numbers at all: the operator fleet page shows
+// a live host's release, tunnel state, memory, load and free disk — and shows
+// an unreachable host as unreachable, with how long it has been dark and NOT
+// ONE of its last happy numbers. A dead box's final reading rendered as a
+// current figure is the failure this whole task exists to prevent.
+func TestFleetHostsDetailSuppressesStaleMetrics(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedHostBoxes(t, fx)
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 {
+ t.Fatalf("/fleet = %d: %s", code, body)
+ }
+ for _, want := range []string{
+ "Hosts", "ada-stargazing", "abc1234", "connected",
+ "1.0 GB", "2.0 GB", "0.42", "5.0 GB", // the fresh box's raw readings
+ "grace-kayaking", "unreachable",
+ } {
+ if !strings.Contains(body, want) {
+ t.Fatalf("/fleet hosts detail missing %q in:\n%s", want, body)
+ }
+ }
+ for _, leaked := range []string{"beefcaf", "8.25", "7.0 GB", "9.0 GB", "3.0 GB"} {
+ if strings.Contains(body, leaked) {
+ t.Fatalf("unreachable host's stale reading %q rendered — a dark box must show no numbers:\n%s", leaked, body)
+ }
+ }
+}
+
+// TestFleetHostsDegradeWhenUnreadable: the Hosts detail is an extra read on a
+// page that already had a job. If the boxes can't be listed, the page still
+// renders the merged feed it could read — metrics are never the reason /fleet
+// 500s.
+func TestFleetHostsDegradeWhenUnreadable(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedFleetJSON(t, fx)
+ fx.srv.cfg.Releases = listFailStore{fx.rel}
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 || !strings.Contains(body, "Fleet status") || !strings.Contains(body, "ada-stargazing") {
+ t.Fatalf("/fleet with an unlistable bucket = %d, want the fleet feed still rendered:\n%s", code, body)
+ }
+}
+
+// TestFleetHostsEscapesHostSuppliedStrings: Release and Tunnel are strings a
+// BOX writes into its own snapshot — the console reads them, it does not
+// mint them — so a compromised or buggy host must not be able to put markup
+// on an operator's page. html/template escapes them because they are plain
+// strings all the way through (never template.HTML, unlike icons.go's
+// vendored SVG); this test is what keeps that true.
+func TestFleetHostsEscapesHostSuppliedStrings(t *testing.T) {
+ fx := newConsoleFixture(t)
+ if err := status.WriteBox(context.Background(), fx.rel, status.Box{
+ GeneratedAt: time.Now(), Label: "leo-juggling", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{Release: "<script>alert(1)</script>", Tunnel: "connected"},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 {
+ t.Fatalf("/fleet = %d: %s", code, body)
+ }
+ if strings.Contains(body, "<script>alert(1)</script>") {
+ t.Fatalf("a host-supplied release string reached the page unescaped:\n%s", body)
+ }
+ if !strings.Contains(body, "&lt;script&gt;alert(1)&lt;/script&gt;") {
+ t.Fatalf("escaped release string not rendered at all:\n%s", body)
+ }
+}
+
+// listFailStore is a store whose List always fails — everything else passes
+// through, so fleet.json still reads and only the boxes sweep breaks.
+type listFailStore struct{ release.Store }
+
+func (listFailStore) List(ctx context.Context, prefix string) ([]string, error) {
+ return nil, errors.New("bucket list refused")
+}
+
// TestFleetMetricsOperatorOnly is Task 11's gate, mirroring TestFleetOperatorOnly
// for /fleet/metrics: signed out and signed in-but-not-operator both get
// exactly the same 404 an unrouted URL gets; an operator gets 200 with a
diff --git a/internal/console/templates/fleet.html b/internal/console/templates/fleet.html
index 6585b394b..f6b745edd 100644
--- a/internal/console/templates/fleet.html
+++ b/internal/console/templates/fleet.html
@@ -93,4 +93,40 @@
{{else}}
<p>No snapshots yet — the fleet status writer hasn't published <code>status/fleet.json</code> on this deployment.</p>
{{end}}
+
+{{if .Hosts}}
+<h2>Hosts</h2>
+<div class="tablewrap">
+<table>
+ <tr>
+ <th>Host</th><th>Region</th><th>Release</th><th>Tunnel</th>
+ <th>Memory free</th><th>Load</th><th>Disk free</th><th>Last report</th>
+ </tr>
+ {{range .Hosts}}
+ <tr>
+ <td>{{.Label}}</td>
+ <td>{{emptyOrDash .Region}}</td>
+ {{if .Metrics}}
+ <td>{{emptyOrDash .Metrics.Release}}</td>
+ <td>{{emptyOrDash .Metrics.Tunnel}}</td>
+ <td>{{bytesOrDash .Metrics.MemAvailBytes}} / {{bytesOrDash .Metrics.MemTotalBytes}}</td>
+ <td>{{loadOrDash .Metrics.Load1}}</td>
+ <td>{{bytesOrDash .Metrics.DiskFreeBytes}}</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{else if .Stale}}
+ {{/* An unreachable host says how long it has been dark and nothing else:
+ its last readings are history, and a history rendered in a "now"
+ column is how an outage gets missed. */}}
+ <td colspan="5"><span class="chip chip-down">unreachable</span> last heartbeat {{.Dark}} ago</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{else}}
+ <td colspan="5" class="small">no metrics block — this host predates raw metric reporting</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{end}}
+ </tr>
+ {{end}}
+</table>
+</div>
+<p class="small">Read from <code>status/boxes/</code> in the deployment bucket, which is not public — these are raw figures, unlike the aggregate feed above, and this page is why they stay behind a session.</p>
+{{end}}
{{template "bottom" .}}{{end}}
diff --git a/internal/console/view.go b/internal/console/view.go
index 2f34e51ed..25aab922a 100644
--- a/internal/console/view.go
+++ b/internal/console/view.go
@@ -11,6 +11,7 @@ import (
"errors"
"os"
"sort"
+ "strconv"
"strings"
"sync"
"time"
@@ -447,6 +448,89 @@ func FleetView(ctx context.Context, s release.Store) (*status.Fleet, error) {
return &f, nil
}
+// HostRow is one box's PRIVATE detail line on the operator fleet page: the
+// raw block status.Box carries and fleet.json deliberately never publishes
+// (internal/status's package comment), plus whether the box is still
+// reporting at all.
+//
+// Metrics is nil for a box that is STALE as well as for one that never
+// reported any — suppression happens here, in the view model, not in the
+// template. That is the point: an unreachable host's last happy numbers must
+// never reach the page, and the only way to be sure of that is for the page
+// to never be handed them. A template condition would be one careless
+// {{if}}-rearrangement away from printing a dead box's memory as a current
+// figure.
+type HostRow struct {
+ Label string
+ Region string
+ // Stale is status.BoxStale — the fleet's one staleness rule, measured
+ // against status.SnapshotPeriod, so this table and the published boxes
+ // table above it can never disagree about which host is dark.
+ Stale bool
+ // LastSeen is the snapshot's own GeneratedAt and Dark is how long ago that
+ // was, pre-rendered ("1h2m"). Dark is set only for a stale row: it is the
+ // one thing an unreachable host is allowed to say about itself.
+ LastSeen time.Time
+ Dark string
+ Metrics *status.BoxMetrics
+}
+
+// HostRows reads every per-box snapshot (status.LoadBoxes — the same loader
+// and the same skip-corrupt rule the public merge uses) and turns it into the
+// operator page's private host table, sorted by label so the order is stable
+// across refreshes.
+//
+// A nil store is not an error: a deployment configured without a bucket
+// handle simply has no host detail to show, and the fleet page renders
+// without the section rather than failing. Same posture for the caller on a
+// read error — see handleFleet.
+func HostRows(ctx context.Context, s release.Store, now time.Time) ([]HostRow, error) {
+ if s == nil {
+ return nil, nil
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ return nil, err
+ }
+ rows := make([]HostRow, 0, len(boxes))
+ for _, b := range boxes {
+ row := HostRow{
+ Label: b.Label,
+ Region: b.Region,
+ Stale: status.BoxStale(b, now, status.SnapshotPeriod),
+ LastSeen: b.GeneratedAt,
+ Metrics: b.Metrics,
+ }
+ if row.Stale {
+ row.Dark = humanAge(now.Sub(b.GeneratedAt))
+ row.Metrics = nil // an unreachable host shows no numbers at all
+ }
+ rows = append(rows, row)
+ }
+ sort.Slice(rows, func(i, j int) bool { return rows[i].Label < rows[j].Label })
+ return rows, nil
+}
+
+// humanAge renders how long a box has been dark at the resolution an operator
+// actually reads it at: minutes up to an hour, then hours and minutes, then
+// days. Never seconds — a snapshot is only ever called stale after three
+// missed five-minute ticks, so second-level precision would be false detail —
+// and never a negative age: a box whose clock runs ahead of the console's
+// reads as "0m", not as a time traveller.
+func humanAge(d time.Duration) string {
+ if d < 0 {
+ d = 0
+ }
+ switch {
+ case d < time.Hour:
+ return strconv.Itoa(int(d.Minutes())) + "m"
+ case d < 24*time.Hour:
+ return strconv.Itoa(int(d.Hours())) + "h" + strconv.Itoa(int(d.Minutes())%60) + "m"
+ default:
+ return strconv.Itoa(int(d.Hours())/24) + "d" + strconv.Itoa(int(d.Hours())%24) + "h"
+ }
+}
+
// versionsCap bounds the app page's release list.
const versionsCap = 20
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index 1dc161356..d4835aabd 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -23,6 +23,7 @@ import (
"flag"
"fmt"
"log"
+ "maps"
"net"
"net/http"
"net/http/httputil"
@@ -99,6 +100,13 @@ type Edge struct {
staticPub ed25519.PublicKey
staticCacheDir string
staticRecheck time.Duration
+ // tunnels, when non-nil, is the reverse-channel hub (tunnel.go): remote
+ // hosts register on its reserved hostname and routes with Tunnel set are
+ // proxied over the channel they name. nil = tunnel serving not configured
+ // on this box: a route's Tunnel binding is then ignored and it falls back
+ // to its socket/addr, or goes unserved if it has neither — the same
+ // nil-means-disabled convention as act/cache/staticStore.
+ tunnels *TunnelHub
// cache, when non-nil, answers cacheable GET/HEAD requests without ever
// invoking the hibernation activator or the backend — the mitigation
// this whole feature exists for. nil disables caching entirely.
@@ -158,6 +166,14 @@ func (e *Edge) EnableCache(cache *Cache) {
e.cache = cache
}
+// EnableTunnels wires the reverse-channel hub in: the edge answers hub.Host()
+// itself (registration) and routes with Tunnel set proxy over their named
+// channel. Call before Reload, same contract as EnableStatic — without a
+// Reload afterwards, tunnel routes stay skipped from the last table build.
+func (e *Edge) EnableTunnels(hub *TunnelHub) {
+ e.tunnels = hub
+}
+
// SetStaticRecheck tunes how long a static route trusts a resolved channel
// pointer before re-reading it (default 60s; 0 = every request). Call after
// EnableStatic and before Reload; handlers built on the next reload use it.
@@ -212,6 +228,9 @@ func (e *Edge) Reload() error {
continue
}
var proxy http.Handler
+ // viaTunnel: this host's bytes leave the box over a reverse channel,
+ // which is what disqualifies it from hibernation below.
+ viaTunnel := false
switch {
// Both reverse proxies get the Carlos-Purge response hook
// (purge_header.go): the app declares its own cache invalidations on
@@ -231,6 +250,30 @@ func (e *Edge) Reload() error {
// Static and status routes get no hook: their responses are the
// edge's own, built from bucket bytes by static.go/status.go, with no
// upstream in a position to declare anything.
+ //
+ // The tunnel case comes FIRST, ahead of socket/addr, because a route
+ // can carry both: `carlos route -tunnel` is a narrow UPDATE that does
+ // not clear socket/addr, so an operator moving a local app onto a
+ // remote host leaves the old column set — and the binding they just
+ // typed is the one they meant. It is guarded on e.tunnels rather than
+ // skipping unconditionally so that an edge with no hub configured
+ // keeps serving such a route from its old socket/addr (degrade to the
+ // previous backend, never take the host off the air); a tunnel-only
+ // route on such an edge matches nothing and falls to
+ // `default: continue`, un-served and un-certed, until the hub lands.
+ //
+ // That degrade path only covers a HUB-LESS edge (e.tunnels == nil).
+ // A hub-configured edge whose client set doesn't (yet) include this
+ // route's channel name still takes this branch — e.tunnels != nil is
+ // all this case checks — and the built proxy 502s every request, even
+ // though t.Addr may hold a perfectly working backend. Deliberate: once
+ // a hub exists, a `-tunnel` binding is the newest operator intent and
+ // wins outright rather than silently falling back to a stale Addr.
+ case t.Tunnel != "" && e.tunnels != nil:
+ p := tunnelProxy(t.Tunnel, e.tunnels)
+ p.ModifyResponse = e.purgeModifier(t)
+ proxy = p
+ viaTunnel = true
case t.Socket != "":
p := unixProxy(t.Socket)
p.ModifyResponse = e.purgeModifier(t)
@@ -252,7 +295,15 @@ func (e *Edge) Reload() error {
// flagged, the edge therefore behaves exactly as it does without one —
// so the binary + env can ship fleet-wide first, and the flag flips
// one instance at a time.
- hib := e.act != nil && t.Kind == "instance" && t.Hibernate
+ //
+ // A tunnel-served host is never activator-managed, however it is
+ // flagged: the process lives on the far side of the channel and this
+ // edge cannot start it, so waking would mean launching a SECOND,
+ // local copy of the app beside the remote one. Leaving such a host out
+ // of `managed` is also what retires the local child a repointed route
+ // left running (RetireExcept, below) — the app moved to the remote
+ // host, so the process it left behind here should go.
+ hib := e.act != nil && t.Kind == "instance" && t.Hibernate && !viaTunnel
if hib {
managed[t.Host] = true
}
@@ -280,6 +331,13 @@ func (e *Edge) current() map[string]upstream {
// Allowed reports whether host is configured — used as the autocert
// HostPolicy, so a certificate is only ever requested for a host we serve.
func (e *Edge) Allowed(host string) bool {
+ // The reserved tunnel hostname is not a registry route, but remote hosts
+ // dial it over TLS on this same 443 listener — so it must be Allowed, and
+ // autocert will mint for it. That is intended: it is a hostname the
+ // operator configured on this box, not a stranger-supplied SNI.
+ if e.tunnels != nil && host == e.tunnels.Host() {
+ return true
+ }
_, ok := e.current()[host]
return ok
}
@@ -289,6 +347,20 @@ func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
+ // The reserved tunnel hostname is answered by the hub itself — before the
+ // route lookup, so registration cannot be intercepted by a route, and
+ // after the host:port split, so a dialer sending "tunnel.host:443" still
+ // lands here. It wins over a registry route of the same name: the operator
+ // owns both names, and a shadowed route is visible (its host stops
+ // answering) where a shadowed hub would be a locked-out fleet.
+ //
+ // Being ahead of the cache path is load-bearing too, not just tidy: the
+ // hub ends its handshake in a hijack, so it must be handed the REAL
+ // ResponseWriter, never a recordingResponseWriter wrapping it.
+ if e.tunnels != nil && host == e.tunnels.Host() {
+ e.tunnels.ServeHTTP(w, r)
+ return
+ }
up, ok := e.current()[host]
if !ok {
e.unknownHost(w, r)
@@ -802,6 +874,27 @@ func tcpProxy(addr string) *httputil.ReverseProxy {
}
}
+// tunnelProxy reverse-proxies over a named reverse channel (TunnelHub).
+// Same Host preservation and purge-header discipline as tcpProxy — a
+// tunnel-backed host is exactly as untrusted as an addr-backed one. The
+// scheme is https because the h2 ClientConn requires it; no dial happens
+// (the channel already exists), so the value is otherwise inert. No channel
+// is resolved here either: hub.transportFor looks it up per request, so a
+// proxy built before a reconnect uses the fresh channel afterwards, and a
+// disconnected one fails fast into the stdlib proxy's 502 rather than hanging.
+func tunnelProxy(name string, hub *TunnelHub) *httputil.ReverseProxy {
+ return &httputil.ReverseProxy{
+ Rewrite: func(pr *httputil.ProxyRequest) {
+ pr.SetXForwarded()
+ pr.Out.URL.Scheme = "https"
+ pr.Out.URL.Host = pr.In.Host // h2 :authority = the real hostname
+ pr.Out.Host = pr.In.Host
+ scrubInboundPurgeHeader(pr)
+ },
+ Transport: hub.transportFor(name),
+ }
+}
+
// dialUnixWithRetry masks the brief gap during an instance restart: while the
// old process has exited and the new one hasn't rebound the socket, dials fail
// with ENOENT/ECONNREFUSED. Retry those for a short window so no in-flight
@@ -843,6 +936,17 @@ func Run(args []string) {
// process. act may be nil: pure proxy, exactly Run's behavior. cache may
// also be nil, disabling response caching entirely.
func RunWithActivator(args []string, act Activator, cache *Cache) {
+ RunWithActivatorReady(args, act, cache, nil)
+}
+
+// RunWithActivatorReady is RunWithActivator plus a ready hook: ready (when
+// non-nil) is called with the fully configured Edge — every EnableXxx done,
+// tunnel hub armed if configured — immediately before the process blocks on
+// its listeners. It is the seam an in-process caller uses to reach the live
+// Edge (e.g. to read tunnel state for status reporting) without racing the
+// configuration it needs. It runs on the calling goroutine, so a hook that
+// blocks holds the front door shut: hand off to a goroutine and return.
+func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready func(*Edge)) {
dataDir := envOr("CARLOS_DATA_DIR", "/data")
fs := flag.NewFlagSet("edge", flag.ExitOnError)
registryPath := fs.String("registry", filepath.Join(dataDir, "registry.db"), "registry SQLite path (default $CARLOS_DATA_DIR/registry.db)")
@@ -873,6 +977,12 @@ func RunWithActivator(args []string, act Activator, cache *Cache) {
log.Fatalf("edge: static serving: %v", err)
}
+ // Reverse channels light up when the environment names a reserved
+ // hostname and at least one client; otherwise tunnel routes stay skipped.
+ if err := enableTunnelsFromEnv(e); err != nil {
+ log.Fatalf("edge: tunnels: %v", err)
+ }
+
go e.Watch(make(chan struct{})) // process-lifetime; never stopped
// Reload routes on SIGHUP too (an operator's explicit "now").
@@ -891,6 +1001,12 @@ func RunWithActivator(args []string, act Activator, cache *Cache) {
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
+ // The Edge is fully configured here — every EnableXxx applied and the
+ // route table rebuilt behind them — and nothing is listening yet.
+ if ready != nil {
+ ready(e)
+ }
+
if *dev {
// ReadHeaderTimeout guards against Slowloris-style slow-header
// connection exhaustion. No ReadTimeout/WriteTimeout — those would
@@ -1014,6 +1130,47 @@ func enableStaticFromEnv(e *Edge, dataDir string) error {
return e.Reload()
}
+// enableTunnelsFromEnv arms the reverse-channel hub when the environment
+// names a reserved hostname (CARLOS_TUNNEL_HOST) and its authorized clients
+// (CARLOS_TUNNEL_CLIENTS, "name=token[,name=token]"). No CARLOS_TUNNEL_HOST is
+// not an error: the box simply serves no tunnels, and routes with Tunnel set
+// keep whatever local backend they have (Reload's tunnel case is guarded on
+// the hub). A host WITH no usable clients IS an error — it would open a
+// registration endpoint no one can ever authenticate against, which reads
+// like a working tunnel edge right up until a host tries to dial it.
+//
+// Channel names are validated here, at the only place a name enters the
+// process from config: a name the CLI would refuse (registry.ValidTunnelName,
+// what `carlos route -tunnel` enforces) can never match a route's Tunnel
+// column, so such a client would authenticate, connect, and silently carry no
+// traffic at all. Fail the boot instead, naming the entry.
+func enableTunnelsFromEnv(e *Edge) error {
+ host := os.Getenv("CARLOS_TUNNEL_HOST")
+ if host == "" {
+ return nil
+ }
+ clients, err := parseTunnelClients(os.Getenv("CARLOS_TUNNEL_CLIENTS"))
+ if err != nil {
+ return err
+ }
+ if len(clients) == 0 {
+ return fmt.Errorf("CARLOS_TUNNEL_HOST=%s set but CARLOS_TUNNEL_CLIENTS is empty", host)
+ }
+ for _, name := range slices.Sorted(maps.Keys(clients)) {
+ if !registry.ValidTunnelName(name) {
+ return fmt.Errorf("CARLOS_TUNNEL_CLIENTS: %q is not a valid tunnel name — want [a-z0-9-]{1,32}, and it must match a route's -tunnel to carry anything", name)
+ }
+ }
+ e.EnableTunnels(NewTunnelHub(host, clients))
+ // Rebuild the route table so tunnel-backed routes that were skipped (or
+ // left on their old socket/addr) by the startup reload pick up the hub.
+ if err := e.Reload(); err != nil {
+ return fmt.Errorf("reload after EnableTunnels: %w", err)
+ }
+ log.Printf("edge: tunnel hub armed on %s (%d client(s))", host, len(clients))
+ return nil
+}
+
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
diff --git a/internal/edge/tunnel.go b/internal/edge/tunnel.go
new file mode 100644
index 000000000..afd6459d0
--- /dev/null
+++ b/internal/edge/tunnel.go
@@ -0,0 +1,224 @@
+// Package-internal reverse-channel hub: remote hosts dial OUT to the edge's
+// existing 443 listener on a reserved hostname, authenticate with a bearer
+// token bound to their channel name, and the connection inverts — the edge
+// becomes an HTTP/2 client over the hijacked conn, the host an HTTP/2 server.
+// Nothing ever connects TO a remote host (issue #53's locked-down property);
+// this file is the edge half of that bargain. Transport judgment call
+// (spec 2026-08-04): reverse h2 rather than a websocket port of titogo's
+// tunnel — x/net/http2 already supplies mux, flow control, and pings.
+package edge
+
+import (
+ "context"
+ "crypto/subtle"
+ "fmt"
+ "log"
+ "net"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/net/http/httpguts"
+ "golang.org/x/net/http2"
+)
+
+const (
+ tunnelRegisterPath = "/register"
+ // tunnelUpgradeProto is the wire contract the host dialer codes against:
+ //
+ // GET /register
+ // Authorization: Bearer <token>
+ // Carlos-Tunnel-Name: <name>
+ // Connection: Upgrade
+ // Upgrade: carlos-tunnel/1
+ //
+ // The dialer MUST also pin NextProtos to {"http/1.1"} on its TLS config.
+ // The edge's 443 listener advertises h2, and a dialer that lets ALPN pick
+ // it authenticates fine and then dies on "hijack unsupported": HTTP/2
+ // response writers are not http.Hijacker, and hijacking is how this
+ // handshake gets the raw conn. The h2 session begins only AFTER the 101,
+ // on that raw conn, with the roles inverted.
+ tunnelUpgradeProto = "carlos-tunnel/1"
+ // tunnelPingEvery / tunnelPingTimeout bound how long a half-dead channel
+ // can look connected: a failed or timed-out ping tears it down, the host
+ // notices its conn die and re-dials (seconds — the bound Task 10 asserts).
+ tunnelPingEvery = 15 * time.Second
+ tunnelPingTimeout = 10 * time.Second
+)
+
+type tunnelConn struct {
+ cc *http2.ClientConn
+ raw net.Conn
+}
+
+type TunnelHub struct {
+ host string // reserved hostname this hub answers on
+ clients map[string]string // channel name -> bearer token
+ mu sync.Mutex
+ conns map[string]*tunnelConn
+}
+
+func NewTunnelHub(host string, clients map[string]string) *TunnelHub {
+ return &TunnelHub{host: host, clients: clients, conns: make(map[string]*tunnelConn)}
+}
+
+func (h *TunnelHub) Host() string { return h.host }
+
+// parseTunnelClients parses CARLOS_TUNNEL_CLIENTS ("name=token[,name=token]").
+func parseTunnelClients(s string) (map[string]string, error) {
+ out := make(map[string]string)
+ for _, pair := range strings.Split(s, ",") {
+ pair = strings.TrimSpace(pair)
+ if pair == "" {
+ continue
+ }
+ name, token, ok := strings.Cut(pair, "=")
+ if !ok || name == "" || token == "" {
+ return nil, fmt.Errorf("tunnel clients: malformed entry %q (want name=token)", pair)
+ }
+ out[name] = token
+ }
+ return out, nil
+}
+
+// ServeHTTP handles ONLY the registration handshake; the Edge routes requests
+// here when r.Host equals the reserved hostname, so ordinary app traffic can
+// never reach it and this handler never sees ordinary traffic.
+func (h *TunnelHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != tunnelRegisterPath {
+ http.NotFound(w, r)
+ return
+ }
+ name := r.Header.Get("Carlos-Tunnel-Name")
+ // CutPrefix's ok is load-bearing: TrimPrefix would return a scheme-less
+ // header (e.g. "Authorization: sekrit") unchanged, letting a credential
+ // that never claimed to be a bearer token compare equal to a configured
+ // one. No "Bearer " prefix is an outright 401, not a token of "".
+ token, hasBearer := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+ want, ok := h.clients[name]
+ // Constant-time compare; the name lookup happening first is fine — the
+ // name is not the secret, the token is. A token is bound to exactly one
+ // channel name, so a leaked token cannot impersonate a different host.
+ // An empty configured token never authenticates anything: parseTunnelClients
+ // rejects one, and a hub built some other way must not turn a blank config
+ // value into a channel that answers to "Authorization: Bearer ".
+ if name == "" || !ok || want == "" || !hasBearer || subtle.ConstantTimeCompare([]byte(token), []byte(want)) != 1 {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ // Wire-contract gate, deliberately AFTER auth: an unauthenticated caller
+ // gets 401 whatever it sends, so probing cannot discover which protocols
+ // this endpoint speaks. Checking it at all is what keeps a version skew
+ // diagnosable — without it we would answer "101 carlos-tunnel/1" to a
+ // client that never asked, and a future carlos-tunnel/2 host would see a
+ // mystery h2-preface failure instead of a refusal naming the version.
+ // Connection is matched token-wise and Upgrade case-insensitively, both
+ // per RFC 7230; the version suffix is what must match exactly.
+ if !httpguts.HeaderValuesContainsToken(r.Header["Connection"], "Upgrade") ||
+ !strings.EqualFold(strings.TrimSpace(r.Header.Get("Upgrade")), tunnelUpgradeProto) {
+ http.Error(w, "expected Connection: Upgrade with Upgrade: "+tunnelUpgradeProto, http.StatusBadRequest)
+ return
+ }
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ http.Error(w, "hijack unsupported", http.StatusInternalServerError)
+ return
+ }
+ conn, bufrw, err := hj.Hijack()
+ if err != nil {
+ log.Printf("edge: tunnel %s: hijack: %v", name, err)
+ return
+ }
+ // The h2 session runs on the RAW conn, so anything the host pipelined
+ // ahead of our 101 is sitting unreadable in bufrw and would corrupt the
+ // handshake. Fail loudly here rather than hang on a broken preface: the
+ // host-side dialer's contract is to write nothing until it reads the 101.
+ if n := bufrw.Reader.Buffered(); n > 0 {
+ log.Printf("edge: tunnel %s: %d bytes sent before 101 — dropping", name, n)
+ conn.Close()
+ return
+ }
+ // Manual 101 on the hijacked conn (the ResponseWriter is spent once hijacked).
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: %s\r\nConnection: Upgrade\r\n\r\n", tunnelUpgradeProto)
+ if err := bufrw.Flush(); err != nil {
+ conn.Close()
+ return
+ }
+ tr := &http2.Transport{}
+ cc, err := tr.NewClientConn(conn)
+ if err != nil {
+ log.Printf("edge: tunnel %s: h2 client conn: %v", name, err)
+ conn.Close()
+ return
+ }
+ h.mu.Lock()
+ if old := h.conns[name]; old != nil {
+ old.raw.Close() // a re-register replaces the previous channel
+ }
+ h.conns[name] = &tunnelConn{cc: cc, raw: conn}
+ h.mu.Unlock()
+ log.Printf("edge: tunnel %s: connected from %s", name, conn.RemoteAddr())
+ go h.pingLoop(name, cc, conn)
+}
+
+// pingLoop is the edge-side liveness check: a dead ping closes the raw conn
+// and unregisters the channel, so requests fail fast to 502 instead of
+// hanging on a black-holed TCP session (the venue-Wi-Fi failure mode).
+func (h *TunnelHub) pingLoop(name string, cc *http2.ClientConn, raw net.Conn) {
+ for {
+ time.Sleep(tunnelPingEvery)
+ ctx, cancel := context.WithTimeout(context.Background(), tunnelPingTimeout)
+ err := cc.Ping(ctx)
+ cancel()
+ if err != nil {
+ log.Printf("edge: tunnel %s: ping failed (%v) — dropping channel", name, err)
+ raw.Close()
+ // Only unregister if WE are still the live channel: a re-register
+ // may already have replaced this entry (and closed our conn, which
+ // is what failed the ping), and deleting then would blank out a
+ // perfectly healthy successor.
+ h.mu.Lock()
+ if cur := h.conns[name]; cur != nil && cur.raw == raw {
+ delete(h.conns, name)
+ }
+ h.mu.Unlock()
+ return
+ }
+ }
+}
+
+func (h *TunnelHub) conn(name string) *http2.ClientConn {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if c := h.conns[name]; c != nil {
+ return c.cc
+ }
+ return nil
+}
+
+// State reports "connected" or "". The heartbeat's tunnel-state field comes
+// from the host-side dial loop, not from here; today this is test-consumed
+// only — no production caller reads it.
+func (h *TunnelHub) State(name string) string {
+ if h.conn(name) != nil {
+ return "connected"
+ }
+ return ""
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// transportFor resolves the channel PER REQUEST, so a proxy built before a
+// reconnect uses the fresh channel afterwards with no reload needed.
+func (h *TunnelHub) transportFor(name string) http.RoundTripper {
+ return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ cc := h.conn(name)
+ if cc == nil {
+ return nil, fmt.Errorf("tunnel %q not connected", name)
+ }
+ return cc.RoundTrip(req)
+ })
+}
diff --git a/internal/edge/tunnel_e2e_test.go b/internal/edge/tunnel_e2e_test.go
new file mode 100644
index 000000000..5aa4ef5a4
--- /dev/null
+++ b/internal/edge/tunnel_e2e_test.go
@@ -0,0 +1,218 @@
+package edge
+
+// The acceptance test issue #53 names: edge + fake remote host over a REAL
+// reverse channel on loopback. Round-trip, header survival, kill-mid-flight,
+// re-dial, and the reconnect bound (asserted, not narrated).
+//
+// The fake remote host is not a hand-rolled dialer: it runs
+// internal/tunnelclient.Loop, the exact loop `carlos agent` runs on a Pi
+// (internal/host/tunnel_client.go is a thin env-reading wrapper around it).
+// That is the whole point of the extraction — a reconnect bound measured
+// against a test double would prove nothing about the fleet.
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/tunnelclient"
+)
+
+// e2eReconnectBound is the promise: a channel that dies is back and serving
+// within this. The client sleeps one jittered backoff step (250-375ms on a
+// first loss, 500-750ms once it has doubled) before re-dialing, so the real
+// figure is well under a second — the bound is deliberately loose enough that
+// a loaded CI box cannot flake it, and tight enough that a regression to the
+// 2s cap compounding, or to no reconnect at all, fails here.
+const e2eReconnectBound = 5 * time.Second
+
+func TestTunnelEndToEnd(t *testing.T) {
+ reg := tunnelRoute(t, registry.Route{
+ Host: "demo.example", Kind: "instance", Status: "ready", Tunnel: "pi-box",
+ })
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The edge listens on loopback; both app traffic and the tunnel
+ // registration ride this ONE listener — the no-new-port property.
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+
+ // The hub's reserved hostname has to be the name the host actually dials,
+ // because the dialer sends the URL's host as its Host header and the edge
+ // routes registration on that. On a loopback edge that name is the
+ // listener's own address; in production it is the operator's
+ // CARLOS_TUNNEL_HOST, resolved by DNS to the same 443.
+ u, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hub := NewTunnelHub(u.Hostname(), map[string]string{"pi-box": "sekrit"})
+ e.EnableTunnels(hub)
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+
+ // Fake remote host: its "local edge" answers with a header + body, and
+ // echoes Host and X-Forwarded-For so the test can prove header discipline
+ // held all the way across the channel. /slow parks until released — that
+ // is what makes the kill below genuinely mid-flight rather than a sleep
+ // and a hope.
+ inflight := make(chan struct{}, 1)
+ release := make(chan struct{})
+ defer close(release)
+ appHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/slow" {
+ select {
+ case inflight <- struct{}{}:
+ default:
+ }
+ // Either the test lets it finish, or the stream dies under it —
+ // which is what a real handler sees when the channel drops, and
+ // what keeps this goroutine from outliving the connection.
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ }
+ }
+ w.Header().Set("X-App-Header", "survived")
+ w.Header().Set("Carlos-Purge", "/should-be-stripped") // edge must strip
+ fmt.Fprintf(w, "host=%s xff=%s", r.Host, r.Header.Get("X-Forwarded-For"))
+ })
+ stop := startFakeRemoteHost(t, srv.URL, "pi-box", "sekrit", appHandler)
+ defer stop()
+
+ waitForTunnelState(t, hub, "pi-box", "connected", e2eReconnectBound)
+
+ do := func(path string) (*http.Response, string) {
+ t.Helper()
+ req, err := http.NewRequest("GET", srv.URL+path, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Host = "demo.example"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request through tunnel: %v", err)
+ }
+ body, _ := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ return resp, string(body)
+ }
+
+ resp, body := do("/")
+ if resp.StatusCode != 200 || resp.Header.Get("X-App-Header") != "survived" {
+ t.Fatalf("round-trip: %d %v", resp.StatusCode, resp.Header)
+ }
+ if resp.Header.Get("Carlos-Purge") != "" {
+ t.Fatal("Carlos-Purge leaked to the client — purge discipline broken over tunnel")
+ }
+ if !strings.Contains(body, "host=demo.example") || !strings.Contains(body, "xff=127.") {
+ t.Fatalf("host/XFF discipline broken: %q", body)
+ }
+
+ // Kill the channel out from under the host with a request PARKED inside
+ // the remote handler, then assert three things: the in-flight request
+ // fails fast rather than hanging, the host re-dials, and the next request
+ // succeeds — all inside the bound.
+ slowDone := make(chan int, 1)
+ go func() {
+ req, err := http.NewRequest("GET", srv.URL+"/slow", nil)
+ if err != nil {
+ slowDone <- -1
+ return
+ }
+ req.Host = "demo.example"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ slowDone <- 0 // the client saw the failure directly
+ return
+ }
+ resp.Body.Close()
+ slowDone <- resp.StatusCode
+ }()
+ select {
+ case <-inflight:
+ case <-time.After(e2eReconnectBound):
+ t.Fatal("/slow never reached the remote host's handler")
+ }
+
+ // Unregister BEFORE closing, so the ordering is deterministic rather than
+ // a race with the host's re-registration: once the entry is gone, State is
+ // "" until the NEW channel lands, and a re-register can never be clobbered
+ // by this teardown.
+ hub.mu.Lock()
+ dead := hub.conns["pi-box"]
+ delete(hub.conns, "pi-box")
+ hub.mu.Unlock()
+ if dead == nil {
+ t.Fatal("no live channel to kill — the hub lost it before the test did")
+ }
+ start := time.Now()
+ dead.raw.Close()
+
+ select {
+ case code := <-slowDone:
+ if code < 0 {
+ t.Fatal("could not build the in-flight request")
+ }
+ if code == http.StatusOK {
+ t.Fatal("the in-flight request succeeded after its channel died — the edge served bytes over a dead channel")
+ }
+ case <-time.After(e2eReconnectBound):
+ t.Fatal("in-flight request neither completed nor failed after its channel died — it hung")
+ }
+
+ waitForTunnelState(t, hub, "pi-box", "connected", e2eReconnectBound)
+ if resp, _ := do("/"); resp.StatusCode != 200 {
+ t.Fatalf("post-reconnect request: %d", resp.StatusCode)
+ }
+ if d := time.Since(start); d > e2eReconnectBound {
+ t.Fatalf("reconnect took %s, bound is %s", d, e2eReconnectBound)
+ } else {
+ t.Logf("reconnected and served in %s (bound %s)", d, e2eReconnectBound)
+ }
+}
+
+// startFakeRemoteHost runs the production host-side loop against edgeURL and
+// returns a stop func that cancels it and WAITS for it to exit — a loop still
+// dialing after its test returned would log into the next test's output and
+// hold a goroutine for the rest of the run.
+func startFakeRemoteHost(t *testing.T, edgeURL, name, token string, handler http.Handler) func() {
+ t.Helper()
+ ctx, cancel := context.WithCancel(context.Background())
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ tunnelclient.Loop(ctx, tunnelclient.Config{
+ URL: edgeURL, Name: name, Token: token, Handler: handler,
+ Logf: func(format string, args ...any) { t.Logf("fake host: "+format, args...) },
+ })
+ }()
+ return func() {
+ cancel()
+ wg.Wait() // no t.Logf after the test returns
+ }
+}
+
+func waitForTunnelState(t *testing.T, hub *TunnelHub, name, want string, within time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(within)
+ for time.Now().Before(deadline) {
+ if hub.State(name) == want {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatalf("tunnel %s never reached state %q within %s", name, want, within)
+}
diff --git a/internal/edge/tunnel_test.go b/internal/edge/tunnel_test.go
new file mode 100644
index 000000000..cc02d65c0
--- /dev/null
+++ b/internal/edge/tunnel_test.go
@@ -0,0 +1,316 @@
+package edge
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// tunnelReq builds a request that is well-formed EXCEPT for the credentials
+// under test: valid upgrade headers, so the auth cases below can only fail on
+// auth. Pass an empty token to send no Authorization header at all.
+func tunnelReq(name, token string) *http.Request {
+ req := httptest.NewRequest("GET", "/register", nil)
+ req.Header.Set("Connection", "Upgrade")
+ req.Header.Set("Upgrade", tunnelUpgradeProto)
+ req.Header.Set("Carlos-Tunnel-Name", name)
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ return req
+}
+
+func TestTunnelRegisterRejectsBadToken(t *testing.T) {
+ // TWO clients on purpose: with only one configured, no test here can tell
+ // "every token is bound to its own channel" apart from "any known token
+ // admits any known channel" — an implementation looping over h.clients
+ // would pass a single-client table unchanged.
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit", "hq": "other"})
+ for _, tc := range []struct{ desc, name, token string }{
+ {"no auth", "", ""},
+ {"wrong token", "pi-box", "nope"},
+ {"unknown name", "nosuch", "sekrit"},
+ {"empty name", "", "sekrit"},
+ // The binding property itself: a valid token may only ever claim ITS
+ // OWN channel name, in either direction.
+ {"cross-name", "hq", "sekrit"},
+ {"cross-name reversed", "pi-box", "other"},
+ } {
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, tunnelReq(tc.name, tc.token))
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("%s (name=%q token=%q): got %d, want 401", tc.desc, tc.name, tc.token, rec.Code)
+ }
+ }
+}
+
+// Scheme-less credential: "Authorization: sekrit" must not authenticate even
+// though "sekrit" is the correct configured token — TrimPrefix would have let
+// this through unchanged (nothing to strip), which is the bug this pins.
+func TestTunnelRegisterRejectsSchemelessAuthorization(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ req := tunnelReq("pi-box", "")
+ req.Header.Set("Authorization", "sekrit") // no "Bearer " prefix
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Fatalf("scheme-less Authorization: got %d, want 401", rec.Code)
+ }
+}
+
+func TestTunnelRegisterAuthPrecedesUpgradeCheck(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ req := httptest.NewRequest("GET", "/register", nil) // no upgrade headers at all
+ req.Header.Set("Authorization", "Bearer nope")
+ req.Header.Set("Carlos-Tunnel-Name", "pi-box")
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Fatalf("got %d, want 401 — an unauthenticated caller must not learn which upgrade protocols we speak", rec.Code)
+ }
+}
+
+func TestTunnelRegisterRejectsWrongUpgrade(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ for _, tc := range []struct{ desc, conn, upgrade string }{
+ {"no upgrade headers", "", ""},
+ {"missing Upgrade", "Upgrade", ""},
+ {"missing Connection", "", tunnelUpgradeProto},
+ {"wrong version", "Upgrade", "carlos-tunnel/2"},
+ {"other protocol", "Upgrade", "websocket"},
+ } {
+ req := httptest.NewRequest("GET", "/register", nil)
+ req.Header.Set("Authorization", "Bearer sekrit")
+ req.Header.Set("Carlos-Tunnel-Name", "pi-box")
+ if tc.conn != "" {
+ req.Header.Set("Connection", tc.conn)
+ }
+ if tc.upgrade != "" {
+ req.Header.Set("Upgrade", tc.upgrade)
+ }
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("%s: got %d, want 400", tc.desc, rec.Code)
+ continue
+ }
+ // A future carlos-tunnel/2 host must get a diagnosable refusal.
+ if !strings.Contains(rec.Body.String(), tunnelUpgradeProto) {
+ t.Errorf("%s: body %q should name the expected protocol", tc.desc, rec.Body.String())
+ }
+ }
+}
+
+func TestTunnelRegisterAcceptsTokenWiseConnectionHeader(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ req := httptest.NewRequest("GET", "/register", nil)
+ req.Header.Set("Authorization", "Bearer sekrit")
+ req.Header.Set("Carlos-Tunnel-Name", "pi-box")
+ req.Header.Set("Connection", "keep-alive, Upgrade") // token-wise, per RFC 7230
+ req.Header.Set("Upgrade", "Carlos-Tunnel/1") // protocol names are case-insensitive
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ // This request passes auth AND the upgrade gate, then dies at the hijack,
+ // which httptest.NewRecorder cannot do — so 500 (not 400) is how we know
+ // the gate let it through. The real 101 is Task 10's e2e.
+ if rec.Code != http.StatusInternalServerError {
+ t.Fatalf("got %d, want 500 (past the upgrade gate, stopped at the hijack)", rec.Code)
+ }
+}
+
+func TestTunnelRegisterRejectsUnknownPath(t *testing.T) {
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ req := httptest.NewRequest("GET", "/anything-else", nil)
+ rec := httptest.NewRecorder()
+ hub.ServeHTTP(rec, req)
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("got %d, want 404", rec.Code)
+ }
+}
+
+func TestParseTunnelClients(t *testing.T) {
+ got, err := parseTunnelClients("pi-box=sekrit,hq=other")
+ if err != nil || got["pi-box"] != "sekrit" || got["hq"] != "other" {
+ t.Fatalf("parse: %v %v", got, err)
+ }
+ if _, err := parseTunnelClients("malformed"); err == nil {
+ t.Fatal("want error on entry without '='")
+ }
+}
+
+// tunnelRoute upserts one route into a fresh registry (openReg, the package's
+// one registry harness — edge_test.go) and returns the registry.
+func tunnelRoute(t *testing.T, rt registry.Route) *registry.Registry {
+ t.Helper()
+ reg := openReg(t)
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ return reg
+}
+
+func TestReloadBuildsTunnelBackedRoute(t *testing.T) {
+ reg := tunnelRoute(t, registry.Route{
+ Host: "demo.example", Kind: "instance", Status: "ready", Tunnel: "pi-box",
+ })
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Before EnableTunnels the route has no backend at all: it must not be
+ // served, and must not burn an ACME issuance either.
+ if e.Allowed("demo.example") {
+ t.Fatal("tunnel-backed route served with no hub configured")
+ }
+ hub := NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"})
+ e.EnableTunnels(hub)
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ if !e.Allowed("demo.example") {
+ t.Fatal("tunnel-backed route not served after EnableTunnels+Reload")
+ }
+ if !e.Allowed("tunnel.test") {
+ t.Fatal("reserved tunnel hostname must be Allowed (autocert HostPolicy)")
+ }
+ // Channel down -> 502, not a hang.
+ req := httptest.NewRequest("GET", "http://demo.example/", nil)
+ req.Host = "demo.example"
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+ if rec.Code != http.StatusBadGateway {
+ t.Fatalf("disconnected tunnel: got %d, want 502", rec.Code)
+ }
+}
+
+// The reserved hostname is answered by the hub itself, ahead of the route
+// table — so a registration attempt reaches the handshake (401 here, no
+// credentials) instead of the unknown-host 404.
+func TestEdgeInterceptsReservedTunnelHost(t *testing.T) {
+ e, err := New(openReg(t), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ e.EnableTunnels(NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"}))
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ req := tunnelReq("pi-box", "nope")
+ req.Host = "tunnel.test:443" // host:port must still match the reserved name
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Fatalf("got %d, want 401 — the reserved host must reach the hub, not the route table", rec.Code)
+ }
+}
+
+// `carlos route -tunnel` is a narrow UPDATE: it does not clear the route's
+// socket/addr. So a route can legitimately carry both, and which one wins is
+// a real decision, not a moot one:
+// - hub configured -> the tunnel wins (the newest, most explicit binding).
+// - no hub on this box -> the old addr keeps serving, rather than the host
+// dropping off the air on an edge that cannot honor the tunnel.
+func TestTunnelBindingWinsOverAddrOnlyWhenHubConfigured(t *testing.T) {
+ backend, _ := hostEchoBackend(t, "local addr backend")
+ reg := tunnelRoute(t, registry.Route{
+ Host: "both.example", Kind: "service", Status: "ready",
+ Addr: backend.Listener.Addr().String(), Tunnel: "pi-box",
+ })
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+ code, body := getHost(t, srv.URL, "both.example")
+ if code != http.StatusOK || body != "local addr backend" {
+ t.Fatalf("no hub configured: got %d %q, want the addr backend to keep serving", code, body)
+ }
+ e.EnableTunnels(NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"}))
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ if code, _ := getHost(t, srv.URL, "both.example"); code != http.StatusBadGateway {
+ t.Fatalf("hub configured: got %d, want 502 from the (disconnected) tunnel", code)
+ }
+}
+
+// The env wiring is the only place a channel name enters the process from
+// config, and a name `carlos route -tunnel` would refuse can never match a
+// route — so it must stop the boot, not connect and carry nothing.
+func TestEnableTunnelsFromEnv(t *testing.T) {
+ newEdge := func() *Edge {
+ e, err := New(openReg(t), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return e
+ }
+ t.Run("unset is not an error", func(t *testing.T) {
+ e := newEdge()
+ if err := enableTunnelsFromEnv(e); err != nil {
+ t.Fatalf("no CARLOS_TUNNEL_HOST: %v", err)
+ }
+ if e.tunnels != nil {
+ t.Fatal("hub armed with no CARLOS_TUNNEL_HOST")
+ }
+ })
+ t.Run("host with no clients", func(t *testing.T) {
+ t.Setenv("CARLOS_TUNNEL_HOST", "tunnel.test")
+ t.Setenv("CARLOS_TUNNEL_CLIENTS", "")
+ if err := enableTunnelsFromEnv(newEdge()); err == nil {
+ t.Fatal("want an error: a registration endpoint nobody can authenticate against")
+ }
+ })
+ t.Run("invalid channel name", func(t *testing.T) {
+ t.Setenv("CARLOS_TUNNEL_HOST", "tunnel.test")
+ t.Setenv("CARLOS_TUNNEL_CLIENTS", "pi-box=sekrit,Pi_Box=other")
+ err := enableTunnelsFromEnv(newEdge())
+ if err == nil || !strings.Contains(err.Error(), "Pi_Box") {
+ t.Fatalf("got %v, want an error naming the bad entry", err)
+ }
+ })
+ t.Run("armed", func(t *testing.T) {
+ t.Setenv("CARLOS_TUNNEL_HOST", "tunnel.test")
+ t.Setenv("CARLOS_TUNNEL_CLIENTS", "pi-box=sekrit")
+ e := newEdge()
+ if err := enableTunnelsFromEnv(e); err != nil {
+ t.Fatal(err)
+ }
+ if e.tunnels == nil || e.tunnels.Host() != "tunnel.test" || !e.Allowed("tunnel.test") {
+ t.Fatalf("hub not armed on the configured host")
+ }
+ })
+}
+
+// A tunnel-served host is never activator-managed: the process lives on the
+// far side of the channel, so a wake here would start a second, local copy.
+// The retire pass must also drop it, killing the child a repointed route left
+// behind.
+func TestTunnelRouteIsNeverActivatorManaged(t *testing.T) {
+ reg := tunnelRoute(t, registry.Route{
+ Host: "moved.example", Kind: "instance", Status: "ready",
+ Tunnel: "pi-box", Hibernate: true,
+ })
+ act := &fakeActivator{}
+ e, err := New(reg, act)
+ if err != nil {
+ t.Fatal(err)
+ }
+ e.EnableTunnels(NewTunnelHub("tunnel.test", map[string]string{"pi-box": "sekrit"}))
+ if err := e.Reload(); err != nil {
+ t.Fatal(err)
+ }
+ if last := act.retired[len(act.retired)-1]; last["moved.example"] {
+ t.Fatalf("tunnel-served host kept in the activator's managed set: %v", last)
+ }
+ req := httptest.NewRequest("GET", "http://moved.example/", nil)
+ req.Host = "moved.example"
+ e.ServeHTTP(httptest.NewRecorder(), req)
+ if act.served != 0 {
+ t.Fatalf("activator woke a local process for a tunnel-served host (%d call(s))", act.served)
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index c8ffe6f57..44b2435a5 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -90,6 +90,7 @@ import (
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/status"
+ "github.com/carlosframework/platform/internal/version"
)
// dataDir resolves the fleet data directory (CARLOS_DATA_DIR, default /data).
@@ -158,7 +159,12 @@ func RunAgent(args []string) {
// The same serving loop as `carlos edge`, with the activator injected.
// Caching applies here too: cached hits still avoid the activator, which
// is the entire point of checking the cache before the hibernation branch.
- edge.RunWithActivator(args, act, cache)
+ // The ready hook fires synchronously, just before the blocking serve, so
+ // tunnelLoop is handed off to its own goroutine rather than called here —
+ // calling it directly would block the front door forever.
+ edge.RunWithActivatorReady(args, act, cache, func(e *edge.Edge) {
+ go tunnelLoop(e)
+ })
// The edge has drained; take the children down too (they're this process's
// children — orphaning them would strand their leases until TTL expiry).
@@ -805,6 +811,16 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) {
// has other env-reading tests; a plain map/func seam avoids that coupling
// entirely.
func resolveStatusBucket(getenv func(string) string) string {
+ return ResolveStatusBucket(getenv)
+}
+
+// ResolveStatusBucket is resolveStatusBucket for callers outside this package
+// — `carlos status -live` reads the boxes an agent wrote, and it must land on
+// the SAME bucket the agent's status loop writes to, including the
+// CARLOS_DEPLOYMENT_BUCKET fallback. Exported as a second name over one
+// implementation rather than renamed at every internal call site, so the
+// defaulting rule documented above stays in exactly one function.
+func ResolveStatusBucket(getenv func(string) string) string {
if b := getenv("CARLOS_STATUS_BUCKET"); b != "" {
return b
}
@@ -852,7 +868,7 @@ func statusLoop(reg *registry.Registry) {
// by accident (or on purpose) and end up with the agent writing status
// to a bucket the edge/console never look at — this line is the one
// place an operator can confirm which bucket actually won.
- log.Printf("agent: status loop armed (5min + jitter), publishing to bucket %s", bucket)
+ log.Printf("agent: status loop armed (%s + jitter), publishing to bucket %s", status.SnapshotPeriod, bucket)
// canaryStore/canaryPub resolve the release deployment config
// countExpiredCanaries needs to answer §5's "how many canaries on this
@@ -869,13 +885,18 @@ func statusLoop(reg *registry.Registry) {
canaryStore, _ := openDeployStore(ctx)
canaryPub, _ := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ // status.SnapshotPeriod IS this cadence — the const is named for the tick
+ // below, and every reader that judges a box stale (the published boxes
+ // table, the console's private host detail, `carlos status -live`) measures
+ // against it, so changing it here must move them together rather than
+ // leaving them judging a heartbeat that arrives on a different beat.
// Jittered on the same shape as adoptLoop's delay: every box on the same
- // 5-minute cadence would all hit the lease and the bucket in the same
- // second otherwise.
+ // cadence would all hit the lease and the bucket in the same second
+ // otherwise.
delay := time.Minute
for {
time.Sleep(delay)
- delay = 5*time.Minute + rand.N(time.Minute)
+ delay = status.SnapshotPeriod + rand.N(time.Minute)
// Routes are listed here (not left to statusTick's own reg.List())
// because the Listening probe needs to know which sockets are
@@ -890,6 +911,7 @@ func statusLoop(reg *registry.Registry) {
continue
}
+ memTotal, memAvail := readMemBytes()
p := status.Probes{
MachineID: readMachineID(),
Region: boxRegion,
@@ -918,6 +940,12 @@ func statusLoop(reg *registry.Registry) {
// skew?) would be worse than the honest null CollectBox already
// knows how to carry — wiring this up is a seam for later, not
// a gap to paper over now.
+ MemTotalBytes: memTotal,
+ MemAvailBytes: memAvail,
+ Load1: readLoad1(),
+ DiskFreeBytes: readDiskFreeBytes(),
+ Release: version.Build,
+ Tunnel: tunnelState(),
}
// path.Join (an S3 key, not a filesystem path — matches
@@ -970,8 +998,10 @@ func statusTick(ctx context.Context, s release.Store, reg *registry.Registry, p
return nil
}
- const fleetPeriod = 5 * time.Minute // matches statusLoop's own tick cadence
- fleet, err := status.BuildFleet(ctx, s, routes, now, fleetPeriod)
+ // status.SnapshotPeriod is this loop's own tick cadence, named once in the
+ // status package so the console's private host table and `carlos status
+ // -live` measure staleness against the same period this publish does.
+ fleet, err := status.BuildFleet(ctx, s, routes, now, status.SnapshotPeriod)
if err != nil {
return err
}
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index 488d52ccb..283fb3d10 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -94,12 +94,51 @@ func systemctlPlan(units []string) [][]string {
func RunBootstrap(args []string) {
fs := flag.NewFlagSet("bootstrap", flag.ExitOnError)
root := fs.String("root", "/", "install root; a staging dir writes files only (no user creation, no systemctl)")
+ offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile")
_ = fs.Parse(args)
+ if *offcloud {
+ if err := bootstrapOffcloud(*root); err != nil {
+ log.Fatalf("bootstrap: %v", err)
+ }
+ return
+ }
if err := bootstrap(*root); err != nil {
log.Fatalf("bootstrap: %v", err)
}
}
+// bootstrapOffcloud is bootstrap for a box with no EC2 instance profile
+// (issue #53 point 4): identical structural work, plus the credential
+// preconditions — a staged 0600 credentials file scoped (at IAM) to the
+// deployment bucket + this host's own region bucket and nothing else, so
+// wrong-region-fails-at-IAM holds off-cloud too. The flag is explicit; no
+// IMDS sniffing (the agent's one IMDS probe is already gated for non-EC2).
+//
+// The credential check runs BEFORE any system mutation: a failed precondition
+// leaves the box untouched, same as a failed bootstrap() would.
+func bootstrapOffcloud(root string) error {
+ cred := filepath.Join(root, "etc", "carlos", "credentials")
+ fi, err := os.Stat(cred)
+ if err != nil {
+ return fmt.Errorf("offcloud: stage the scoped credentials file first: %s (0600, see infra/remote-host.md): %w", cred, err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ return fmt.Errorf("offcloud: %s must be mode 0600, is %o", cred, fi.Mode().Perm())
+ }
+ if err := bootstrap(root); err != nil {
+ return err
+ }
+ // Ownership is a live-box-only step, same as ensureServiceUser/chownToService
+ // in bootstrap() above — a staging root has no service user to chown to.
+ if filepath.Clean(root) == "/" {
+ if err := chownToService(cred); err != nil { // same helper the data dirs use
+ return err
+ }
+ }
+ fmt.Printf("offcloud bootstrap complete.\n ensure /etc/carlos/host.env sets:\n AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials\n then stage /opt/carlos/carlos and start carlos-edge.service (see infra/remote-host.md)\n")
+ return nil
+}
+
// bootstrap provisions (or re-converges) a box under root: units into
// <root>/etc/systemd/system, the standard dirs, the tmpfiles.d entry, and —
// only when root is "/" — the service user, ownership, and the systemctl
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index 3ef098d9b..e810e304d 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -513,6 +513,33 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
}
}
+// TestBootstrapOffcloudRequiresCredentialsFile pins the off-cloud precondition
+// (issue #53 point 4): a box with no EC2 instance profile must have its scoped
+// credentials staged — and staged at 0600 — before bootstrap proceeds. This is
+// the staging-root path (no root privileges available in tests), so the
+// service-user chown below never runs; that only happens on a live root.
+func TestBootstrapOffcloudRequiresCredentialsFile(t *testing.T) {
+ root := t.TempDir()
+ if err := bootstrapOffcloud(root); err == nil {
+ t.Fatal("want error when <root>/etc/carlos/credentials is absent")
+ }
+ credDir := filepath.Join(root, "etc", "carlos")
+ os.MkdirAll(credDir, 0o755)
+ credFile := filepath.Join(credDir, "credentials")
+ os.WriteFile(credFile, []byte("[default]\naws_access_key_id=X\naws_secret_access_key=Y\n"), 0o644)
+ if err := bootstrapOffcloud(root); err == nil {
+ t.Fatal("want error on 0644 credentials (must be 0600)")
+ }
+ os.Chmod(credFile, 0o600)
+ if err := bootstrapOffcloud(root); err != nil {
+ t.Fatalf("offcloud bootstrap with staged 0600 credentials: %v", err)
+ }
+ // The normal bootstrap artifacts must exist too — offcloud is a superset.
+ if _, err := os.Stat(filepath.Join(root, "etc", "systemd", "system", "carlos-edge.service")); err != nil {
+ t.Fatalf("units not installed: %v", err)
+ }
+}
+
// TestAdoptTimerStateReading pins the probe's honesty rules at the seam that
// needs no systemd to test: systemd's own word for the timer's state maps to
// running/not-running, and anything this probe cannot interpret — no systemd
@@ -546,6 +573,26 @@ func TestAdoptTimerStateReading(t *testing.T) {
func boolPtr(v bool) *bool { return &v }
+func TestParseMeminfoBytes(t *testing.T) {
+ total, avail := parseMeminfoBytes(strings.NewReader(
+ "MemTotal: 3961472 kB\nMemFree: 100 kB\nMemAvailable: 1980736 kB\n"))
+ if total == nil || *total != 3961472*1024 || avail == nil || *avail != 1980736*1024 {
+ t.Fatalf("got %v %v", total, avail)
+ }
+ if tt, aa := parseMeminfoBytes(strings.NewReader("garbage")); tt != nil || aa != nil {
+ t.Fatal("unparseable meminfo must yield nil, not zero")
+ }
+}
+
+func TestParseLoad1(t *testing.T) {
+ if l := parseLoad1("0.42 0.36 0.30 1/234 5678\n"); l == nil || *l != 0.42 {
+ t.Fatalf("got %v", l)
+ }
+ if l := parseLoad1(""); l != nil {
+ t.Fatal("empty loadavg must yield nil")
+ }
+}
+
// TestProbedUnitsAreTheOnesBootstrapArms pins the pairing that makes both
// systemd probes meaningful: each one must name a unit bootstrap actually
// installs AND arms. A probe pointed at a unit no box has answers "unknown"
diff --git a/internal/host/probes.go b/internal/host/probes.go
index ba0966d8e..1e3bd5d64 100644
--- a/internal/host/probes.go
+++ b/internal/host/probes.go
@@ -12,6 +12,7 @@ package host
import (
"bufio"
+ "io"
"os"
"os/exec"
"strconv"
@@ -236,3 +237,86 @@ func wrapListeningProbe(unitBacked map[string]bool, probe func(string) *bool) fu
return probe(sock)
}
}
+
+// parseMeminfoBytes reads /proc/meminfo's MemTotal and MemAvailable (both in
+// kB) and returns them converted to bytes. Either field missing or
+// unparsable -> nil for both.
+func parseMeminfoBytes(r io.Reader) (*int64, *int64) {
+ var totalKB, availKB int64
+ haveTotal, haveAvail := false, false
+ scanner := bufio.NewScanner(r)
+ for scanner.Scan() {
+ fields := strings.Fields(scanner.Text())
+ if len(fields) < 2 {
+ continue
+ }
+ v, err := strconv.ParseInt(fields[1], 10, 64)
+ if err != nil {
+ continue
+ }
+ switch fields[0] {
+ case "MemTotal:":
+ totalKB, haveTotal = v, true
+ case "MemAvailable:":
+ availKB, haveAvail = v, true
+ }
+ }
+ if err := scanner.Err(); err != nil || !haveTotal || !haveAvail || totalKB <= 0 {
+ return nil, nil
+ }
+ totalBytes := totalKB * 1024
+ availBytes := availKB * 1024
+ return &totalBytes, &availBytes
+}
+
+// readMemBytes reads /proc/meminfo and returns MemTotal and MemAvailable as
+// bytes. Unreadable or unparseable -> nil, nil.
+func readMemBytes() (*int64, *int64) {
+ f, err := os.Open("/proc/meminfo")
+ if err != nil {
+ return nil, nil
+ }
+ defer f.Close()
+ return parseMeminfoBytes(f)
+}
+
+// parseLoad1 parses /proc/loadavg's first field (1-minute load average) from
+// a string. The file format is: "load1 load5 load15 running/total lastpid".
+// Empty or unparseable -> nil.
+func parseLoad1(s string) *float64 {
+ fields := strings.Fields(s)
+ if len(fields) == 0 {
+ return nil
+ }
+ l, err := strconv.ParseFloat(fields[0], 64)
+ if err != nil {
+ return nil
+ }
+ return &l
+}
+
+// readLoad1 reads /proc/loadavg and returns the 1-minute load average.
+// Unreadable or unparseable -> nil.
+func readLoad1() *float64 {
+ b, err := os.ReadFile("/proc/loadavg")
+ if err != nil {
+ return nil
+ }
+ return parseLoad1(string(b))
+}
+
+// readDiskFreeBytes reports how much free space is available on the data
+// volume, via syscall.Statfs on dataDir() — /data by default, CARLOS_DATA_DIR
+// when set. Statfs failure or a zero-block filesystem -> nil rather than a
+// crash.
+func readDiskFreeBytes() *int64 {
+ var st syscall.Statfs_t
+ if err := syscall.Statfs(dataDir(), &st); err != nil {
+ return nil
+ }
+ if st.Blocks == 0 {
+ return nil
+ }
+ freeBytes := int64(st.Bavail) * st.Bsize
+ return &freeBytes
+}
diff --git a/internal/host/tunnel_client.go b/internal/host/tunnel_client.go
new file mode 100644
index 000000000..555605a00
--- /dev/null
+++ b/internal/host/tunnel_client.go
@@ -0,0 +1,59 @@
+// The agent's wiring for the host half of the reverse channel (issue #53 / C6
+// Step 3). The dial/serve/backoff machinery itself lives in
+// internal/tunnelclient — a leaf package, so the edge's acceptance test can
+// drive the SAME loop this agent runs (internal/host imports internal/edge,
+// so the edge tests can never import this package). What is left here is the
+// part that is genuinely agent-shaped: reading the env, and publishing the
+// state the status heartbeat reports.
+package host
+
+import (
+ "context"
+ "log"
+ "net/http"
+ "os"
+ "sync/atomic"
+
+ "github.com/carlosframework/platform/internal/tunnelclient"
+)
+
+// tunnelStateVal is "" (disabled/never started), "connected", or
+// "reconnecting" — read by statusLoop's heartbeat probe (probes must fail
+// toward honest values; "" means the loop isn't running at all).
+var tunnelStateVal atomic.Value
+
+func tunnelState() string {
+ if v, ok := tunnelStateVal.Load().(string); ok {
+ return v
+ }
+ return ""
+}
+
+// tunnelLoop dials CARLOS_TUNNEL_URL and serves handler over the inverted
+// conn, forever, with jittered capped backoff. Inert unless all three env
+// vars are set — the same convention as every other optional agent loop.
+func tunnelLoop(handler http.Handler) {
+ rawURL := os.Getenv("CARLOS_TUNNEL_URL")
+ token := os.Getenv("CARLOS_TUNNEL_TOKEN")
+ name := os.Getenv("CARLOS_TUNNEL_NAME")
+ if rawURL == "" || token == "" || name == "" {
+ if rawURL != "" || token != "" || name != "" {
+ log.Printf("agent: tunnel loop disabled: need all of CARLOS_TUNNEL_URL, CARLOS_TUNNEL_TOKEN, CARLOS_TUNNEL_NAME")
+ }
+ return
+ }
+ log.Printf("agent: tunnel loop armed (%s as %q)", rawURL, name)
+ // context.Background(): the agent's tunnel runs for the life of the
+ // process and is torn down with it. The ctx exists for callers that need
+ // to stop the loop deterministically (the edge's e2e), not for this one.
+ tunnelclient.Loop(context.Background(), tunnelclient.Config{
+ URL: rawURL,
+ Name: name,
+ Token: token,
+ Handler: handler,
+ OnState: func(s string) { tunnelStateVal.Store(s) },
+ // The "agent: " prefix is what operators grep the journal for, so it
+ // is preserved here rather than baked into the leaf package.
+ Logf: func(format string, args ...any) { log.Printf("agent: "+format, args...) },
+ })
+}
diff --git a/internal/host/tunnel_client_test.go b/internal/host/tunnel_client_test.go
new file mode 100644
index 000000000..ed8904d87
--- /dev/null
+++ b/internal/host/tunnel_client_test.go
@@ -0,0 +1,100 @@
+package host
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+)
+
+// The dial/serve/handshake tests for this channel live in
+// internal/tunnelclient, where that code now lives. What is testable HERE is
+// the agent's wrapper: the env convention, and the state transitions the
+// status heartbeat reads off tunnelState().
+
+// TestTunnelLoopStateTransitions drives tunnelLoop against a fake edge that
+// hijacks, answers 101, then holds the raw conn open until the test releases
+// it — enough to observe tunnelState() go "" -> "connected", and then
+// "connected" -> "reconnecting" once the conn is torn down. This is the state
+// Task 6's heartbeat probe reads, so its transitions are worth asserting
+// through the wrapper that publishes them, not only in the package that
+// reports them.
+func TestTunnelLoopStateTransitions(t *testing.T) {
+ release := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, bufrw, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ t.Errorf("hijack: %v", err)
+ return
+ }
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: carlos-tunnel/1\r\nConnection: Upgrade\r\n\r\n")
+ bufrw.Flush()
+ <-release
+ conn.Close()
+ }))
+ defer srv.Close()
+
+ t.Setenv("CARLOS_TUNNEL_URL", srv.URL)
+ t.Setenv("CARLOS_TUNNEL_TOKEN", "sekrit")
+ t.Setenv("CARLOS_TUNNEL_NAME", "pi-box")
+
+ // tunnelLoop never returns while configured (by design — it retries
+ // forever on context.Background()), so this goroutine outlives the test;
+ // the fake edge is torn down via defer, and any further redial attempts
+ // just fail fast against a closed listener.
+ go tunnelLoop(http.NewServeMux())
+
+ waitForTunnelState(t, "connected")
+
+ close(release)
+
+ waitForTunnelState(t, "reconnecting")
+}
+
+// TestTunnelLoopInertUnlessFullyConfigured checks the documented convention:
+// tunnelLoop must return immediately — never dial, never block — unless all
+// three of CARLOS_TUNNEL_URL/_TOKEN/_NAME are set.
+func TestTunnelLoopInertUnlessFullyConfigured(t *testing.T) {
+ cases := []struct {
+ name, url, token, tname string
+ }{
+ {"all unset", "", "", ""},
+ {"url only", "http://tunnel.invalid", "", ""},
+ {"token only", "", "tok", ""},
+ {"name only", "", "", "pi-box"},
+ {"url and token", "http://tunnel.invalid", "tok", ""},
+ {"url and name", "http://tunnel.invalid", "", "pi-box"},
+ {"token and name", "", "tok", "pi-box"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ t.Setenv("CARLOS_TUNNEL_URL", c.url)
+ t.Setenv("CARLOS_TUNNEL_TOKEN", c.token)
+ t.Setenv("CARLOS_TUNNEL_NAME", c.tname)
+
+ done := make(chan struct{})
+ go func() {
+ tunnelLoop(http.NewServeMux())
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(500 * time.Millisecond):
+ t.Fatal("tunnelLoop did not return promptly with an incomplete env config")
+ }
+ })
+ }
+}
+
+func waitForTunnelState(t *testing.T, want string) {
+ t.Helper()
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ if tunnelState() == want {
+ return
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ t.Fatalf("tunnelState() = %q, want %q before deadline", tunnelState(), want)
+}
diff --git a/internal/lease/lease.go b/internal/lease/lease.go
index 57c6d99a4..4827481ce 100644
--- a/internal/lease/lease.go
+++ b/internal/lease/lease.go
@@ -111,9 +111,20 @@ type renewer struct {
func startRenewer(leaser *lss3.Leaser, lease *litestream.Lease) *renewer {
ctx, cancel := context.WithCancel(context.Background())
rn := &renewer{leaser: leaser, lease: lease, cancel: cancel, done: make(chan struct{})}
+ // Renew at a third of the lease's OWN TTL, not the package constant: the
+ // leaser is the one that stamps expiry (leaser.TTL), so reading the
+ // interval off it keeps renewal and expiry in step for any TTL. In
+ // production leaser.TTL is always LeaseTTL, so this is LeaseTTL/3 as
+ // before; under a test's short TTL it is what makes "stopped renewing"
+ // distinguishable from "never renewed at all". The fallback only guards
+ // NewTicker's panic on a non-positive duration.
+ renewEvery := leaser.TTL / 3
+ if renewEvery <= 0 {
+ renewEvery = LeaseTTL / 3
+ }
go func() {
defer close(rn.done)
- t := time.NewTicker(LeaseTTL / 3)
+ t := time.NewTicker(renewEvery)
defer t.Stop()
for {
select {
@@ -175,6 +186,16 @@ type Held struct {
// signal the activator reacts to. Region may be empty; the AWS default chain
// (env/profile) resolves it.
func Acquire(ctx context.Context, bucket, prefix, region string) (*Held, error) {
+ return acquireTTL(ctx, bucket, prefix, region, LeaseTTL)
+}
+
+// acquireTTL is Acquire with the lease TTL as a seam. Production always
+// passes LeaseTTL; tests pass a short TTL so an expiry-and-takeover sequence
+// (the failover race) fits inside a test run instead of 15 minutes. The TTL
+// governs renewal spacing too — startRenewer ticks at leaser.TTL/3 — so a
+// short-TTL lease is renewed proportionally often and lapses proportionally
+// fast, which is what makes "the holder went dark" observable.
+func acquireTTL(ctx context.Context, bucket, prefix, region string, ttl time.Duration) (*Held, error) {
// The leaser wants a ready-made S3 client (unlike the replica client,
// which builds its own). Default AWS credential chain.
var opts []func(*awsconfig.LoadOptions) error
@@ -190,7 +211,7 @@ func Acquire(ctx context.Context, bucket, prefix, region string) (*Held, error)
leaser.SetClient(awss3.NewFromConfig(cfg))
leaser.Bucket = bucket
leaser.Path = prefix // lock lives at <prefix>/lock.json, colocated with the replica
- leaser.TTL = LeaseTTL
+ leaser.TTL = ttl
lease, err := leaser.AcquireLease(ctx)
if err != nil {
diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go
index 299f5737a..8129595d3 100644
--- a/internal/lease/lease_test.go
+++ b/internal/lease/lease_test.go
@@ -233,3 +233,98 @@ func TestReplicateWithLease_MutualExclusion(t *testing.T) {
t.Fatalf("third close: %v", err)
}
}
+
+// TestDarkHostCannotReclaimAfterFailover proves the dark-then-returns half of
+// failover (issue #53): host A holds the lease and goes DARK — stops renewing
+// but never releases, the kill -9 / power-cut shape. Its lock.json stays in
+// S3 with the expiry already stamped on it, so:
+//
+// 1. while that expiry is still in the future a peer is refused — dark is
+// not the same as expired, and nobody may jump a live lease;
+// 2. once it lapses, host B claims it — the failover wake;
+// 3. A comes back and tries to pick up where it left off, and MUST lose,
+// loudly, with a *HeldError. A returning dark host silently reclaiming a
+// lease that has moved on is the split-brain this lock exists to prevent.
+//
+// The other half of failover — a fallback host's activator waking the
+// instance from the regional bucket — is the existing activator wake path and
+// is covered by the activator's own tests; nothing about it is new here.
+//
+// Runs on a 15s TTL through the acquireTTL seam; at the production LeaseTTL
+// the same sequence would take 15 minutes.
+func TestDarkHostCannotReclaimAfterFailover(t *testing.T) {
+ bucket, prefix, region := liveConfig(t)
+ ctx := context.Background()
+ const shortTTL = 15 * time.Second
+
+ a, err := acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ if err != nil {
+ t.Fatalf("A initial acquire: %v", err)
+ }
+ acquiredAt := time.Now()
+ // A goes dark. rn.stop() cancels the renewal goroutine's context and
+ // waits for it to exit; it makes no S3 call of any kind, so lock.json
+ // survives untouched and simply runs out its stamped TTL. Without this
+ // line A would renew every shortTTL/3 and step 2 below could never
+ // happen — that is what makes the stop load-bearing.
+ a.rn.stop()
+
+ // Dark is not expired: A's lock is still in force, so B must be refused —
+ // and refused BY THE LEASE, not by a throttle, a 5xx or a credential
+ // hiccup, which would let this leg pass while proving nothing. So the
+ // refusal has to be a *HeldError carrying a live expiry, exactly as
+ // TestReplicateWithLease_MutualExclusion demands of its second claimant.
+ // (Both "hosts" run in this one process and therefore share the leaser's
+ // owner string, so the expiry — not the name — is what identifies the
+ // holder here.)
+ peer, err := acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ if err == nil {
+ elapsed := time.Since(acquiredAt)
+ _ = peer.Release(ctx)
+ if elapsed >= shortTTL {
+ t.Fatalf("second acquire only returned %s after A acquired, past the %s TTL: this environment is too slow for the test's timing margins, which is not the same as a mutual-exclusion failure", elapsed, shortTTL)
+ }
+ t.Fatalf("B acquired after %s while A's un-expired %s lease was still held", elapsed, shortTTL)
+ }
+ var stillHeld *HeldError
+ if !errors.As(err, &stillHeld) {
+ t.Fatalf("peer refused while A was dark-but-live, but not with *HeldError: %v — an S3 or credential failure here would make this leg vacuous", err)
+ }
+ if !stillHeld.ExpiresAt.After(time.Now()) {
+ t.Fatalf("peer refused by an ALREADY-EXPIRED lease (expires %s, %s after A acquired); want A's still-live lease", stillHeld.ExpiresAt.Format(time.RFC3339), time.Since(acquiredAt))
+ }
+ t.Logf("dark is not expired: peer refused by A's live lease until %s", stillHeld.ExpiresAt.Format(time.RFC3339))
+
+ time.Sleep(shortTTL + 5*time.Second) // let A's abandoned lease lapse
+
+ b, err := acquireTTL(ctx, bucket, prefix, region, shortTTL)
+ if err != nil {
+ t.Fatalf("B acquire after A's lease expired (the failover wake): %v", err)
+ }
+ bHeldFrom := time.Now()
+ defer b.Release(ctx)
+
+ // A returns from the dead and tries to resume.
+ if revenant, err := acquireTTL(ctx, bucket, prefix, region, shortTTL); err == nil {
+ elapsed := time.Since(bHeldFrom)
+ _ = revenant.Release(ctx)
+ if elapsed >= shortTTL {
+ // B's own lease had lapsed by then (B is not renewed here either),
+ // so A winning proves nothing. Say so rather than cry split brain.
+ t.Fatalf("A's re-acquire only returned %s after B took over, past the %s TTL: this environment is too slow for the test's timing margins", elapsed, shortTTL)
+ }
+ t.Fatal("returning dark host reclaimed the lease B now holds — split brain")
+ } else {
+ var held *HeldError
+ if !errors.As(err, &held) {
+ t.Fatalf("returning dark host got %v, want *HeldError (it must lose the race)", err)
+ }
+ // The refusal has to come from B's LIVE lease. An expiry already in
+ // the past would mean the acquire lost for some other reason and the
+ // test would be proving nothing.
+ if !held.ExpiresAt.After(time.Now()) {
+ t.Fatalf("returning host refused by an already-expired lease (expires %s); want B's live lease", held.ExpiresAt.Format(time.RFC3339))
+ }
+ t.Logf("returning dark host lost the race, as required: %v", err)
+ }
+}
diff --git a/internal/registry/registry.go b/internal/registry/registry.go
index d3431f19e..05694ab6d 100644
--- a/internal/registry/registry.go
+++ b/internal/registry/registry.go
@@ -12,6 +12,7 @@ package registry
import (
"database/sql"
"fmt"
+ "regexp"
"strings"
"time"
@@ -68,6 +69,11 @@ type Route struct {
// that survivable rather than silent, but a column the reconciler
// rebuilds is not one this verb should be handing an operator.
Environment string
+ // Tunnel is the name of the reverse channel backing this route at the
+ // edge ("" = not tunnel-backed — routed to a Socket/Addr upstream as
+ // today). Written only by SetTunnel / carlos route -tunnel; Upsert
+ // preserves non-empty values — see Environment's comment for why.
+ Tunnel string
Hibernate bool
Canary bool
@@ -107,6 +113,26 @@ type Route struct {
// default so pre-existing rows keep today's behavior byte-identical.
const BackingUnit = "unit"
+// tunnelNameRe: lowercase letters, digits and hyphens, 1-32 characters — the
+// same shape as appconfig.ValidEnvironmentName, for the same reason: this
+// name travels through a CLI argument an operator types by hand, and later
+// work in the remote-host-agent feature (issue #53) matches it against a
+// dialed-in tunnel identity and may build keys from it — a permissive shape
+// (path separators, spaces, "..") would let a bad value reach the column
+// verbatim with nothing downstream able to retroactively fix it.
+var tunnelNameRe = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)
+
+// ValidTunnelName reports whether name is usable as a route's Tunnel
+// binding. "" is refused as a NAME here — it is how callers ask for "no
+// tunnel" (Route.Tunnel's zero value), never a tunnel to bind to; the CLI's
+// "-tunnel -" clear sentinel resolves to "" before this is ever called on
+// it, the same way -environment's envClear does for ValidEnvironmentName.
+// Lives here rather than in a CLI or edge package because Tunnel is this
+// package's column — same precedent as BackingUnit above it.
+func ValidTunnelName(name string) bool {
+ return tunnelNameRe.MatchString(name)
+}
+
func Open(path string) (*Registry, error) {
d, err := sql.Open("sqlite", path)
if err != nil {
@@ -179,6 +205,7 @@ CREATE TABLE IF NOT EXISTS routes (
`ALTER TABLE routes ADD COLUMN cert_expires_at INTEGER NOT NULL DEFAULT 0`,
`ALTER TABLE routes ADD COLUMN account TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE routes ADD COLUMN environment TEXT NOT NULL DEFAULT ''`,
+ `ALTER TABLE routes ADD COLUMN tunnel TEXT NOT NULL DEFAULT ''`,
} {
if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") {
return err
@@ -344,7 +371,7 @@ func (r *Registry) Accounts() ([]Account, error) {
const routeCols = `host, slug, kind, socket, addr, status, last_error, db_path,
version, prev_version, app, pool, region, channel, hibernate, canary,
replication, next_due, backing, status_named, platform, dns_status,
-dns_checked_at, cert_expires_at, account, environment, created_at`
+dns_checked_at, cert_expires_at, account, environment, tunnel, created_at`
func scanRoute(scan func(...any) error) (Route, error) {
var t Route
@@ -353,7 +380,7 @@ func scanRoute(scan func(...any) error) (Route, error) {
&t.LastError, &t.DBPath, &t.Version, &t.PrevVersion, &t.App, &t.Pool,
&t.Region, &t.Channel, &hib, &can, &t.Replication, &t.NextDue, &t.Backing,
&sn, &plat, &t.DNSStatus, &t.DNSCheckedAt, &t.CertExpiresAt, &t.Account,
- &t.Environment, &t.CreatedAt)
+ &t.Environment, &t.Tunnel, &t.CreatedAt)
t.Hibernate = hib != 0
t.Canary = can != 0
t.StatusNamed = sn != 0
@@ -413,17 +440,18 @@ func (r *Registry) Delete(host string) error {
return err
}
-// Upsert inserts or updates a route by host. One column is deliberately not
-// a plain overwrite: `environment` keeps the existing row's value when the
-// incoming Route carries the zero value (2026-08-02 review, IMPORTANT 3).
-// Every full-row Upsert caller — `carlos add` and the domains reconciler's
-// two mint paths — builds its Route without knowing anything about
-// environment bindings, so a plain excluded.environment silently unbound a
-// route on any re-add or re-mint, and the next config sync then rendered the
-// default bundle over that route's env file and restarted its unit. An
-// explicit non-empty value still wins, and clearing goes through
-// SetEnvironment (what `carlos route -environment -` calls), which is the
-// only place a clear is ever intended.
+// Upsert inserts or updates a route by host. Two columns are deliberately not
+// a plain overwrite: `environment` and `tunnel` keep the existing row's value
+// when the incoming Route carries the zero value (2026-08-02 review,
+// IMPORTANT 3, and the same rationale extended to Tunnel). Every full-row
+// Upsert caller — `carlos add` and the domains reconciler's two mint paths —
+// builds its Route without knowing anything about environment or tunnel
+// bindings, so a plain excluded.environment/excluded.tunnel silently unbound
+// a route on any re-add or re-mint, and (for environment) the next config
+// sync then rendered the default bundle over that route's env file and
+// restarted its unit. An explicit non-empty value still wins, and clearing
+// goes through SetEnvironment / SetTunnel (what `carlos route -environment -`
+// / `-tunnel -` call), which is the only place a clear is ever intended.
func (r *Registry) Upsert(t Route) error {
if t.CreatedAt == 0 {
t.CreatedAt = time.Now().Unix()
@@ -442,7 +470,7 @@ func (r *Registry) Upsert(t Route) error {
}
_, err := r.sql.Exec(`
INSERT INTO routes (`+routeCols+`)
-VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(host) DO UPDATE SET
slug=excluded.slug, kind=excluded.kind, socket=excluded.socket,
addr=excluded.addr, status=excluded.status, last_error=excluded.last_error,
@@ -455,12 +483,13 @@ ON CONFLICT(host) DO UPDATE SET
platform=excluded.platform, dns_status=excluded.dns_status,
dns_checked_at=excluded.dns_checked_at,
cert_expires_at=excluded.cert_expires_at, account=excluded.account,
- environment=CASE WHEN excluded.environment = '' THEN routes.environment ELSE excluded.environment END`,
+ environment=CASE WHEN excluded.environment = '' THEN routes.environment ELSE excluded.environment END,
+ tunnel=CASE WHEN excluded.tunnel = '' THEN routes.tunnel ELSE excluded.tunnel END`,
t.Host, t.Slug, t.Kind, t.Socket, t.Addr, t.Status, t.LastError,
t.DBPath, t.Version, t.PrevVersion, t.App, t.Pool, t.Region, t.Channel,
b2i(t.Hibernate), b2i(t.Canary), t.Replication, t.NextDue, t.Backing,
b2i(t.StatusNamed), b2i(t.Platform), t.DNSStatus, t.DNSCheckedAt,
- t.CertExpiresAt, t.Account, t.Environment, t.CreatedAt)
+ t.CertExpiresAt, t.Account, t.Environment, t.Tunnel, t.CreatedAt)
return err
}
@@ -562,6 +591,28 @@ func (r *Registry) SetEnvironment(host, env string) error {
return nil
}
+// SetTunnel repoints a route's reverse-channel binding as a narrow UPDATE —
+// same rationale and same "error on an unmatched host" contract as
+// SetEnvironment (a human types a hostname into `carlos route -tunnel`
+// directly, and a silent no-op on a typo would be worse than a refusal).
+// tunnel == "" clears the binding back to a Socket/Addr-routed upstream; the
+// CLI's "-tunnel -" sentinel resolves to that before calling here, so this
+// function itself has no notion of "clear" beyond writing "".
+func (r *Registry) SetTunnel(host, tunnel string) error {
+ res, err := r.sql.Exec(`UPDATE routes SET tunnel = ? WHERE host = ?`, tunnel, host)
+ if err != nil {
+ return err
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ return fmt.Errorf("no route for host %q", host)
+ }
+ return nil
+}
+
// SetEnvironments is SetEnvironment's bulk sibling, mirroring SetChannels:
// one transaction across every host in hosts, so `carlos route -app
// -environment` can never leave an app split across two environments the
@@ -711,6 +762,15 @@ WHERE kind = 'instance' AND next_due > 0 AND next_due <= ? ORDER BY host`, now)
// edge's cheap change poll. It must change on any row insert/delete and on
// the column writes reloads care about; it must see other processes' commits
// (fresh connection, same WAL-snapshot quirk as List).
+//
+// Text columns contribute their LENGTH under a per-column weight, so a rewrite
+// to a DIFFERENT value of the SAME length (channel canary/1 -> canary/2,
+// tunnel pi-box -> hq-box) is invisible to this poll. That is a deliberate
+// cost of keeping it a single cheap aggregate: an operator's `carlos route`
+// runs beside an edge they can SIGHUP, and every same-length rewrite reaches
+// the fleet on the next reload for any other reason. Weights (3,5,7,11,13…)
+// keep DIFFERENT columns from cancelling each other out, which is the failure
+// that would actually hide a change forever.
func (r *Registry) Fingerprint() (string, error) {
db, err := sql.Open("sqlite", r.path)
if err != nil {
@@ -725,7 +785,7 @@ func (r *Registry) Fingerprint() (string, error) {
SELECT COUNT(*) || ':' || COALESCE(MAX(rowid),0) || ':' ||
COALESCE(SUM(rowid*17 + LENGTH(status)*3 + LENGTH(socket) + LENGTH(addr)
+ LENGTH(version)*5 + hibernate*7 + LENGTH(channel)
- + LENGTH(backing)*11),0)
+ + LENGTH(backing)*11 + LENGTH(tunnel)*13),0)
FROM routes`).Scan(&fp)
return fp, err
}
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go
index aba9dc0b4..435e9dcc7 100644
--- a/internal/registry/registry_test.go
+++ b/internal/registry/registry_test.go
@@ -168,6 +168,19 @@ func TestFingerprintChangesOnWrite(t *testing.T) {
if withBacking == final {
t.Fatal("fingerprint blind to a backing flip")
}
+ // Tunnel bindings must reload the edge too: this poll is the ONLY thing
+ // that notices a `carlos route -tunnel` written by another process, and a
+ // route the edge never reloads stays pointed at its old backend.
+ if err := reg.SetTunnel("c.example.com", "pi-box"); err != nil {
+ t.Fatal(err)
+ }
+ withTunnel, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if withTunnel == withBacking {
+ t.Fatal("fingerprint blind to a tunnel binding")
+ }
}
func TestSetVersionsAndHibernateAreNarrow(t *testing.T) {
@@ -778,3 +791,110 @@ func TestUpsertPreservesAnExistingEnvironmentBinding(t *testing.T) {
t.Fatalf("Environment = %q after an explicit clear, want it cleared", got.Environment)
}
}
+
+func TestTunnelColumnRoundTrips(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{Host: "demo.example", Kind: "instance", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetTunnel("demo.example", "pi-box"); err != nil {
+ t.Fatal(err)
+ }
+ got, err := reg.Get("demo.example")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.Tunnel != "pi-box" {
+ t.Fatalf("Tunnel = %q, want pi-box", got.Tunnel)
+ }
+}
+
+// Same rationale as TestUpsertPreservesAnExistingEnvironmentBinding: a
+// full-row Upsert caller that does not know about tunnel bindings must not
+// silently unbind one by re-writing the zero value.
+func TestUpsertPreservesTunnel(t *testing.T) {
+ reg := openTestRegistry(t)
+ base := Route{Host: "demo.example", Kind: "instance", Status: "ready"}
+ if err := reg.Upsert(base); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetTunnel("demo.example", "pi-box"); err != nil {
+ t.Fatal(err)
+ }
+ // A full-row re-Upsert with zero-value Tunnel must not unbind.
+ if err := reg.Upsert(base); err != nil {
+ t.Fatal(err)
+ }
+ got, err := reg.Get("demo.example")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.Tunnel != "pi-box" {
+ t.Fatalf("Upsert clobbered Tunnel: %q", got.Tunnel)
+ }
+ // Clearing has exactly one door: SetTunnel(host, "").
+ if err := reg.SetTunnel("demo.example", ""); err != nil {
+ t.Fatal(err)
+ }
+ got, err = reg.Get("demo.example")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.Tunnel != "" {
+ t.Fatalf("SetTunnel clear failed: %q", got.Tunnel)
+ }
+}
+
+// Same contract as TestSetEnvironmentUnknownHostErrors: a human types a
+// hostname into `carlos route -tunnel` directly, and a silent no-op on a
+// typo would be worse than a refusal.
+func TestSetTunnelUnknownHostErrors(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.SetTunnel("nope.example.com", "pi-box"); err == nil {
+ t.Fatal("SetTunnel on an unknown host: got nil error, want one")
+ }
+}
+
+// Mirrors appconfig.TestValidEnvironmentName: ValidTunnelName gates what
+// reaches the `tunnel` column before it does — a name later matched against
+// a dialed-in tunnel identity and used to build keys (remote-host-agent
+// feature, issue #53), so path separators, spaces, and "" must never reach
+// SetTunnel/Upsert unvalidated. Unlike ValidEnvironmentName there is no
+// reserved name (no "default" tunnel), so "" is invalid only because it
+// fails the character class, not a separate reservation.
+func TestValidTunnelName(t *testing.T) {
+ valid := []string{"demo", "a", "pi-box", "connect-timeslots", "0-9"}
+ invalid := []string{
+ "", // empty
+ "Demo", // uppercase
+ "has space", // space
+ "has_us", // underscore not allowed
+ "../../x", // path traversal
+ "UP",
+ string(make([]byte, 33)), // too long (33 NUL bytes, also fails the char class)
+ }
+ for _, n := range valid {
+ if !ValidTunnelName(n) {
+ t.Errorf("ValidTunnelName(%q) = false, want true", n)
+ }
+ }
+ for _, n := range invalid {
+ if ValidTunnelName(n) {
+ t.Errorf("ValidTunnelName(%q) = true, want false", n)
+ }
+ }
+}
+
+func TestValidTunnelNameLengthBoundary(t *testing.T) {
+ ok32 := "01234567890123456789012345678901" // 32 chars
+ if len(ok32) != 32 {
+ t.Fatalf("test fixture is %d chars, want 32", len(ok32))
+ }
+ if !ValidTunnelName(ok32) {
+ t.Errorf("ValidTunnelName(32 chars) = false, want true")
+ }
+ tooLong := ok32 + "x"
+ if ValidTunnelName(tooLong) {
+ t.Errorf("ValidTunnelName(33 chars) = true, want false")
+ }
+}
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go
index 43cff20a6..0ad706fa6 100644
--- a/internal/status/contract_test.go
+++ b/internal/status/contract_test.go
@@ -38,9 +38,22 @@ var allowedBoxKeys = map[string][]string{
// armed, i.e. whether config reaches unit-backed routes in seconds or only
// on the hourly floor. Same publication argument — a platform-own unit's
// liveness on an opaquely-labelled box, no tenant in it anywhere.
- "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "expired_canaries"},
+ //
+ // "metrics" is the one key in this allowlist that is EXEMPT from the
+ // band-word-not-a-number rule every other Box field follows (Task 5,
+ // internal/status/snapshot.go's package doc): it carries raw byte
+ // counts, a load average, a release sha, and tunnel state, unbanded.
+ // That's safe only because Box (and this "metrics" key with it) never
+ // leaves the bucket — status/boxes/<label>.json is read by fleet-status
+ // machinery and operators, never served publicly (internal/edge/status.go's
+ // fixed three-object menu doesn't include it) — and because
+ // allowedFleetKeys below has no "metrics" entry: BuildFleet's explicit
+ // field-by-field copy into FleetBox never names it, so TestFleetSchemaAllowlisted
+ // would fail the moment that ever changed.
+ "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "expired_canaries", "metrics"},
"routes": {"static", "resident", "hibernating", "awake", "asleep"},
"replication": {"healthy", "lag_seconds"},
+ "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel"},
}
// allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as
@@ -186,12 +199,25 @@ func fixtureRoutes() []registry.Route {
// fixtureProbes creates test probes with known fractions so that banding
// produces non-nil results. It uses values in the middle bands so that
// load and capacity both resolve to band words (not nil).
+//
+// The six Metrics fields (memTotal/memAvail/load1/diskFree/release/tunnel)
+// are populated here too, with values DISTINCT from dryrun.go's own
+// dryRunMetricsNeedles fixtures — the two harnesses check different things
+// (this file's allowlist tests guard Box/Fleet's declared JSON schema;
+// dryrun.go's leak scan guards the real publish pipeline end to end) and
+// must never share fixture values, or a bug that only one harness would
+// catch could hide behind the other's coincidentally-matching needle.
func fixtureProbes() status.Probes {
memFrac := 0.75 // Between 0.70 and 0.90 -> "busy"
diskFrac := 0.80 // Between 0.70 and 0.90 -> "getting-full"
lagSecs := int64(300) // 5 minutes, healthy (< 15 min)
adopting := true // a real reading, so the field is scanned with a value in it
+ memTotal := int64(8123456789) // distinct from dryrun.go's 4242424242
+ memAvail := int64(3123456789) // distinct from dryrun.go's 4242424241
+ diskFree := int64(1123456789) // distinct from dryrun.go's 4242424240
+ load1 := 1.23 // distinct from dryrun.go's 42.42
+
// Listening function returns true for hibernating routes, false otherwise.
listening := func(sock string) *bool {
if strings.HasPrefix(sock, "/var/run/") {
@@ -203,13 +229,19 @@ func fixtureProbes() status.Probes {
}
return status.Probes{
- MachineID: "test-machine-id",
- Region: "us-west-1",
- MemFrac: &memFrac,
- DiskFrac: &diskFrac,
- Listening: listening,
- LagSecs: &lagSecs,
- AdoptRunning: &adopting,
+ MachineID: "test-machine-id",
+ Region: "us-west-1",
+ MemFrac: &memFrac,
+ DiskFrac: &diskFrac,
+ Listening: listening,
+ LagSecs: &lagSecs,
+ AdoptRunning: &adopting,
+ MemTotalBytes: &memTotal,
+ MemAvailBytes: &memAvail,
+ Load1: &load1,
+ DiskFreeBytes: &diskFree,
+ Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha"
+ Tunnel: "connected", // distinct from dryrun.go's "reconnecting"
}
}
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go
index 83b840a92..14db89cc1 100644
--- a/internal/status/dryrun.go
+++ b/internal/status/dryrun.go
@@ -46,6 +46,25 @@ var dryRunNeedles = []string{
"0.95", // fixture DiskFrac, verbatim
}
+// dryRunMetricsNeedles are the distinctive raw values Task 5's BoxMetrics
+// carries — real numbers and strings that Box.Metrics is the ONE deliberate
+// exception privileged to publish (see snapshot.go's package doc), and which
+// therefore MUST appear in the published box object. Unlike dryRunNeedles
+// above (banned everywhere, box object included), these are allowed inside
+// status/boxes/*.json and banned only from everything else DryRun publishes
+// — see the scope split in DryRun's scan loop below. Each value is
+// deliberately distinctive (the repeated 42s, a value with no legitimate
+// reason to appear in a template or a fleet-wide aggregate) so a false
+// negative from an incidental digit collision is effectively impossible.
+var dryRunMetricsNeedles = []string{
+ "4242424242", // fixture MemTotalBytes, verbatim
+ "4242424241", // fixture MemAvailBytes, verbatim
+ "4242424240", // fixture DiskFreeBytes, verbatim
+ "42.42", // fixture Load1, verbatim
+ "dryrun4242sha", // fixture Release, verbatim
+ "reconnecting", // fixture Tunnel, verbatim (a real, valid Tunnel value)
+}
+
// ScanForLeaks reports which of banned's strings appear anywhere in j. It is
// the single implementation both DryRun's self-check and
// contract_test.go's scanForLeaks test helper run — the test package can't
@@ -90,6 +109,14 @@ func dryRunFixtures() ([]registry.Route, Probes) {
adopting := true // a known reading, so adoption_running publishes non-null and gets scanned
configPath := true // ditto for config_path_running (issue #44)
expiredCanaries := 2 // ditto for expired_canaries — a known count, not the unexercised nil path
+
+ // Task 5's raw Metrics fixtures: distinctive values (see
+ // dryRunMetricsNeedles) that MUST land in the published box object and
+ // MUST NOT land anywhere else DryRun publishes — the positive half of
+ // proving Box.Metrics's private exception actually stays private.
+ memTotal, memAvail, diskFree := int64(4242424242), int64(4242424241), int64(4242424240)
+ load1 := 42.42
+
probes := Probes{
MachineID: "ip-10-0-0-7",
Region: "us-east-1",
@@ -101,7 +128,13 @@ func dryRunFixtures() ([]registry.Route, Probes) {
ExpiredCanaries: &expiredCanaries,
// Every hibernating route reports "listening" so Awake/Asleep publish
// non-null — an unexercised null field is a field DryRun can't scan.
- Listening: func(string) *bool { v := true; return &v },
+ Listening: func(string) *bool { v := true; return &v },
+ MemTotalBytes: &memTotal,
+ MemAvailBytes: &memAvail,
+ Load1: &load1,
+ DiskFreeBytes: &diskFree,
+ Release: "dryrun4242sha",
+ Tunnel: "reconnecting",
}
return routes, probes
}
@@ -200,6 +233,13 @@ func DryRun(dir string, out io.Writer) error {
return fmt.Errorf("dry run: nothing published under status/ — publish path is broken")
}
+ // boxKeySeen tracks whether the scan actually reached a status/boxes/
+ // object carrying the Metrics fixtures — the positive half of the
+ // Metrics scope check below, the same reason appPublished exists for
+ // the opt-in app: a leak-scan that finds nothing proves nothing about
+ // whether the thing that SHOULD be there actually is.
+ boxKeySeen := false
+
for _, key := range keys {
body, _, err := s.Get(ctx, key)
if err != nil {
@@ -211,6 +251,25 @@ func DryRun(dir string, out io.Writer) error {
return fmt.Errorf("dry run: %s leaked %v", key, leaks)
}
+ // dryRunMetricsNeedles are scoped: status/boxes/*.json is Box.Metrics's
+ // one privileged, private home (see snapshot.go's package doc), so
+ // the raw fixture values are EXPECTED there and must NOT be scanned
+ // out — that isn't a leak, it's the field doing its job. Every other
+ // key DryRun publishes (fleet.json, index.html, status.css) is
+ // exactly what BuildFleet/PublishFleet expose beyond the bucket, so
+ // those raw values must never appear there.
+ if strings.HasPrefix(key, "status/boxes/") {
+ if leaks := ScanForLeaks(body, dryRunMetricsNeedles); len(leaks) == 0 {
+ // None of the fixtures made it in at all — Metrics isn't
+ // being attached, which means this check isn't exercising
+ // anything (a false "no leak").
+ return fmt.Errorf("dry run: %s: expected raw Metrics fixtures %v, found none — Box.Metrics not attached?", key, dryRunMetricsNeedles)
+ }
+ boxKeySeen = true
+ } else if leaks := ScanForLeaks(body, dryRunMetricsNeedles); len(leaks) > 0 {
+ return fmt.Errorf("dry run: %s leaked raw Metrics fixture %v — Box.Metrics must never reach the public feed", key, leaks)
+ }
+
// Page assets (index.html, status.css) aren't JSON and simply fail
// to decode here — silently skipped, since the band-digit check
// only means anything for objects that actually carry Load/Capacity
@@ -223,6 +282,10 @@ func DryRun(dir string, out io.Writer) error {
}
}
+ if !boxKeySeen {
+ return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — Metrics scope check never ran")
+ }
+
fmt.Fprintln(out, "dry-run: no leaks")
return nil
}
diff --git a/internal/status/fleet.go b/internal/status/fleet.go
index 8810c8c0b..eaf12a48a 100644
--- a/internal/status/fleet.go
+++ b/internal/status/fleet.go
@@ -103,9 +103,52 @@ var capacityRank = map[string]int{
// without this helper.
func intPtr(v int) *int { return &v }
+// SnapshotPeriod is how often every box republishes its own
+// status/boxes/<label>.json (the agent's status loop tick, 5 minutes +
+// jitter). It lives here rather than at each reader because it is the unit
+// BoxStale measures staleness in: the console's private host detail and
+// `carlos status -live` both have to answer "is this box dark?" without an
+// agent's local const to borrow, and a reader that guessed a different period
+// would call a box unreachable at a different moment than the published
+// boxes table does.
+const SnapshotPeriod = 5 * time.Minute
+
+// staleFactor is how many missed snapshot periods make a box stale. Three
+// consecutive misses: past any single hiccup (a slow S3 write, one skipped
+// tick), and the point at which a box has plainly stopped self-reporting.
+// One constant, one rule — BuildFleet's published FleetBox.Stale and every
+// private reader go through BoxStale below, so nothing can drift into calling
+// the same box fresh on one page and dark on another.
+const staleFactor = 3
+
+// BoxStale is the fleet's one staleness rule: a snapshot older than
+// staleFactor×period means the box stopped reporting. Every caller — the
+// public merge, the console's private host table, the CLI — asks this
+// function, never its own subtraction.
+//
+// Callers that render must treat true as "this box's numbers are history":
+// per the design spec an unreachable host shows how long it has been dark and
+// NOTHING else, never its last happy readings.
+func BoxStale(b Box, now time.Time, period time.Duration) bool {
+ return now.Sub(b.GeneratedAt) > staleFactor*period
+}
+
+// LoadBoxes reads every per-box snapshot in the bucket — the same read
+// BuildFleet does, exported for the private readers (the console's operator
+// fleet page, `carlos status -live`) so there is exactly ONE parser of the
+// status/boxes/ shape and one skip-corrupt rule, rather than a second loader
+// that could disagree about which snapshots count.
+//
+// Unlike the merged feed, what comes back here still carries Box.Metrics: raw
+// numbers that must reach a private surface only (bucket credentials or a
+// session-gated operator page), never fleet.json. See the package comment.
+func LoadBoxes(ctx context.Context, s release.Store) ([]Box, error) {
+ return loadBoxes(ctx, s)
+}
+
// BuildFleet merges every status/boxes/*.json (stale = GeneratedAt older
-// than 3×period), folds the local registry's routes into apps/services,
-// and never invents a number a snapshot didn't carry.
+// than staleFactor×period — see BoxStale), folds the local registry's routes
+// into apps/services, and never invents a number a snapshot didn't carry.
func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, now time.Time, period time.Duration) (*Fleet, error) {
boxes, err := loadBoxes(ctx, s)
if err != nil {
@@ -120,7 +163,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n
// fresh boxes.
stale := make([]bool, len(boxes))
for i, b := range boxes {
- stale[i] = now.Sub(b.GeneratedAt) > 3*period
+ stale[i] = BoxStale(b, now, period)
f.Boxes = append(f.Boxes, FleetBox{
Label: b.Label,
Region: b.Region,
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 11cf48ab9..92b69ad1c 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -4,6 +4,23 @@
// HERE, in the process that does the counting and banding, not downstream —
// a merge step or a page template can only leak what this package already
// let through.
+//
+// Box.Metrics is the one deliberate exception to that rule, not a hole in
+// it: a raw, unbanded block (real byte counts, a load average, the running
+// build, tunnel state) that CollectBox attaches to the Box it returns, which
+// WriteBox lands in status/boxes/<label>.json alongside everything else this
+// package computes. That file is bucket-private — the deployment bucket
+// blocks all public access (infra/modules/deployment-buckets), and the edge
+// process's fixed three-object menu (internal/edge/status.go) only ever
+// serves "/", "/fleet.json", and "/status.css"; a request for
+// status/boxes/<label>.json 404s the same as any other unlisted path. Only
+// fleet-status machinery (loadBoxes in fleet.go) and operators with bucket
+// credentials ever read it. The public merge step, fleet.go's BuildFleet,
+// copies Box's fields into FleetBox one at a time by field name and simply
+// never names Metrics — so the banded-words-only commitment on fleet.json
+// holds by construction, not by a filter that could be forgotten. See
+// dryrun.go's leak fixtures, which assert a raw metric value survives into
+// the box object but never into fleet.json or index.html.
package status
import (
@@ -52,6 +69,37 @@ type Probes struct {
// zero; 0 is itself a real, confident answer when every canary route on
// the box is fresh, or there are none at all.
ExpiredCanaries *int
+
+ // MemTotalBytes, MemAvailBytes, Load1, and DiskFreeBytes are the raw
+ // numbers MemFrac/DiskFrac above are already computed FROM — CollectBox
+ // still bands MemFrac/DiskFrac into Load/Capacity as always, but also
+ // carries these through unbanded into Box.Metrics, the package's one
+ // documented private exception (see the package doc comment). nil means
+ // unknown, same as every other pointer field here.
+ MemTotalBytes *int64
+ MemAvailBytes *int64
+ Load1 *float64
+ DiskFreeBytes *int64
+ // Release is the running binary's version.Build (a short sha or similar),
+ // and Tunnel is one of ""|"connected"|"reconnecting" — both land in
+ // Box.Metrics unbanded, like the four numeric fields above. "" means
+ // unknown/not-applicable for both, never a guess.
+ Release string
+ Tunnel string
+}
+
+// BoxMetrics is the raw, PRIVATE block: real numbers, never banded. It lives
+// in status/boxes/<label>.json — which only fleet-status machinery and
+// operators read — and is deliberately NOT copied into FleetBox, so the
+// publicly served fleet.json keeps the banded-words-only commitment. See the
+// package doc comment for why this block is safe to exist at all.
+type BoxMetrics struct {
+ MemTotalBytes *int64 `json:"mem_total_bytes,omitempty"`
+ MemAvailBytes *int64 `json:"mem_avail_bytes,omitempty"`
+ Load1 *float64 `json:"load1,omitempty"`
+ DiskFreeBytes *int64 `json:"disk_free_bytes,omitempty"`
+ Release string `json:"release,omitempty"` // running binary's version.Build
+ Tunnel string `json:"tunnel,omitempty"` // ""|"connected"|"reconnecting"
}
// RouteCounts is aggregate-only: no host, no socket path, just how many.
@@ -77,7 +125,14 @@ type Replication struct {
// Box is one box's self-report: what CollectBox produces and WriteBox
// serializes. Every field here has already had its needles pulled — there is
-// no hostname, machine id, or raw ratio anywhere in this type.
+// no hostname, machine id, or raw ratio anywhere in this type — with exactly
+// one deliberate exception: Metrics, the raw/private block documented on the
+// package comment and on BoxMetrics itself. Every OTHER field on Box is
+// still bound by the no-raw-numbers rule; Metrics is not "one more field",
+// it's the one place that rule is intentionally suspended, gated by staying
+// bucket-private (see the package doc) and excluded from FleetBox by
+// construction (fleet.go's BuildFleet copies fields one at a time and never
+// names Metrics).
type Box struct {
GeneratedAt time.Time `json:"generated_at"`
Label string `json:"label"`
@@ -110,6 +165,12 @@ type Box struct {
// is exactly the kind of identifying detail this package's whole design
// refuses to carry, but how many is aggregate and safe.
ExpiredCanaries *int `json:"expired_canaries"`
+ // Metrics is the raw/private exception documented above and on
+ // BoxMetrics — omitted entirely (not a present-but-empty object) when
+ // CollectBox has nothing to report, so an old parser reading a box this
+ // package wrote before Metrics existed, and a box that genuinely can't
+ // answer any of the six fields, produce the identical absent-key shape.
+ Metrics *BoxMetrics `json:"metrics,omitempty"`
}
// replHealthyWithin is the lag under which replication counts as healthy.
@@ -265,7 +326,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
healthy = &h
}
- return Box{
+ box := Box{
GeneratedAt: now,
Label: Label(p.MachineID),
Region: p.Region,
@@ -277,6 +338,22 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
ConfigPathRunning: p.ConfigPathRunning,
ExpiredCanaries: p.ExpiredCanaries,
}
+
+ // Metrics attaches only when something is known — an all-unknown Probes
+ // publishes no block at all, the same "unknown stays absent, never a
+ // guess" rule every other field on Box follows, and it keeps a box that
+ // can't answer any of the six fields indistinguishable on the wire from
+ // a box running code that predates Metrics entirely.
+ if p.MemTotalBytes != nil || p.MemAvailBytes != nil || p.Load1 != nil ||
+ p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" {
+ box.Metrics = &BoxMetrics{
+ MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,
+ Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes,
+ Release: p.Release, Tunnel: p.Tunnel,
+ }
+ }
+
+ return box
}
// WriteBox marshals a Box indented (human-diffable in the bucket) and writes
diff --git a/internal/status/status_test.go b/internal/status/status_test.go
index aa8a5adbc..8c73ed3eb 100644
--- a/internal/status/status_test.go
+++ b/internal/status/status_test.go
@@ -215,6 +215,64 @@ func TestWriteBoxRetriesOnceOnCASConflict(t *testing.T) {
}
}
+// ---- Task 5: raw Metrics block (private to the bucket) --------------------
+
+// TestCollectBoxCarriesRawMetrics proves CollectBox attaches an unbanded
+// Metrics block straight through from Probes when the six fields are known —
+// the one deliberate exception to this package's band-everything rule (see
+// snapshot.go's package doc).
+func TestCollectBoxCarriesRawMetrics(t *testing.T) {
+ mt, ma, df := int64(4056547328), int64(2028273664), int64(9999999)
+ l1 := 0.42
+ b := status.CollectBox(nil, status.Probes{
+ MachineID: "m", MemTotalBytes: &mt, MemAvailBytes: &ma, Load1: &l1, DiskFreeBytes: &df,
+ Release: "abc1234", Tunnel: "connected",
+ }, time.Now())
+ m := b.Metrics
+ if m == nil || *m.MemTotalBytes != mt || *m.Load1 != l1 || m.Release != "abc1234" || m.Tunnel != "connected" {
+ t.Fatalf("metrics block wrong: %+v", m)
+ }
+}
+
+// TestCollectBoxOmitsMetricsWhenAllUnknown proves the all-unknown case
+// really omits the block rather than attaching an empty-but-present
+// BoxMetrics{} — the same "unknown stays absent" rule every other Box field
+// follows, and what makes an old parser's absent-key shape and a
+// can't-answer box's shape identical on the wire.
+func TestCollectBoxOmitsMetricsWhenAllUnknown(t *testing.T) {
+ b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now())
+ if b.Metrics != nil {
+ t.Fatalf("metrics block should be nil when nothing is known, got %+v", b.Metrics)
+ }
+}
+
+// TestBoxBackCompatParse: yesterday's Box shape (no metrics key) must parse
+// today, and a FUTURE box's unknown keys (inside or alongside metrics) must
+// not break today's parse either — additive wire shape, per CLAUDE.md's
+// "consumer = everything that PARSES the shape" rule.
+func TestBoxBackCompatParse(t *testing.T) {
+ yesterday := []byte(`{"generated_at":"2026-08-03T10:00:00Z","label":"ada-stargazing",
+ "routes":{"static":1,"resident":2,"hibernating":3,"awake":null,"asleep":null},
+ "load":"comfortable","capacity":"plenty",
+ "replication":{"healthy":true,"lag_seconds":5},
+ "adoption_running":true,"expired_canaries":0}`)
+ var b status.Box
+ if err := json.Unmarshal(yesterday, &b); err != nil {
+ t.Fatalf("yesterday's shape failed to parse: %v", err)
+ }
+ if b.Metrics != nil {
+ t.Fatal("absent metrics must decode as nil, not zero-value")
+ }
+ tomorrow := []byte(`{"generated_at":"2026-08-05T10:00:00Z","label":"ada-stargazing",
+ "routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},
+ "load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},
+ "adoption_running":null,"expired_canaries":null,
+ "metrics":{"mem_total_bytes":1,"some_future_field":true}}`)
+ if err := json.Unmarshal(tomorrow, &b); err != nil {
+ t.Fatalf("tomorrow's shape failed to parse: %v", err)
+ }
+}
+
// ---- Task 3: fleet merge + publisher --------------------------------------
// teststore is a fresh DirStore over a scratch directory — every fleet test
@@ -815,3 +873,77 @@ func TestFleetBoxCarriesAdoption(t *testing.T) {
f.Boxes[0].AdoptionRunning)
}
}
+
+// ---- Task 7: the exported box loader and the one staleness rule -------------
+
+// TestLoadBoxesAndStaleness pins the two things the console's private host
+// detail and `carlos status -live` both read the bucket through: LoadBoxes
+// returns every snapshot WITH its private Metrics block intact (the merged
+// public feed drops it — this loader must not), and BoxStale answers the same
+// "older than 3 periods" question BuildFleet's own boxes table answers, so the
+// two renderings can never disagree about which host is dark.
+func TestLoadBoxesAndStaleness(t *testing.T) {
+ s := teststore(t)
+ ctx := context.Background()
+ now := time.Now()
+ fresh := status.Box{GeneratedAt: now, Label: "ada-stargazing",
+ Metrics: &status.BoxMetrics{Release: "abc1234", Tunnel: "connected"}}
+ dark := status.Box{GeneratedAt: now.Add(-time.Hour), Label: "grace-kayaking"}
+ for _, b := range []status.Box{fresh, dark} {
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil || len(boxes) != 2 {
+ t.Fatalf("LoadBoxes: %v, %d boxes", err, len(boxes))
+ }
+ for _, b := range boxes {
+ wantStale := b.Label == "grace-kayaking"
+ if got := status.BoxStale(b, now, 5*time.Minute); got != wantStale {
+ t.Fatalf("%s: stale = %v, want %v", b.Label, got, wantStale)
+ }
+ if b.Label == "ada-stargazing" {
+ if b.Metrics == nil || b.Metrics.Release != "abc1234" || b.Metrics.Tunnel != "connected" {
+ t.Fatalf("LoadBoxes dropped the private metrics block: %+v", b.Metrics)
+ }
+ }
+ }
+}
+
+// TestBoxStaleMatchesFleetBoxStale is the anti-drift assertion: whatever
+// BoxStale says about a box must be exactly what BuildFleet publishes in that
+// box's FleetBox.Stale — one rule, one constant, two renderings.
+func TestBoxStaleMatchesFleetBoxStale(t *testing.T) {
+ s := teststore(t)
+ ctx := context.Background()
+ now := time.Now()
+ const period = 5 * time.Minute
+ for _, b := range []status.Box{
+ boxFixture("just-now", "plenty", now),
+ boxFixture("two-periods", "plenty", now.Add(-2*period)),
+ boxFixture("exactly-three", "plenty", now.Add(-3*period)),
+ boxFixture("long-dark", "plenty", now.Add(-time.Hour)),
+ } {
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ f, err := status.BuildFleet(ctx, s, nil, now, period)
+ if err != nil {
+ t.Fatal(err)
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ byLabel := map[string]bool{}
+ for _, fb := range f.Boxes {
+ byLabel[fb.Label] = fb.Stale
+ }
+ for _, b := range boxes {
+ if got, want := status.BoxStale(b, now, period), byLabel[b.Label]; got != want {
+ t.Errorf("%s: BoxStale = %v, FleetBox.Stale = %v — the rule has drifted", b.Label, got, want)
+ }
+ }
+}
diff --git a/internal/tunnelclient/tunnelclient.go b/internal/tunnelclient/tunnelclient.go
new file mode 100644
index 000000000..92b4680f6
--- /dev/null
+++ b/internal/tunnelclient/tunnelclient.go
@@ -0,0 +1,267 @@
+// Package tunnelclient is the host half of the reverse channel (issue #53 /
+// C6 Step 3): dial OUT to the edge, invert the connection, and serve a local
+// http.Handler over HTTP/2 on it. Reconnect fast — hq's dual-WAN failover and
+// a Pi's Wi-Fi blip are both "the conn died, dial again", and that latency is
+// user-visible.
+//
+// It is deliberately a LEAF package: stdlib plus golang.org/x/net only, no
+// CARLOS imports. internal/host imports internal/edge, so the edge package's
+// own tests cannot import internal/host — and the acceptance test for this
+// channel (internal/edge/tunnel_e2e_test.go) has to drive the REAL client
+// loop, backoff and all, or the reconnect bound it asserts would be measuring
+// a test double. Keeping dial/serve/backoff here is what lets both sides use
+// one implementation. internal/host/tunnel_client.go is now a thin wrapper:
+// it reads the env and stores the state this package reports.
+package tunnelclient
+
+import (
+ "bufio"
+ "context"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "log"
+ "math/rand/v2"
+ "net"
+ "net/http"
+ "net/url"
+ "time"
+
+ "golang.org/x/net/http2"
+)
+
+// The two states Loop reports through Config.OnState. "" (never reported
+// here) means the loop is not running at all — probes must fail toward honest
+// values, so the caller's zero value has to be distinguishable from anything
+// this package says.
+const (
+ StateConnected = "connected"
+ StateReconnecting = "reconnecting"
+)
+
+const (
+ BackoffFloor = 250 * time.Millisecond
+ BackoffCap = 2 * time.Second
+ // StableConnected is how long a connection has to have stayed up before
+ // its loss resets backoff to the floor. Below this, the disconnect is
+ // treated like a dial failure and backoff keeps escalating: the edge's
+ // re-register-replaces-the-previous-channel semantics mean two peers
+ // sharing a channel name (a cloned Pi env, the agent started twice, a
+ // promote/adopt/restart overlap window) would otherwise evict each other
+ // in an unbounded zero-sleep loop.
+ StableConnected = 5 * time.Second
+ // HandshakeTimeout bounds dial+TLS (via net.Dialer / tls.Dialer) and,
+ // separately, the /register write+ReadResponse round trip: an edge that
+ // accepts the TCP conn and then never answers must not park Loop forever
+ // with no retry and no visible state change.
+ HandshakeTimeout = 10 * time.Second
+)
+
+// Config is one channel's worth of dialing instructions. URL, Name, Token and
+// Handler are required — Loop does not police them, because the caller is the
+// one holding the config source and owes its user the specific complaint (see
+// internal/host's env check). A blank or unparseable URL simply fails every
+// dial and backs off to BackoffCap, which is a bad config loudly logged, not
+// a storm.
+type Config struct {
+ URL string // the edge's registration endpoint, http:// or https://
+ Name string // this channel's name, as the edge has it configured
+ Token string // the bearer token bound to Name
+ Handler http.Handler // served over the inverted conn
+
+ // OnState, when non-nil, is called with StateConnected /
+ // StateReconnecting on every transition — synchronously, from Loop's
+ // goroutine, so it must not block.
+ OnState func(string)
+ // Logf defaults to log.Printf. The agent passes one that prefixes
+ // "agent: " so the journal lines stay exactly what operators grep for.
+ Logf func(format string, args ...any)
+}
+
+// Loop dials cfg.URL and serves cfg.Handler over the inverted conn, forever,
+// with jittered capped backoff. It returns only when ctx is cancelled; a
+// cancelled ctx tears down the live conn rather than waiting for it to die on
+// its own. State is left at whatever the last transition reported — this
+// package never invents a "stopped" state, because the agent's heartbeat
+// probe reads "" as "the loop was never started", which is a different fact.
+func Loop(ctx context.Context, cfg Config) {
+ logf := cfg.Logf
+ if logf == nil {
+ logf = log.Printf
+ }
+ setState := func(s string) {
+ if cfg.OnState != nil {
+ cfg.OnState(s)
+ }
+ }
+ backoff := BackoffFloor
+ for ctx.Err() == nil {
+ // Set BEFORE dialing, not only on error: Dial can block for up to
+ // HandshakeTimeout on a black-holed handshake, and a stale "" or
+ // "connected" during that window would read to the agent's heartbeat
+ // probe as either "never started" or "healthy" — both wrong while
+ // we're actually hung waiting on a dead edge.
+ setState(StateReconnecting)
+ conn, err := Dial(ctx, cfg.URL, cfg.Name, cfg.Token)
+ if err != nil {
+ sleep := backoff + rand.N(backoff/2)
+ logf("tunnel dial: %v (retry in %s)", err, sleep)
+ if !wait(ctx, sleep) {
+ return
+ }
+ if backoff *= 2; backoff > BackoffCap {
+ backoff = BackoffCap
+ }
+ continue
+ }
+ setState(StateConnected)
+ logf("tunnel connected as %q", cfg.Name)
+ connectedAt := time.Now()
+ serveUntilDead(ctx, conn, cfg.Handler) // blocks until the conn dies
+ setState(StateReconnecting)
+ logf("tunnel lost — re-dialing")
+ // A connection that stayed up a while earns a fast retry (the
+ // dual-WAN-failover / Wi-Fi-blip case); one that dies instantly does
+ // not — see StableConnected. Either way we always sleep some amount
+ // before redialing: a zero-delay retry is exactly the storm this
+ // guards against.
+ if time.Since(connectedAt) >= StableConnected {
+ backoff = BackoffFloor
+ } else if backoff *= 2; backoff > BackoffCap {
+ backoff = BackoffCap
+ }
+ sleep := backoff + rand.N(backoff/2)
+ logf("tunnel retry in %s", sleep)
+ if !wait(ctx, sleep) {
+ return
+ }
+ }
+}
+
+// wait sleeps d, or returns false if ctx is cancelled first. Every sleep in
+// Loop goes through it: a loop that ignored cancellation would outlive the
+// test that started it and keep dialing a closed listener.
+func wait(ctx context.Context, d time.Duration) bool {
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-ctx.Done():
+ return false
+ case <-t.C:
+ return true
+ }
+}
+
+// serveUntilDead is Serve plus cancellation: http2.Server.ServeConn blocks
+// until the conn dies and has no other stop signal, so closing the conn is
+// how a cancelled ctx gets Loop back.
+func serveUntilDead(ctx context.Context, conn net.Conn, handler http.Handler) {
+ done := make(chan struct{})
+ go func() {
+ select {
+ case <-ctx.Done():
+ conn.Close()
+ case <-done:
+ }
+ }()
+ defer close(done)
+ Serve(conn, handler)
+}
+
+// bufferedConn replays bytes the handshake's bufio.Reader over-read (the h2
+// preface can arrive in the same packet as the 101) before the raw conn.
+type bufferedConn struct {
+ r io.Reader
+ net.Conn
+}
+
+func (b bufferedConn) Read(p []byte) (int, error) { return b.r.Read(p) }
+
+// Dial performs the registration handshake and returns the raw conn, ready to
+// serve h2 on. https URLs use real TLS (the edge's public cert — server auth
+// by PKI, client auth by bearer token); http is for tests.
+//
+// ctx bounds the TCP connect and the TLS handshake. The /register round trip
+// after that is bounded by HandshakeTimeout instead, not by ctx: a cancel
+// arriving mid-handshake is therefore observed within HandshakeTimeout rather
+// than immediately, which is the one place cancellation is not instant and is
+// deliberately left simple — the alternative is a watchdog goroutine per dial
+// to buy 10 seconds on a shutdown path.
+func Dial(ctx context.Context, rawURL, name, token string) (net.Conn, error) {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return nil, fmt.Errorf("tunnel url: %w", err)
+ }
+ addr := u.Host
+ dialer := &net.Dialer{Timeout: HandshakeTimeout}
+ var conn net.Conn
+ switch u.Scheme {
+ case "https":
+ if u.Port() == "" {
+ addr = net.JoinHostPort(u.Hostname(), "443")
+ }
+ // tls.Dialer applies dialer.Timeout to the TCP connect and the TLS
+ // handshake as a whole (crypto/tls docs) — a black-holed TLS peer
+ // fails the dial instead of hanging it — and ctx on top of that.
+ td := &tls.Dialer{NetDialer: dialer, Config: &tls.Config{
+ ServerName: u.Hostname(),
+ NextProtos: []string{"http/1.1"}, // the handshake is 1.1; h2 starts after the 101
+ }}
+ conn, err = td.DialContext(ctx, "tcp", addr)
+ case "http":
+ conn, err = dialer.DialContext(ctx, "tcp", addr)
+ default:
+ return nil, fmt.Errorf("tunnel url: unsupported scheme %q", u.Scheme)
+ }
+ if err != nil {
+ return nil, err
+ }
+ // The dial above only bounds TCP connect + TLS handshake; the /register
+ // write+read is a separate hang risk (an edge that accepts the conn and
+ // then never answers). Bound it too, and clear the deadline before
+ // returning success — the live h2 session that follows must not inherit a
+ // stale deadline from the handshake.
+ if err := conn.SetDeadline(time.Now().Add(HandshakeTimeout)); err != nil {
+ conn.Close()
+ return nil, fmt.Errorf("tunnel: set handshake deadline: %w", err)
+ }
+ req := &http.Request{
+ Method: "GET",
+ URL: &url.URL{Path: "/register"},
+ Host: u.Hostname(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + token},
+ "Carlos-Tunnel-Name": {name},
+ "Connection": {"Upgrade"},
+ "Upgrade": {"carlos-tunnel/1"},
+ },
+ }
+ if err := req.Write(conn); err != nil {
+ conn.Close()
+ return nil, err
+ }
+ br := bufio.NewReader(conn)
+ resp, err := http.ReadResponse(br, req)
+ if err != nil {
+ conn.Close()
+ return nil, err
+ }
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusSwitchingProtocols {
+ conn.Close()
+ return nil, fmt.Errorf("tunnel register: %s", resp.Status)
+ }
+ if err := conn.SetDeadline(time.Time{}); err != nil {
+ conn.Close()
+ return nil, fmt.Errorf("tunnel: clear handshake deadline: %w", err)
+ }
+ if br.Buffered() > 0 {
+ return bufferedConn{r: io.MultiReader(br, conn), Conn: conn}, nil
+ }
+ return conn, nil
+}
+
+// Serve serves handler over the inverted conn until it dies.
+func Serve(conn net.Conn, handler http.Handler) {
+ (&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: handler})
+}
diff --git a/internal/tunnelclient/tunnelclient_test.go b/internal/tunnelclient/tunnelclient_test.go
new file mode 100644
index 000000000..64e448cb5
--- /dev/null
+++ b/internal/tunnelclient/tunnelclient_test.go
@@ -0,0 +1,154 @@
+package tunnelclient
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "golang.org/x/net/http2"
+)
+
+// fakeTunnelEdge accepts one registration, answers 101, then drives ONE h2
+// request over the inverted conn and returns the response body it read.
+func fakeTunnelEdge(t *testing.T, wantToken, wantName string) (*httptest.Server, chan string) {
+ t.Helper()
+ got := make(chan string, 1)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer "+wantToken ||
+ r.Header.Get("Carlos-Tunnel-Name") != wantName {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ conn, bufrw, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ t.Errorf("hijack: %v", err)
+ return
+ }
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: carlos-tunnel/1\r\nConnection: Upgrade\r\n\r\n")
+ bufrw.Flush()
+ cc, err := (&http2.Transport{}).NewClientConn(conn)
+ if err != nil {
+ t.Errorf("h2 client: %v", err)
+ return
+ }
+ req, _ := http.NewRequest("GET", "https://demo.example/hello", nil)
+ resp, err := cc.RoundTrip(req)
+ if err != nil {
+ t.Errorf("roundtrip over tunnel: %v", err)
+ got <- ""
+ return
+ }
+ body, _ := io.ReadAll(resp.Body)
+ got <- resp.Header.Get("X-From-Host") + ":" + string(body)
+ }))
+ return srv, got
+}
+
+func TestDialAndServeInvertsConnection(t *testing.T) {
+ srv, got := fakeTunnelEdge(t, "sekrit", "pi-box")
+ defer srv.Close()
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-From-Host", r.Host)
+ fmt.Fprint(w, "pong")
+ })
+ conn, err := Dial(context.Background(), srv.URL, "pi-box", "sekrit")
+ if err != nil {
+ t.Fatalf("Dial: %v", err)
+ }
+ go Serve(conn, handler)
+ if v := <-got; v != "demo.example:pong" {
+ t.Fatalf("over-tunnel response = %q, want demo.example:pong", v)
+ }
+}
+
+func TestDialRejectsNon101(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ }))
+ defer srv.Close()
+ if _, err := Dial(context.Background(), srv.URL, "pi-box", "wrong"); err == nil {
+ t.Fatal("want error on 401 handshake")
+ }
+}
+
+// A cancelled ctx must stop the loop from BOTH of its blocking positions:
+// parked in backoff between dials (no edge at all), and parked in ServeConn on
+// a live channel that is perfectly healthy. The second one is the load-bearing
+// case — http2.Server.ServeConn has no stop signal of its own, so cancellation
+// has to reach it by closing the conn — and it is what lets a test tear the
+// fake host down deterministically instead of leaking a dialer for the rest of
+// the run.
+func TestLoopStopsOnContextCancel(t *testing.T) {
+ t.Run("while backing off", func(t *testing.T) {
+ // A server that is already gone: every dial is refused, so the loop
+ // spends essentially all of its time parked in wait().
+ dead := httptest.NewServer(http.NotFoundHandler())
+ url := dead.URL
+ dead.Close()
+ assertLoopReturns(t, url, nil)
+ })
+ t.Run("while connected", func(t *testing.T) {
+ release := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, bufrw, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ t.Errorf("hijack: %v", err)
+ return
+ }
+ fmt.Fprintf(bufrw, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: carlos-tunnel/1\r\nConnection: Upgrade\r\n\r\n")
+ bufrw.Flush()
+ // Hold the channel open from this side: the edge never hangs up,
+ // so the ONLY thing that can end ServeConn is cancellation
+ // closing the conn — which is the property under test.
+ <-release
+ conn.Close()
+ }))
+ // LIFO: release the fake edge's handler first, then shut it down.
+ defer srv.Close()
+ defer close(release)
+ assertLoopReturns(t, srv.URL, make(chan struct{}, 1))
+ })
+}
+
+// assertLoopReturns runs Loop against url, waits for it to reach the state
+// gate (connected, when connected is non-nil), cancels, and requires a prompt
+// return.
+func assertLoopReturns(t *testing.T, url string, connected chan struct{}) {
+ t.Helper()
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ Loop(ctx, Config{
+ URL: url, Name: "pi-box", Token: "sekrit",
+ Handler: http.NewServeMux(),
+ OnState: func(s string) {
+ if s == StateConnected && connected != nil {
+ select {
+ case connected <- struct{}{}:
+ default:
+ }
+ }
+ },
+ Logf: func(string, ...any) {}, // the failing-dial case is noisy
+ })
+ }()
+ if connected != nil {
+ select {
+ case <-connected:
+ case <-time.After(5 * time.Second):
+ cancel()
+ t.Fatal("loop never connected")
+ }
+ }
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Loop did not return after its context was cancelled")
+ }
+}
diff --git a/statuslive_test.go b/statuslive_test.go
new file mode 100644
index 000000000..028bae66d
--- /dev/null
+++ b/statuslive_test.go
@@ -0,0 +1,127 @@
+package main
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func f64(v float64) *float64 { return &v }
+func i64(v int64) *int64 { return &v }
+
+// TestPrintLiveBoxesSuppressesStaleMetrics is the CLI half of the rule the
+// console page enforces: a host that stopped reporting prints as UNREACHABLE
+// with how long it has been dark, and NONE of its last readings. The operator
+// running this has bucket credentials, so the numbers are theirs to see — the
+// point is that a dead box's numbers are not a current answer to anything.
+func TestPrintLiveBoxesSuppressesStaleMetrics(t *testing.T) {
+ now := time.Now()
+ boxes := []status.Box{
+ {
+ GeneratedAt: now, Label: "ada-stargazing", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: i64(2 << 30), MemAvailBytes: i64(1 << 30),
+ Load1: f64(0.42), DiskFreeBytes: i64(5 << 30),
+ Release: "abc1234", Tunnel: "connected",
+ },
+ },
+ {
+ GeneratedAt: now.Add(-90 * time.Minute), Label: "grace-kayaking", Region: "eu-west-2",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: i64(9 << 30), MemAvailBytes: i64(7 << 30),
+ Load1: f64(8.25), DiskFreeBytes: i64(3 << 30),
+ Release: "beefcaf", Tunnel: "connected",
+ },
+ },
+ }
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, boxes, now)
+ out := buf.String()
+
+ for _, want := range []string{
+ "ada-stargazing", "eu-west-1", "release=abc1234", "tunnel=connected",
+ "mem=1.0GB/2.0GB", "load=0.42", "disk-free=5.0GB",
+ "grace-kayaking", "UNREACHABLE", "1h30m0s",
+ } {
+ if !strings.Contains(out, want) {
+ t.Fatalf("missing %q in:\n%s", want, out)
+ }
+ }
+ for _, leaked := range []string{"beefcaf", "8.25", "7.0GB", "9.0GB", "3.0GB"} {
+ if strings.Contains(out, leaked) {
+ t.Fatalf("stale host's reading %q printed:\n%s", leaked, out)
+ }
+ }
+}
+
+// TestPrintLiveBoxesPreUpgradeHost: a box running code older than the metrics
+// block is fresh and fine — it says so, in the same columns, rather than
+// printing a row of fabricated zeros or dropping out of the listing.
+func TestPrintLiveBoxesPreUpgradeHost(t *testing.T) {
+ now := time.Now()
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "alan-foraging"}, // no metrics, no region
+ {GeneratedAt: now, Label: "iris-sailing", Metrics: &status.BoxMetrics{Tunnel: "reconnecting"}},
+ }, now)
+ out := buf.String()
+ if !strings.Contains(out, "alan-foraging") || !strings.Contains(out, "no metrics block") {
+ t.Fatalf("pre-upgrade host row wrong:\n%s", out)
+ }
+ // A metrics block with only some fields known dashes the rest — never a
+ // zero, the same "unknown is not a number" rule the status package holds.
+ if !strings.Contains(out, "tunnel=reconnecting") || !strings.Contains(out, "mem=-/-") ||
+ !strings.Contains(out, "load=-") || !strings.Contains(out, "disk-free=-") ||
+ !strings.Contains(out, "release=-") {
+ t.Fatalf("partial metrics row wrong:\n%s", out)
+ }
+ for _, line := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+ if !strings.HasPrefix(line, "alan-foraging") && !strings.HasPrefix(line, "iris-sailing") {
+ t.Fatalf("row does not start with its label (alignment broken):\n%s", out)
+ }
+ }
+}
+
+// TestPrintLiveBoxesSortsByLabel: the listing is stable across runs, so a
+// diff between two invocations is a change in the fleet, not in list order.
+func TestPrintLiveBoxesSortsByLabel(t *testing.T) {
+ now := time.Now()
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "zoe-welding"},
+ {GeneratedAt: now, Label: "ada-stargazing"},
+ }, now)
+ if a, z := strings.Index(buf.String(), "ada-"), strings.Index(buf.String(), "zoe-"); a > z {
+ t.Fatalf("rows are not label-sorted:\n%s", buf.String())
+ }
+}
+
+func TestHumanBytesAndFloat(t *testing.T) {
+ for _, c := range []struct {
+ in *int64
+ want string
+ }{
+ {nil, "-"},
+ {i64(0), "0B"},
+ {i64(512), "512B"},
+ {i64(1536), "1.5KB"},
+ {i64(3 << 20), "3.0MB"},
+ {i64(2 << 30), "2.0GB"},
+ } {
+ if got := humanBytes(c.in); got != c.want {
+ t.Errorf("humanBytes(%v) = %q, want %q", c.in, got, c.want)
+ }
+ }
+ if got := humanFloat(nil); got != "-" {
+ t.Errorf("humanFloat(nil) = %q, want -", got)
+ }
+ if got := humanFloat(f64(1.5)); got != "1.50" {
+ t.Errorf("humanFloat(1.5) = %q, want 1.50", got)
+ }
+ if got := orDash(""); got != "-" {
+ t.Errorf("orDash(\"\") = %q, want -", got)
+ }
+}