amadan
edge: tunnel rule keys on how this box served, not the route row
The fleet box that serves a fleet-placed app carries the same fleet/<name> placement on its own row, has no hub, and serves from the socket. Keying the "front box files only its hop" rule on the row's Tunnel/Region made that box skip filing too, so a fleet-placed app's 500s and 503s were filed nowhere (Fable follow-up review). The rule now keys on upstream.viaTunnel, the build-time decision that this box proxies over a reverse channel. TestEdgeServingFleetBoxFilesItsOwn pins the far end (fails under the old rule); spec §11 reworded and the two remaining edges recorded.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+86
−37
docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md+22 −10internal/edge/edge.go+19 −14internal/edge/edgeerrors_test.go+45 −13
CI passed — run details
diff --git a/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md b/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md| index 457218a24..12f9dc8b3 100644 |
| --- a/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md |
| +++ b/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md |
| @@ -909,16 +909,28 @@ timed out at all, so a monitor or visitor giving up first is how those |
| outages end — skipping them would hide exactly the worst ones. |
| **Behind a tunnel, the front box files only the hop (decision, Fable |
| -review).** On a tunnel-backed or fleet-placed route the upstream is not |
| -the app but another box's edge — the agent's, the only tunnel client — |
| -which files what it sees serving the app, and deliberately files nothing |
| -for a wake 503 its own activator answered. A 5xx that merely passed back |
| -through the front box was therefore filed twice, and under this |
| -amendment would have been labelled "the app answered it itself" for an |
| -app that never saw the request. The front box now files only failures it |
| -noted itself (`tunnel`, `fleet_dark`, `abandoned`); everything else is the |
| -serving box's record. This also ends the double filing of tunnelled 500s, |
| -which predates the amendment. |
| +review; corrected in its follow-up).** When a box proxies a request over |
| +a reverse channel, the upstream is not the app but another box's edge — |
| +the agent's, the only tunnel client — which files what it sees serving |
| +the app, and deliberately files nothing for a wake 503 its own activator |
| +answered. A 5xx that merely passed back through the front box was |
| +therefore filed twice, and under this amendment would have been labelled |
| +"the app answered it itself" for an app that never saw the request. The |
| +front box now files only failures it noted itself (`tunnel`, |
| +`fleet_dark`, `abandoned`); everything else is the serving box's record. |
| +This also ends the double filing of tunnelled 500s, which predates the |
| +amendment. |
| + |
| +The rule keys on how THIS box served the request (`upstream.viaTunnel`, |
| +the build-time decision), never on the route row: the fleet box that |
| +serves the app carries the same `fleet/<name>` placement on its own row, |
| +has no hub, and serves from the socket — it must file everything. (The |
| +first cut keyed on the row, and a fleet-placed app's 500s were filed |
| +nowhere; `TestEdgeServingFleetBoxFilesItsOwn` pins the far end.) Two |
| +edges remain: a far agent whose status store failed to open has no error |
| +sink and files nothing, which the front box no longer covers; and a |
| +client that leaves past the grace can be filed `abandoned` on both boxes, |
| +since the stream reset reaches the far box's request too. |
| **The overflow row is per family (decision, from review).** The sink's |
| hourly `_overflow` row copied its source, kind and status from the first |
diff --git a/internal/edge/edge.go b/internal/edge/edge.go| index 3a5cfba56..840f58a85 100644 |
| --- a/internal/edge/edge.go |
| +++ b/internal/edge/edge.go |
| @@ -285,6 +285,12 @@ type upstream struct { |
| // whether its process is live (it's the edge's child), so "dead backend" |
| // detection is the activator's state table, not a failed dial. |
| hibernating bool |
| + // viaTunnel marks a host THIS box proxies over a reverse channel (a |
| + // Tunnel-backed or fleet-placed route with a hub to carry it). It is the |
| + // build-time decision, not the row's columns: the fleet box that serves |
| + // the app carries the same fleet placement on its row and serves it from |
| + // the socket, and must file what it sees (emitEdgeError). |
| + viaTunnel bool |
| // alias marks an entry mapped in from route_aliases: it shares the |
| // primary's proxy/route/hibernating verbatim (so hibernation wakes the |
| // primary's actEntry and #152's per-primary prober state applies), and |
| @@ -384,19 +390,18 @@ func (e *Edge) EnableErrors(s ErrorSink) { |
| // noted; none means the upstream wrote the status itself. A failure whose |
| // client had already gone is not filed at all. |
| // |
| -// On a tunnel-backed or fleet-placed route the upstream is not the app: it |
| -// is another box's edge (the agent's, the only tunnel client), which files |
| -// what it observes serving the app — including deciding NOT to file a wake |
| -// 503 its activator answered (MarkAnswered). So this box files only what it |
| -// noted itself, the hop failing; a status that merely passed through would |
| -// be the same failure filed twice, and a 502/503 labelled "the app answered |
| -// it itself" when the app never saw the request. |
| -func (e *Edge) emitEdgeError(rt registry.Route, r *http.Request, status int, cause string) { |
| +// When this box proxied the request over a reverse channel (viaTunnel), |
| +// the upstream is not the app: it is another box's edge (the agent's, the |
| +// only tunnel client), which files what it observes serving the app — |
| +// including deciding NOT to file a wake 503 its activator answered |
| +// (MarkAnswered). So this box files only what it noted itself, the hop |
| +// failing; a status that merely passed through would be the same failure |
| +// filed twice, and a 502/503 labelled "the app answered it itself" when the |
| +// app never saw the request. The box at the far end serves the same route |
| +// from its socket (viaTunnel false) and files everything as usual. |
| +func (e *Edge) emitEdgeError(rt registry.Route, viaTunnel bool, r *http.Request, status int, cause string) { |
| ref := e.errSink.Load() |
| - if ref == nil || cause == causeClientGone { |
| - return |
| - } |
| - if _, fleetPlaced := registry.FleetPlacement(rt.Region); cause == "" && (rt.Tunnel != "" || fleetPlaced) { |
| + if ref == nil || cause == causeClientGone || (viaTunnel && cause == "") { |
| return |
| } |
| path := apperr.SanitizePath(r.URL.Path) |
| @@ -786,7 +791,7 @@ func (e *Edge) Reload() error { |
| "route %s is a hibernating instance but this box has no activator, so nothing here will create %s or start its process. If this box runs `carlos agent`, set CARLOS_HIBERNATE_BUCKET in /etc/carlos/host.env and restart carlos-edge; `carlos edge` on its own never runs instances.", |
| t.Host, t.Socket)) |
| } |
| - m[t.Host] = upstream{proxy: proxy, route: t, hibernating: hib} |
| + m[t.Host] = upstream{proxy: proxy, route: t, hibernating: hib, viaTunnel: viaTunnel} |
| } |
| // Path routes, folded BEFORE aliases so an alias of a prefix-routed |
| // hostname inherits its map along with everything else it copies — an |
| @@ -1483,7 +1488,7 @@ func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| m.Record(up.route, 1, bytes, hopped) |
| } |
| if ob.status >= 500 && !out.answered { |
| - e.emitEdgeError(up.route, r, ob.status, out.cause) |
| + e.emitEdgeError(up.route, up.viaTunnel, r, ob.status, out.cause) |
| } |
| e.observeAnalytics(up.route, r, ob, bytes) |
| }() |
diff --git a/internal/edge/edgeerrors_test.go b/internal/edge/edgeerrors_test.go| index d7c9b3067..ca6f9764d 100644 |
| --- a/internal/edge/edgeerrors_test.go |
| +++ b/internal/edge/edgeerrors_test.go |
| @@ -283,37 +283,69 @@ func TestProxyCauseClientGone(t *testing.T) { |
| sink := &memErrSink{} |
| e := &Edge{} |
| e.EnableErrors(sink) |
| - e.emitEdgeError(registry.Route{Host: "h", App: "web"}, r, 502, causeClientGone) |
| + e.emitEdgeError(registry.Route{Host: "h", App: "web"}, false, r, 502, causeClientGone) |
| if n := len(sink.snap()); n != 0 { |
| t.Fatalf("a client-gone 502 was filed (%d records)", n) |
| } |
| } |
| -// TestEdgeFilesOnlyItsOwnFailuresThroughATunnel: behind a tunnel or a fleet |
| -// placement the upstream is another box's edge, which files what it saw |
| -// serving the app. This box files the hop failing, never a status that |
| -// merely passed through it. |
| +// TestEdgeFilesOnlyItsOwnFailuresThroughATunnel: when this box proxies over |
| +// a reverse channel, the upstream is another box's edge, which files what |
| +// it saw serving the app. This box files the hop failing, never a status |
| +// that merely passed through it. |
| func TestEdgeFilesOnlyItsOwnFailuresThroughATunnel(t *testing.T) { |
| sink := &memErrSink{} |
| e := &Edge{} |
| e.EnableErrors(sink) |
| r, _ := withOutcome(httptest.NewRequest("GET", "http://x/admin", nil)) |
| - tunneled := registry.Route{Host: "t.example.com", App: "web", Tunnel: "box1"} |
| - fleeted := registry.Route{Host: "f.example.com", App: "web", Region: registry.PlacementTarget("eleven")} |
| - for _, rt := range []registry.Route{tunneled, fleeted} { |
| - e.emitEdgeError(rt, r, 503, "") // the serving box's own 503, passed through |
| - e.emitEdgeError(rt, r, 500, "") // the app's 500, filed on the serving box |
| - } |
| + rt := registry.Route{Host: "t.example.com", App: "web", Tunnel: "box1"} |
| + e.emitEdgeError(rt, true, r, 503, "") // the serving box's own 503, passed through |
| + e.emitEdgeError(rt, true, r, 500, "") // the app's 500, filed on the serving box |
| if n := len(sink.snap()); n != 0 { |
| t.Fatalf("pass-through statuses were filed on the front box: %+v", sink.snap()) |
| } |
| - e.emitEdgeError(tunneled, r, 502, apperr.CauseTunnel) |
| - e.emitEdgeError(fleeted, r, 503, apperr.CauseFleetDark) |
| + e.emitEdgeError(rt, true, r, 502, apperr.CauseTunnel) |
| + e.emitEdgeError(rt, true, r, 503, apperr.CauseFleetDark) |
| if recs := sink.snap(); len(recs) != 2 || recs[0].Cause != apperr.CauseTunnel || recs[1].Cause != apperr.CauseFleetDark { |
| t.Fatalf("the hop's own failures must be filed: %+v", recs) |
| } |
| } |
| +// TestEdgeServingFleetBoxFilesItsOwn is the other end of the same route: |
| +// the fleet box that SERVES a fleet-placed app carries the placement on its |
| +// row too, but has no hub and serves from the app's own address. Everything |
| +// the app answers there must be filed — it is the only box that can. |
| +func TestEdgeServingFleetBoxFilesItsOwn(t *testing.T) { |
| + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if r.URL.Path == "/busy" { |
| + http.Error(w, "busy", http.StatusServiceUnavailable) |
| + return |
| + } |
| + http.Error(w, "boom", http.StatusInternalServerError) |
| + })) |
| + t.Cleanup(backend.Close) |
| + reg := openReg(t) |
| + rt := registry.Route{Host: "f.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web", |
| + Region: registry.PlacementTarget("eleven"), Tunnel: "box1"} |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + e, err := New(reg, nil) // no hub: this box is the far end, not the front |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + sink := &memErrSink{} |
| + e.EnableErrors(sink) |
| + for _, path := range []string{"/boom", "/busy"} { |
| + rec := httptest.NewRecorder() |
| + e.ServeHTTP(rec, httptest.NewRequest("GET", "http://f.example.com"+path, nil)) |
| + } |
| + recs := sink.snap() |
| + if len(recs) != 2 || recs[0].Status != 500 || recs[1].Cause != apperr.CauseUpstream { |
| + t.Fatalf("the serving box must file its app's 500 and 503: %+v", recs) |
| + } |
| +} |
| + |
| // TestEdgeFilesAbandonedWaitEndToEnd drives a real request at an instance |
| // that never answers, through ServeHTTP and the proxy, with a client that |
| // gives up after the grace: the wait is measured from where the edge |