amadan
Availability: edge 502/503/504 grouped by cause, on their own tab
A day of titogo's Errors tab held 29 groups: 25 edge 502s, 3 edge 503s
and one 500. Only the 500 was a failure of what a request asked for; the
rest said the instance was down, restarting or unreachable, and the
per-path fingerprint split each outage into one row per page it hit (17
rows were one scanner probing a dead canary).
- The edge notes WHY it could not get an answer (refused, reset, timeout,
no healthy upstream, fleet dark, tunnel, static, abandoned) through a
per-request outcome on the context, and files 502/503/504 as kind
unavailable, named and fingerprinted by status and cause, never path.
- A client that left within 5s is not filed; one kept waiting longer is
(abandoned) — a dead socket is redialled for 13s and a wedged instance
is never timed out, so that is how the worst outages end.
- Behind a tunnel or fleet placement the front box files only its own
hop failures; the serving box's edge files the rest (ends a double
filing and a false "the app answered it itself").
- New Availability sub-tab, GET /api/cli/apps/{account}/{app}/availability
and `carlos availability`. Platform lifecycle events (warn+) move there;
Errors keeps the app's own 500s and log lines.
- Stored history regroups on read into "<status> · cause not recorded".
- The sink's overflow row is per family.
- Sub-tab pages keep their h2 for screen readers only.
Spec: docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md §11.
Reviewed by Opus and Fable; their confirmed findings are fixed.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
28 files changed,
+1104
−67
apimode.go+13 −3availability_cmd.go+86 −0docs/site/cli.md+25 −2docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md+98 −0errors_cmd.go+1 −1errors_cmd_test.go+21 −0internal/apperr/availability.go+97 −0internal/apperr/availability_test.go+52 −0internal/apperr/record.go+3 −1internal/apperr/sink.go+17 −11internal/apperr/sink_test.go+33 −0internal/applog/sink.go+13 −1internal/console/apierrors.go+41 −0internal/console/apierrors_test.go+103 −11internal/console/authcli.go+1 −0internal/console/errorsview.go+90 −11internal/console/server.go+5 −0internal/console/svcroutes_test.go+7 −0internal/console/templates/app_analytics.html+1 −1internal/console/templates/app_availability.html+61 −0internal/console/templates/app_errors.html+2 −2internal/console/templates/app_logs.html+1 −1internal/console/templates/appnav.html+2 −1internal/edge/answered.go+52 −4internal/edge/edge.go+93 −16internal/edge/edgeerrors_test.go+179 −0internal/edge/static.go+2 −0main.go+5 −1
CI passed — run details
diff --git a/apimode.go b/apimode.go| index d68941126..1e13c746e 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -3277,7 +3277,7 @@ func (s *apiSession) logsGet(ctx context.Context, a apiAcct, app string, query u |
| return res, nil |
| } |
| -// apiErrorsResponse is the console's errors answer (spec |
| +// apiErrorsResponse is the console's errors and availability answer (spec |
| // 2026-09-07-errors-and-analytics-design.md §3.6) — groups and records are |
| // apperr's own wire shapes, decoded as-is. |
| type apiErrorsResponse struct { |
| @@ -3293,7 +3293,17 @@ type apiErrorsResponse struct { |
| // params exactly as the endpoint speaks them; apiPath may already hold a |
| // query (exact=sqid), hence the separator dance. |
| func (s *apiSession) errorsGet(ctx context.Context, a apiAcct, app string, query url.Values) (apiErrorsResponse, error) { |
| - path := apiPath(a, app, "/errors") |
| + return s.errorFamilyGet(ctx, a, app, "errors", query) |
| +} |
| + |
| +// availabilityGet is errorsGet for the Availability half (the console's |
| +// /availability): the same shape, with no log level. |
| +func (s *apiSession) availabilityGet(ctx context.Context, a apiAcct, app string, query url.Values) (apiErrorsResponse, error) { |
| + return s.errorFamilyGet(ctx, a, app, "availability", query) |
| +} |
| + |
| +func (s *apiSession) errorFamilyGet(ctx context.Context, a apiAcct, app, family string, query url.Values) (apiErrorsResponse, error) { |
| + path := apiPath(a, app, "/"+family) |
| if enc := query.Encode(); enc != "" { |
| if strings.Contains(path, "?") { |
| path += "&" + enc |
| @@ -3310,7 +3320,7 @@ func (s *apiSession) errorsGet(ctx context.Context, a apiAcct, app string, query |
| return res, apiErrorFromBody(status, body) |
| } |
| if err := json.Unmarshal(body, &res); err != nil { |
| - return res, fmt.Errorf("errors: unreadable response: %w", err) |
| + return res, fmt.Errorf("%s: unreadable response: %w", family, err) |
| } |
| return res, nil |
| } |
diff --git a/availability_cmd.go b/availability_cmd.go| new file mode 100644 |
| index 000000000..e0baac349 |
| --- /dev/null |
| +++ b/availability_cmd.go |
| @@ -0,0 +1,86 @@ |
| +// `carlos availability` — the requests an app's instances could not answer, |
| +// with no box access (spec 2026-09-07-errors-and-analytics-design.md §3.2, |
| +// amendment 2026-09-24): edge 502/503/504 grouped by cause, and the |
| +// platform's events about the instance's lifecycle, read through the |
| +// console API. The Errors command's sibling; API mode only, like it. |
| +package main |
| + |
| +import ( |
| + "context" |
| + "flag" |
| + "log" |
| + "net/url" |
| + |
| + "github.com/carlosframework/platform/internal/apperr" |
| +) |
| + |
| +const availabilityUsage = `usage: carlos availability --app <a> [flags] |
| + |
| +Lists the requests the app's instances could not answer, newest first: one |
| +row per cause (the instance refused the connection, hung up, timed out, no |
| +fleet box connected, the app answered 503 itself...) with its count in the |
| +window and the hosts and releases it was seen on, plus the platform's own |
| +events about the instance (failed wakes, exits). A 502 on any page of a |
| +dead instance is one row, not one per page. --fp <id> expands one cause to |
| +its occurrences, each with the path that was being asked for. |
| +` |
| + |
| +func runAvailability(args []string) { |
| + fs := flag.NewFlagSet("availability", flag.ExitOnError) |
| + app := appFlag(fs, "app whose availability to read") |
| + since := fs.String("since", "24h", "window start: duration back from now, or RFC3339") |
| + until := fs.String("until", "", "window end (default: now)") |
| + host := fs.String("host", "", "only one instance (route host)") |
| + rel := fs.String("release", "", "only one release") |
| + fp := fs.String("fp", "", "expand one cause: its occurrences, oldest first") |
| + consoleFlag := consoleOption(fs, "console to read through (default: the only logged-in one)") |
| + acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)") |
| + acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account") |
| + fs.Usage = flagUsage(fs, availabilityUsage) |
| + fs.Parse(args) |
| + |
| + sess := requireAPISession("availability", *consoleFlag) |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + app2 := resolvedApp(sess, acct, *app, availabilityUsage) |
| + |
| + q := url.Values{} |
| + for k, v := range map[string]string{"since": *since, "until": *until, "host": *host, "release": *rel, "fp": *fp} { |
| + if v != "" { |
| + q.Set(k, v) |
| + } |
| + } |
| + res, err := sess.availabilityGet(context.Background(), acct, app2, q) |
| + if err != nil { |
| + log.Fatalf("availability: %v", err) |
| + } |
| + if *fp != "" { |
| + printErrorRecords(stdout, res.Records) |
| + if len(res.Records) == 0 { |
| + mutedf(stdout, "no occurrences of %s in this window", *fp) |
| + } |
| + return |
| + } |
| + printErrorGroups(stdout, platformSourced(res.Groups)) |
| + if len(res.Groups) == 0 { |
| + mutedf(stdout, "no availability problems in this window — try --since 168h") |
| + } else { |
| + mutedf(stdout, "counts are to the nearest flush; the app's own 500s are under `carlos errors`") |
| + } |
| + if res.Truncated { |
| + mutedf(stdout, "(the window held more than could be read; counts are within what was read — narrow it)") |
| + } |
| +} |
| + |
| +// platformSourced names a log-derived group by what it is here: every log |
| +// line Availability shows is a platform event, and "log" would read as the |
| +// app's own output. |
| +func platformSourced(groups []apperr.Group) []apperr.Group { |
| + out := make([]apperr.Group, len(groups)) |
| + for i, g := range groups { |
| + if g.Source == apperr.SourceLog { |
| + g.Source = "platform" |
| + } |
| + out[i] = g |
| + } |
| + return out |
| +} |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index f8428bf1e..cedeb32f0 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -820,8 +820,10 @@ narrows to one instance. |
| One row per distinct error, newest first, with how many times it happened |
| in the window and where. Two sources need no wiring at all: log lines the |
| -app tagged `error` (or `warn`, if you set the level) and responses of 500 |
| -or above that the edge saw. Server-side and browser hooks come later. |
| +app tagged `error` (or `warn`, if you set the level) and 500s the app |
| +answered through the edge. Server-side and browser hooks come later. A 502, |
| +503 or 504 is not here: it says the instance could not answer, not that the |
| +request failed, and `carlos availability` has it. |
| ```sh |
| carlos errors --app hello --since 24h |
| @@ -833,6 +835,27 @@ carlos errors --app hello --set-level warn |
| read without changing the setting; `--set-level` changes it for the app. |
| The same view is the Errors tab on the app's dashboard. |
| +### carlos availability |
| + |
| +The requests an app's instances could not answer — every 502, 503 and 504 |
| +the edge saw — grouped by cause rather than by page: the instance refused |
| +the connection (down, or mid-restart), closed it mid-request, timed out, no |
| +box of the fleet was connected, or the app answered 503 itself. One outage |
| +is one row however many pages it hit, and a scanner probing forty paths of |
| +a dead instance is one row too. The platform's own events about the |
| +instance — failed wakes and exits — are listed beside them. |
| + |
| +```sh |
| +carlos availability --app hello --since 24h |
| +carlos availability --app hello --fp 3f2a9c1b5e7d0a4c |
| +``` |
| + |
| +`--fp` expands one cause to its occurrences, each with the path it was |
| +asking for. A request whose visitor left within five seconds is not |
| +counted; one who gave up after waiting longer is, as an outage. |
| +Records the edge filed before causes were recorded are grouped under |
| +"cause not recorded". The same view is the Availability tab beside Errors. |
| + |
| ### carlos analytics |
| Page views, daily visitors, bytes served, top pages, top referers and named |
diff --git a/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md b/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md| index 30d307ecf..457218a24 100644 |
| --- a/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md |
| +++ b/docs/superpowers/specs/2026-09-07-errors-and-analytics-design.md |
| @@ -841,3 +841,101 @@ Misra–Gries tables are labelled approximate above 500 keys; the |
| wake failures are suppressed at the writer when the activator already |
| emitted; tombstones have a scope, an authorisation, and a stated |
| non-effect on Logs. Nothing from round 2 was declined. |
| + |
| +## 11. Amendment 2026-09-24: Availability |
| + |
| +**Why.** A day of the flagship's Errors tab (titogo, 2026-09-23) held 29 |
| +groups: 25 edge 502s, 3 edge 503s and one 500. Only the 500 was a failure |
| +of what a request asked for. The rest said the instance was down, |
| +restarting or unreachable when some page was asked for, and §3.2's |
| +`"<status> <method> <path>"` fingerprint split each outage into one row |
| +per page that happened to hit it. 17 of the 29 rows were one scanner |
| +probing `/.env`, `/phpinfo.php` and so on against a dead canary. The |
| +record also could not say *why* the edge had no answer: the proxies |
| +logged the error and threw it away. |
| + |
| +**Availability is the Errors tab's sibling (decision, Bill 2026-09-24).** |
| +A sub-tab between Errors and Analytics (`GET /a/{sqid}/apps/{app}/availability`), |
| +an API path beside `/errors` with the same gate |
| +(`GET /api/cli/apps/{account}/{app}/availability`), and `carlos |
| +availability`. It holds: |
| + |
| +- every edge-observed 502, 503 and 504, whoever wrote it; and |
| +- the platform stream's events at `warn` and above (failed wakes, |
| + exits, a sidecar that will not restart), whatever `errors.log_level` |
| + says, since an exit on a signal is filed at `warn` and is exactly the |
| + event an outage row needs beside it. The synthetic rate-cap line |
| + (`applog.IsDropNotice`) is about the logging budget and is left out. |
| + |
| +Errors keeps the rest: the app's own 500, 501 and 505+, and the `log` |
| +source's app-stream lines. Nothing is in both. |
| + |
| +**Grouped by cause, never by path (decision).** An edge 502/503/504 is |
| +filed as `kind: unavailable` with `cause` and `name: "<status> · <cause |
| +in the member's terms>"`, and so fingerprinted by status and cause alone; |
| +the path stays on the record as the occurrence's sample. The cause comes |
| +from the ErrorHandler that wrote the status, through a per-request |
| +`outcome` on the context (`edge/answered.go`, which already carried the |
| +activator's answered flag and now rides every observed request, not only |
| +hibernating ones): |
| + |
| +| cause | where | |
| +|---------------|--------------------------------------------------------------| |
| +| `refused` | ECONNREFUSED / ENOENT on the dial: nothing listening | |
| +| `reset` | EOF, unexpected EOF, ECONNRESET, EPIPE: hung up mid-request | |
| +| `timeout` | a deadline or a net timeout | |
| +| `abandoned` | the client left after waiting 5 s or more (below) | |
| +| `no_upstream` | `multiProxy`: every upstream failing its probe | |
| +| `fleet_dark` | `fleetProxy`: no box of the fleet connected | |
| +| `tunnel` | any other failure through a fleet box's tunnel | |
| +| `static` | a static site's files could not be resolved | |
| +| `proxy` | any other proxy failure | |
| +| `upstream` | no handler noted one: the app wrote the status itself (direct routes only; below) | |
| + |
| +`unixProxy`, `tcpProxy` and `tunnelProxy` had no ErrorHandler; they get |
| +one that is `httputil.ReverseProxy`'s default (same log line, same bare |
| +502) plus the note. |
| + |
| +**A visitor who left quickly is not an outage; one kept waiting is |
| +(decision, amended in review).** When the client's own context is done by |
| +the time a proxy fails, `ReverseProxy` still writes a 502, to nobody, and |
| +the error is the cancellation whatever the instance was doing. Within |
| +5 s of the request (`clientGoneGrace`) the edge notes `client_gone` and |
| +files nothing: a visitor navigating away mid-load is not the instance's |
| +fault, and a single-count 502 on a font file is the typical shape of it. |
| +Past 5 s it files `abandoned`. The review caught why that half matters: |
| +a dead unix socket is redialled for 13 s and a wedged instance is never |
| +timed out at all, so a monitor or visitor giving up first is how those |
| +outages end — skipping them would hide exactly the worst ones. |
| + |
| +**Behind a tunnel, the front box files only the hop (decision, Fable |
| +review).** On a tunnel-backed or fleet-placed route the upstream is not |
| +the app but another box's edge — the agent's, the only tunnel client — |
| +which files what it sees serving the app, and deliberately files nothing |
| +for a wake 503 its own activator answered. A 5xx that merely passed back |
| +through the front box was therefore filed twice, and under this |
| +amendment would have been labelled "the app answered it itself" for an |
| +app that never saw the request. The front box now files only failures it |
| +noted itself (`tunnel`, `fleet_dark`, `abandoned`); everything else is the |
| +serving box's record. This also ends the double filing of tunnelled 500s, |
| +which predates the amendment. |
| + |
| +**The overflow row is per family (decision, from review).** The sink's |
| +hourly `_overflow` row copied its source, kind and status from the first |
| +record that overflowed, so one family's excess could land on the other |
| +tab. It is now one row per family (`IsAvailability`), and |
| +`AsAvailability` leaves an `_overflow` row alone. |
| + |
| +**History regroups on read (decision).** Records filed before this carry |
| +`kind: http`, no `cause`, and the per-path fingerprint. The read |
| +recognises them by status and regroups them as `"<status> · cause not |
| +recorded"` (`apperr.AsAvailability`); the FP filter applies after the |
| +regroup so the row the tab shows expands. The bucket is not rewritten. |
| + |
| +**Not changed.** The wire shape is additive (`cause` and a read-only |
| +`stream` on `apperr.Record`, both `omitempty`); nothing under `logs/` or |
| +`errors/` moves. The sending half of a backhaul still files nothing when |
| +its hop fails: it has no route row and so no app to file under, the |
| +same reason it bills nothing. Correlating a row with the nearest deploy or restart |
| +on that host, and alerting on a new cause, are the natural next steps |
| +and are not built here. |
diff --git a/errors_cmd.go b/errors_cmd.go| index 3b8991c08..a7baf1435 100644 |
| --- a/errors_cmd.go |
| +++ b/errors_cmd.go |
| @@ -107,7 +107,7 @@ func printErrorGroups(w io.Writer, groups []apperr.Group) { |
| count += " (overflow)" |
| } |
| line := c.muted(g.Last.UTC().Format("2006-01-02 15:04")) + " " + c.head(fmt.Sprintf("%6s", count)) + " " + |
| - c.muted(fmt.Sprintf("%-7s", g.Source)) + " " + c.bad(g.Name) |
| + c.muted(fmt.Sprintf("%-8s", g.Source)) + " " + c.bad(g.Name) |
| line += "\n" + " " + c.muted("fp="+g.FP) |
| if len(g.Hosts) > 0 { |
| line += c.muted(" hosts=" + joinOr(g.Hosts, "")) |
diff --git a/errors_cmd_test.go b/errors_cmd_test.go| index d4c44aa6d..dd2cef188 100644 |
| --- a/errors_cmd_test.go |
| +++ b/errors_cmd_test.go |
| @@ -23,3 +23,24 @@ func TestPrintErrorGroups(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// TestAvailabilityNamesPlatformEvents: a log-derived row on Availability is |
| +// a platform event, and prints as one rather than as the app's own "log". |
| +func TestAvailabilityNamesPlatformEvents(t *testing.T) { |
| + now := time.Date(2026, 9, 24, 10, 0, 0, 0, time.UTC) |
| + in := []apperr.Group{ |
| + {FP: "a", Source: apperr.SourceLog, Name: "wake failed: no head", Count: 1, Last: now}, |
| + {FP: "b", Source: apperr.SourceEdge, Name: apperr.AvailabilityName(502, apperr.CauseRefused), Count: 4, Last: now}, |
| + } |
| + var buf bytes.Buffer |
| + printErrorGroups(&buf, platformSourced(in)) |
| + out := buf.String() |
| + for _, want := range []string{"platform wake failed", "edge 502 · the instance refused the connection"} { |
| + if !strings.Contains(out, want) { |
| + t.Errorf("output missing %q:\n%s", want, out) |
| + } |
| + } |
| + if in[0].Source != apperr.SourceLog { |
| + t.Fatal("platformSourced must not rewrite its input") |
| + } |
| +} |
diff --git a/internal/apperr/availability.go b/internal/apperr/availability.go| new file mode 100644 |
| index 000000000..e6399b2e1 |
| --- /dev/null |
| +++ b/internal/apperr/availability.go |
| @@ -0,0 +1,97 @@ |
| +package apperr |
| + |
| +import ( |
| + "strconv" |
| + |
| + "github.com/carlosframework/platform/internal/applog" |
| +) |
| + |
| +// Availability is the Errors tab's sibling (spec 2026-09-07 §3.2, |
| +// amendment 2026-09-24): the failures that are about whether an instance |
| +// could answer at all, not about what the request asked it to do. A 502 |
| +// on /admin/sandbox says nothing about the sandbox page; it says the |
| +// instance was down, restarting or unreachable when that page was asked |
| +// for. So these group by CAUSE, never by path: one outage is one row, and |
| +// a scanner probing forty paths of a dead instance is one row, not forty. |
| + |
| +// KindUnavailable marks an edge record filed under Availability. Records |
| +// written before the amendment carry KindHTTP and no Cause; IsAvailability |
| +// still recognises them by status, and AsAvailability regroups them. |
| +const KindUnavailable = "unavailable" |
| + |
| +// Causes the edge records for a 502/503/504, from the proxy error that |
| +// produced it (edge.proxyCause). The vocabulary is closed: an unknown |
| +// value on read renders as itself, never guessed into one of these. |
| +const ( |
| + CauseRefused = "refused" // nothing listening: the instance is down or mid-restart |
| + CauseReset = "reset" // the instance hung up mid-request |
| + CauseTimeout = "timeout" // the instance did not answer in time |
| + CauseAbandoned = "abandoned" // the visitor gave up after waiting long enough that the wait was the problem |
| + CauseNoUpstream = "no_upstream" // every upstream of a multi-upstream route is failing its probe |
| + CauseFleetDark = "fleet_dark" // no box of the route's fleet is connected |
| + CauseTunnel = "tunnel" // the tunnel to the fleet box failed |
| + CauseStatic = "static" // a static site's files could not be resolved |
| + CauseProxy = "proxy" // any other proxy failure |
| + CauseUpstream = "upstream" // the app itself answered 502/503/504 |
| +) |
| + |
| +var causeText = map[string]string{ |
| + CauseRefused: "the instance refused the connection", |
| + CauseReset: "the instance closed the connection mid-request", |
| + CauseTimeout: "the instance did not answer in time", |
| + CauseAbandoned: "the visitor gave up waiting for the instance", |
| + CauseNoUpstream: "no healthy upstream", |
| + CauseFleetDark: "no box of the fleet is connected", |
| + CauseTunnel: "the tunnel to the fleet box failed", |
| + CauseStatic: "the static site could not be read", |
| + CauseProxy: "the proxy could not complete the request", |
| + CauseUpstream: "the app answered it itself", |
| + "": "cause not recorded", |
| +} |
| + |
| +// UnavailableStatus reports whether an edge-observed status belongs under |
| +// Availability rather than Errors. 500, 501 and 505+ stay errors: the app |
| +// answered and what it answered was a failure of the request. |
| +func UnavailableStatus(status int) bool { return status >= 502 && status <= 504 } |
| + |
| +// AvailabilityName is the row name for one (status, cause): the status and |
| +// the cause in the member's terms, and deliberately no path. |
| +func AvailabilityName(status int, cause string) string { |
| + text, ok := causeText[cause] |
| + if !ok { |
| + text = cause |
| + } |
| + return strconv.Itoa(status) + " · " + text |
| +} |
| + |
| +// IsAvailability reports whether r belongs under Availability: an edge |
| +// 502/503/504 (whichever version of the edge wrote it), or a platform |
| +// event about the instance's own lifecycle (a wake failure, a crash). |
| +func IsAvailability(r Record) bool { |
| + switch r.Source { |
| + case SourceEdge: |
| + return r.Kind == KindUnavailable || (r.Kind == KindHTTP && UnavailableStatus(r.Status)) |
| + case SourceLog: |
| + return r.Stream == applog.StreamPlatform |
| + } |
| + return false |
| +} |
| + |
| +// AsAvailability regroups a record written before the amendment — an edge |
| +// 502/503/504 named "<status> <method> <path>" and fingerprinted by it — |
| +// under its status and an unrecorded cause, so the history in the bucket |
| +// collapses the way new records do instead of staying one row per path. |
| +// Every other record is returned unchanged. |
| +// |
| +// The overflow row is left alone: it is a count of whatever exceeded the |
| +// hourly cap, not an error of its own, and keeps its FP and its label. |
| +func AsAvailability(r Record) Record { |
| + if r.FP == OverflowFP || r.Source != SourceEdge || r.Kind != KindHTTP || !UnavailableStatus(r.Status) { |
| + return r |
| + } |
| + r.Kind = KindUnavailable |
| + r.Name = AvailabilityName(r.Status, r.Cause) |
| + r.FP = Fingerprint(SourceEdge, KindUnavailable, r.Name, "") |
| + r.FPV = FPVersion |
| + return r |
| +} |
diff --git a/internal/apperr/availability_test.go b/internal/apperr/availability_test.go| new file mode 100644 |
| index 000000000..b5ba5a46a |
| --- /dev/null |
| +++ b/internal/apperr/availability_test.go |
| @@ -0,0 +1,52 @@ |
| +package apperr |
| + |
| +import ( |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/applog" |
| +) |
| + |
| +func TestIsAvailability(t *testing.T) { |
| + for _, c := range []struct { |
| + r Record |
| + want bool |
| + }{ |
| + {Record{Source: SourceEdge, Kind: KindUnavailable, Status: 502}, true}, |
| + {Record{Source: SourceEdge, Kind: KindHTTP, Status: 503}, true}, // written before the amendment |
| + {Record{Source: SourceEdge, Kind: KindHTTP, Status: 504}, true}, |
| + {Record{Source: SourceEdge, Kind: KindHTTP, Status: 500}, false}, // the app's own failure |
| + {Record{Source: SourceEdge, Kind: KindHTTP, Status: 505}, false}, |
| + {Record{Source: SourceLog, Stream: applog.StreamPlatform, Level: "error"}, true}, |
| + {Record{Source: SourceLog, Stream: applog.StreamApp, Level: "error"}, false}, |
| + {Record{Source: SourceServer, Status: 503}, false}, |
| + } { |
| + if got := IsAvailability(c.r); got != c.want { |
| + t.Errorf("IsAvailability(%+v) = %v, want %v", c.r, got, c.want) |
| + } |
| + } |
| +} |
| + |
| +// TestAsAvailabilityRegroupsHistory: records the edge filed before the |
| +// amendment, one per path, collapse into one row per status. |
| +func TestAsAvailabilityRegroupsHistory(t *testing.T) { |
| + a := AsAvailability(Record{Source: SourceEdge, Kind: KindHTTP, Status: 502, Name: "502 GET /.env", Path: "/.env", FP: "old1"}) |
| + b := AsAvailability(Record{Source: SourceEdge, Kind: KindHTTP, Status: 502, Name: "502 GET /phpinfo.php", Path: "/phpinfo.php", FP: "old2"}) |
| + c := AsAvailability(Record{Source: SourceEdge, Kind: KindHTTP, Status: 503, Name: "503 GET /.env", Path: "/.env", FP: "old3"}) |
| + if a.FP != b.FP || a.FP == c.FP { |
| + t.Fatalf("fps = %s %s %s: want one per status", a.FP, b.FP, c.FP) |
| + } |
| + if a.Name != "502 · cause not recorded" || a.Kind != KindUnavailable || a.Path != "/.env" { |
| + t.Fatalf("regrouped = %+v", a) |
| + } |
| + // A new record is already in shape; a 500 is not Availability at all. |
| + fresh := Record{Source: SourceEdge, Kind: KindUnavailable, Status: 502, Cause: CauseRefused, Name: AvailabilityName(502, CauseRefused), FP: "keep"} |
| + if got := AsAvailability(fresh); got.FP != "keep" { |
| + t.Fatalf("a new record was regrouped: %+v", got) |
| + } |
| + if got := AsAvailability(Record{Source: SourceEdge, Kind: KindHTTP, Status: 500, FP: "keep"}); got.FP != "keep" { |
| + t.Fatalf("a 500 was regrouped: %+v", got) |
| + } |
| + if got := AvailabilityName(503, "martian"); got != "503 · martian" { |
| + t.Fatalf("an unknown cause renders as itself, got %q", got) |
| + } |
| +} |
diff --git a/internal/apperr/record.go b/internal/apperr/record.go| index 5dcfb19cd..8187d301f 100644 |
| --- a/internal/apperr/record.go |
| +++ b/internal/apperr/record.go |
| @@ -48,6 +48,8 @@ type Record struct { |
| Level string `json:"level,omitempty"` // log source: warn | error |
| Path string `json:"path,omitempty"` |
| Status int `json:"status,omitempty"` |
| + Cause string `json:"cause,omitempty"` // Availability: why the edge could not get an answer (availability.go) |
| + Stream string `json:"stream,omitempty"` // log source only: the applog stream, derived on read and never stored |
| Release string `json:"release,omitempty"` |
| Stack string `json:"stack,omitempty"` |
| Fields map[string]string `json:"fields,omitempty"` |
| @@ -69,7 +71,7 @@ func FromLog(r applog.Record) Record { |
| name := Sanitize(r.Msg) |
| return Record{ |
| TS: r.TS, Box: r.Box, Host: r.Host, |
| - Source: SourceLog, Kind: KindLog, Name: name, Level: r.Level, |
| + Source: SourceLog, Kind: KindLog, Name: name, Level: r.Level, Stream: r.Stream, |
| Fields: r.Fields, |
| FP: Fingerprint(SourceLog, KindLog, name, ""), FPV: FPVersion, |
| Count: 1, First: r.TS, Last: r.TS, Seq: r.Seq, |
diff --git a/internal/apperr/sink.go b/internal/apperr/sink.go| index 55e80a85e..d3c95b6ba 100644 |
| --- a/internal/apperr/sink.go |
| +++ b/internal/apperr/sink.go |
| @@ -37,8 +37,8 @@ type groupKey struct{ fp, host, release string } |
| type appBuf struct { |
| groups map[groupKey]*Record |
| - order []groupKey // insertion order, for a deterministic drain |
| - overflow *Record |
| + order []groupKey // insertion order, for a deterministic drain |
| + overflow [2]*Record // one per family, indexed by IsAvailability: Errors' excess never lands on Availability's row, nor the reverse |
| hour string // the UTC hour `seen` counts for |
| seen map[groupKey]bool // keys admitted this hour, across every drain — the cap counter; Drain must not touch it |
| } |
| @@ -95,17 +95,21 @@ func (s *Sink) Add(acct, app string, r Record) { |
| return |
| } |
| if !b.seen[k] && len(b.seen) >= MaxGroupsPerAppHour { |
| - if b.overflow == nil { |
| + fam := 0 |
| + if IsAvailability(r) { |
| + fam = 1 |
| + } |
| + if b.overflow[fam] == nil { |
| o := r |
| o.FP = OverflowFP |
| o.TS, o.First, o.Last, o.Count = now, now, now, 0 |
| o.Box = s.box |
| s.seq++ |
| o.Seq = s.seq |
| - b.overflow = &o |
| + b.overflow[fam] = &o |
| } |
| - b.overflow.Count++ |
| - b.overflow.Last = now |
| + b.overflow[fam].Count++ |
| + b.overflow[fam].Last = now |
| return |
| } |
| b.seen[k] = true |
| @@ -127,20 +131,22 @@ func (s *Sink) Drain() map[applog.AppKey][]Record { |
| defer s.mu.Unlock() |
| out := make(map[applog.AppKey][]Record) |
| for k, b := range s.apps { |
| - if len(b.order) == 0 && b.overflow == nil { |
| + if len(b.order) == 0 && b.overflow == [2]*Record{} { |
| continue |
| } |
| - recs := make([]Record, 0, len(b.order)+1) |
| + recs := make([]Record, 0, len(b.order)+2) |
| for _, gk := range b.order { |
| recs = append(recs, *b.groups[gk]) |
| } |
| - if b.overflow != nil { |
| - recs = append(recs, *b.overflow) |
| + for _, o := range b.overflow { |
| + if o != nil { |
| + recs = append(recs, *o) |
| + } |
| } |
| out[k] = recs |
| b.groups = make(map[groupKey]*Record) |
| b.order = nil |
| - b.overflow = nil |
| + b.overflow = [2]*Record{} |
| } |
| return out |
| } |
diff --git a/internal/apperr/sink_test.go b/internal/apperr/sink_test.go| index 8fa9a3bcf..b523dbc42 100644 |
| --- a/internal/apperr/sink_test.go |
| +++ b/internal/apperr/sink_test.go |
| @@ -2,6 +2,7 @@ package apperr |
| import ( |
| "fmt" |
| + "strconv" |
| "testing" |
| "time" |
| @@ -170,3 +171,35 @@ func TestSinkCapCountsDistinctKeysAcrossDrains(t *testing.T) { |
| t.Fatalf("next hour = %+v", recs) |
| } |
| } |
| + |
| +// TestSinkOverflowPerFamily: past the hourly cap, an app's own errors and |
| +// its availability failures overflow into separate rows, so neither tab's |
| +// excess is counted on the other's. |
| +func TestSinkOverflowPerFamily(t *testing.T) { |
| + now := time.Date(2026, 9, 24, 10, 0, 0, 0, time.UTC) |
| + s := NewSink("box1", func() time.Time { return now }) |
| + for i := 0; i < MaxGroupsPerAppHour; i++ { |
| + s.Add("bab", "web", Record{Source: SourceEdge, Kind: KindHTTP, Name: "500 GET /p" + strconv.Itoa(i), Status: 500, Host: "h1"}) |
| + } |
| + s.Add("bab", "web", Record{Source: SourceEdge, Kind: KindHTTP, Name: "500 GET /x", Status: 500, Host: "h1"}) |
| + for i := 0; i < 3; i++ { |
| + s.Add("bab", "web", Record{Source: SourceEdge, Kind: KindUnavailable, Cause: CauseRefused, Name: AvailabilityName(502, CauseRefused) + strconv.Itoa(i), Status: 502, Host: "h1"}) |
| + } |
| + var errs, avail int |
| + for _, r := range s.Drain()[applog.AppKey{Acct: "bab", App: "web"}] { |
| + if r.FP != OverflowFP { |
| + continue |
| + } |
| + if IsAvailability(r) { |
| + avail += r.Count |
| + } else { |
| + errs += r.Count |
| + } |
| + } |
| + if errs != 1 || avail != 3 { |
| + t.Fatalf("overflow errors=%d availability=%d, want 1 and 3", errs, avail) |
| + } |
| + if got := AsAvailability(Record{Source: SourceEdge, Kind: KindHTTP, Status: 502, FP: OverflowFP}); got.FP != OverflowFP { |
| + t.Fatalf("the overflow row was regrouped: %+v", got) |
| + } |
| +} |
diff --git a/internal/applog/sink.go b/internal/applog/sink.go| index 309b46d8a..6f1f68d78 100644 |
| --- a/internal/applog/sink.go |
| +++ b/internal/applog/sink.go |
| @@ -2,6 +2,7 @@ package applog |
| import ( |
| "fmt" |
| + "strings" |
| "sync" |
| "time" |
| ) |
| @@ -125,6 +126,17 @@ func (s *Sink) Event(acct, app, host, level, msg string) { |
| } |
| } |
| +// dropNoticeSuffix ends the synthetic line Drain appends for an app whose |
| +// lines hit the rate cap. |
| +const dropNoticeSuffix = " (rate cap)" |
| + |
| +// IsDropNotice reports whether r is that synthetic line: a platform event |
| +// about the logging budget, not about the instance, so Availability (which |
| +// takes the rest of the platform stream) leaves it out. |
| +func IsDropNotice(r Record) bool { |
| + return r.Stream == StreamPlatform && strings.HasPrefix(r.Msg, "dropped ") && strings.HasSuffix(r.Msg, dropNoticeSuffix) |
| +} |
| + |
| // Drain returns everything buffered, keyed by app, and resets the buffers |
| // (sequence numbers and rate-bucket state persist — they are the process's, |
| // not the batch's). An app whose lines were dropped since the last drain |
| @@ -139,7 +151,7 @@ func (s *Sink) Drain() map[AppKey][]Record { |
| s.append(b, Record{ |
| Stream: StreamPlatform, |
| Level: LevelWarn, |
| - Msg: fmt.Sprintf("dropped %d app log line(s) (rate cap)", b.dropped), |
| + Msg: fmt.Sprintf("dropped %d app log line(s)"+dropNoticeSuffix, b.dropped), |
| }) |
| b.dropped = 0 |
| } |
diff --git a/internal/console/apierrors.go b/internal/console/apierrors.go| index 150159381..85e038b30 100644 |
| --- a/internal/console/apierrors.go |
| +++ b/internal/console/apierrors.go |
| @@ -57,6 +57,47 @@ func (s *Server) handleAPIErrors(w http.ResponseWriter, r *http.Request) { |
| writeJSON(w, http.StatusOK, resp) |
| } |
| +// availabilityResponse is the wire shape `carlos availability` reads: the |
| +// errors shape without the log level, which Availability does not use. |
| +type availabilityResponse struct { |
| + Groups []apperr.Group `json:"groups"` |
| + Records []apperr.Record `json:"records,omitempty"` |
| + Truncated bool `json:"truncated"` |
| + Sources []string `json:"sources"` |
| +} |
| + |
| +// handleAPIAvailability answers `carlos availability`: the errors API's |
| +// gate and filter, the other half of the records (errView). source and |
| +// level parse as they do for errors but do not narrow this read. |
| +func (s *Server) handleAPIAvailability(w http.ResponseWriter, r *http.Request) { |
| + _, acct, app, ok := s.instanceGate(w, r) |
| + if !ok { |
| + return |
| + } |
| + f, _, bad := errorsFilterFromQuery(r.URL.Query(), time.Now()) |
| + if bad != "" { |
| + jsonError(w, http.StatusBadRequest, bad) |
| + return |
| + } |
| + f.Source = "" |
| + ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| + defer cancel() |
| + win, err := s.readAvailability(ctx, acct.Sqid, app, f) |
| + if err != nil { |
| + log.Printf("console: api availability %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "could not read availability") |
| + return |
| + } |
| + resp := availabilityResponse{Groups: win.Groups, Records: win.Records, Truncated: win.Truncated, Sources: win.Sources} |
| + if resp.Groups == nil { |
| + resp.Groups = []apperr.Group{} |
| + } |
| + if resp.Sources == nil { |
| + resp.Sources = []string{} |
| + } |
| + writeJSON(w, http.StatusOK, resp) |
| +} |
| + |
| // handleAPITelemetryPut sets the app-level telemetry object: the errors log |
| // level, the analytics keys, or both in one call. Operate only — an app's |
| // own token must not widen what its owner sees. |
diff --git a/internal/console/apierrors_test.go b/internal/console/apierrors_test.go| index 04e181c39..dec272a99 100644 |
| --- a/internal/console/apierrors_test.go |
| +++ b/internal/console/apierrors_test.go |
| @@ -25,19 +25,33 @@ func errorsURL(fx *consoleFixture, acct, app, query string) string { |
| } |
| // seedErrors ships one coalesced edge error and, via seedLogChunks, one |
| -// error-level platform line and one warn-level app line. |
| +// error-level platform line and one warn-level app line — plus, for |
| +// Availability, a warn-level platform exit, two 502s filed before the |
| +// amendment (one row per path) and one filed after it (by cause). |
| func seedErrors(t *testing.T, fx *consoleFixture, now time.Time) { |
| t.Helper() |
| seedLogChunks(t, fx, now) |
| warn := []applog.Record{{TS: now.Add(-7 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| - Stream: applog.StreamApp, Level: applog.LevelWarn, Msg: "slow query 4123ms", Seq: 150}} |
| + Stream: applog.StreamApp, Level: applog.LevelWarn, Msg: "slow query 4123ms", Seq: 150}, |
| + {TS: now.Add(-6 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| + Stream: applog.StreamPlatform, Level: applog.LevelWarn, Msg: "instance exited: signal: killed", Seq: 151}} |
| body, _ := applog.EncodeChunk(warn) |
| if _, err := fx.rel.Put(context.Background(), applog.ChunkKey(release.OpsAccount, "titogo", "box1", applog.HourOf(warn[0].TS), 150, len(body)), body, ""); err != nil { |
| t.Fatal(err) |
| } |
| e := []apperr.Record{{TS: now.Add(-3 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| Source: apperr.SourceEdge, Kind: apperr.KindHTTP, Name: "500 POST /checkout", Path: "/checkout", Status: 500, |
| - Release: "v3", FP: "edge500", FPV: 1, Count: 12, First: now.Add(-9 * time.Minute), Last: now.Add(-3 * time.Minute), Seq: 200}} |
| + Release: "v3", FP: "edge500", FPV: 1, Count: 12, First: now.Add(-9 * time.Minute), Last: now.Add(-3 * time.Minute), Seq: 200}, |
| + {TS: now.Add(-8 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| + Source: apperr.SourceEdge, Kind: apperr.KindHTTP, Name: "502 GET /.env", Path: "/.env", Status: 502, |
| + Release: "v3", FP: "old502env", FPV: 1, Count: 1, First: now.Add(-8 * time.Minute), Last: now.Add(-8 * time.Minute), Seq: 201}, |
| + {TS: now.Add(-8 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| + Source: apperr.SourceEdge, Kind: apperr.KindHTTP, Name: "502 GET /phpinfo.php", Path: "/phpinfo.php", Status: 502, |
| + Release: "v3", FP: "old502php", FPV: 1, Count: 1, First: now.Add(-8 * time.Minute), Last: now.Add(-8 * time.Minute), Seq: 202}, |
| + {TS: now.Add(-2 * time.Minute), Box: "box1", Host: "titogo.example.com", |
| + Source: apperr.SourceEdge, Kind: apperr.KindUnavailable, Cause: apperr.CauseRefused, |
| + Name: apperr.AvailabilityName(502, apperr.CauseRefused), Path: "/admin", Status: 502, |
| + Release: "v4", FP: "new502", FPV: 1, Count: 5, First: now.Add(-4 * time.Minute), Last: now.Add(-2 * time.Minute), Seq: 203}} |
| eb, _ := apperr.EncodeChunk(e) |
| if _, err := fx.rel.Put(context.Background(), apperr.ChunkKey(release.OpsAccount, "titogo", "box1", apperr.HourOf(e[0].TS), 200, len(eb)), eb, ""); err != nil { |
| t.Fatal(err) |
| @@ -60,23 +74,24 @@ func TestAPIErrorsGroupsBothFamilies(t *testing.T) { |
| } |
| return resp |
| } |
| + // The 502s and the platform events are Availability's, not Errors'. |
| resp := get("") |
| - if len(resp.Groups) != 2 { |
| + if len(resp.Groups) != 1 { |
| t.Fatalf("groups = %+v", resp.Groups) |
| } |
| if resp.Groups[0].FP != "edge500" || resp.Groups[0].Count != 12 || resp.Groups[0].Releases[0] != "v3" { |
| t.Fatalf("newest first, coalesced count: %+v", resp.Groups[0]) |
| } |
| - if resp.Groups[1].Source != apperr.SourceLog || !strings.Contains(resp.Groups[1].Name, "wake failed") { |
| - t.Fatalf("log-derived group: %+v", resp.Groups[1]) |
| - } |
| - if resp.LogLevel != "error" || len(resp.Sources) != 2 { |
| + if resp.LogLevel != "error" || len(resp.Sources) != 1 { |
| t.Fatalf("meta = %q %v", resp.LogLevel, resp.Sources) |
| } |
| // A warn read widens without changing the setting. |
| - if resp = get("level=warn"); len(resp.Groups) != 3 { |
| + if resp = get("level=warn"); len(resp.Groups) != 2 { |
| t.Fatalf("level=warn groups = %d", len(resp.Groups)) |
| } |
| + if resp.Groups[1].Source != apperr.SourceLog || !strings.Contains(resp.Groups[1].Name, "slow query") { |
| + t.Fatalf("log-derived group: %+v", resp.Groups[1]) |
| + } |
| // Source and fp filters. |
| if resp = get("source=edge"); len(resp.Groups) != 1 || resp.Groups[0].FP != "edge500" { |
| t.Fatalf("source filter = %+v", resp.Groups) |
| @@ -143,7 +158,7 @@ func TestAPITelemetryPutSetsLogLevel(t *testing.T) { |
| status, _, body = rawReq(t, http.MethodGet, errorsURL(fx, fx.sqid, "titogo", ""), token, "") |
| var resp errorsResponse |
| json.Unmarshal([]byte(body), &resp) |
| - if status != 200 || resp.LogLevel != "warn" || len(resp.Groups) != 3 { |
| + if status != 200 || resp.LogLevel != "warn" || len(resp.Groups) != 2 { |
| t.Fatalf("after set: %d level=%q groups=%d", status, resp.LogLevel, len(resp.Groups)) |
| } |
| status, _, body = rawReq(t, http.MethodPut, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/titogo/telemetry", op, `{"errors":{"log_level":"loud"}}`) |
| @@ -219,11 +234,16 @@ func TestAppErrorsPage(t *testing.T) { |
| if status != http.StatusOK { |
| t.Fatalf("errors page: %d %s", status, body) |
| } |
| - for _, want := range []string{"500 POST /checkout", "wake failed", "carlos errors --app titogo", "Sources live: log, edge", "to the nearest flush"} { |
| + for _, want := range []string{"500 POST /checkout", "carlos errors --app titogo", "Sources live: edge", "to the nearest flush"} { |
| if !strings.Contains(body, want) { |
| t.Errorf("page missing %q", want) |
| } |
| } |
| + for _, not := range []string{"wake failed", "502", "phpinfo"} { |
| + if strings.Contains(body, not) { |
| + t.Errorf("errors page shows Availability's %q", not) |
| + } |
| + } |
| if strings.Contains(body, "slow query") { |
| t.Error("warn lines must not show at the default level") |
| } |
| @@ -240,3 +260,75 @@ func TestAppErrorsPage(t *testing.T) { |
| t.Fatalf("empty window: %d", status) |
| } |
| } |
| + |
| +// TestAvailability: the API and the tab hold the other half — edge |
| +// 502/503/504 by cause, history filed per path regrouped by status, and |
| +// the platform's lifecycle events at warn and above, whatever the errors |
| +// level — and none of the app's own errors. |
| +func TestAvailability(t *testing.T) { |
| + fx, token := instanceFixture(t, "code-avail-1") |
| + now := time.Now().UTC() |
| + seedErrors(t, fx, now) |
| + api := func(query string) availabilityResponse { |
| + t.Helper() |
| + u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/titogo/availability" |
| + if query != "" { |
| + u += "?" + query |
| + } |
| + status, _, body := rawReq(t, http.MethodGet, u, token, "") |
| + if status != http.StatusOK { |
| + t.Fatalf("GET ?%s = %d: %s", query, status, body) |
| + } |
| + var resp availabilityResponse |
| + if err := json.Unmarshal([]byte(body), &resp); err != nil { |
| + t.Fatal(err) |
| + } |
| + return resp |
| + } |
| + resp := api("") |
| + byName := map[string]apperr.Group{} |
| + for _, g := range resp.Groups { |
| + byName[g.Name] = g |
| + } |
| + if len(resp.Groups) != 4 { |
| + t.Fatalf("groups = %+v", resp.Groups) |
| + } |
| + if g := byName["502 · the instance refused the connection"]; g.Count != 5 || g.Releases[0] != "v4" { |
| + t.Fatalf("new-style group = %+v", g) |
| + } |
| + old, ok := byName["502 · cause not recorded"] |
| + if !ok || old.Count != 2 { |
| + t.Fatalf("history must regroup into one row per status: %+v", resp.Groups) |
| + } |
| + for _, want := range []string{"wake failed: nothing is promoted for this instance's channel", "instance exited: signal: killed"} { |
| + if g, ok := byName[want]; !ok || g.Source != apperr.SourceLog { |
| + t.Errorf("platform event %q missing: %+v", want, resp.Groups) |
| + } |
| + } |
| + if _, ok := byName["500 POST /checkout"]; ok { |
| + t.Fatal("the app's own 500 is Errors', not Availability's") |
| + } |
| + // Expanding the regrouped row finds both paths, by the FP the tab shows |
| + // rather than the one stored. |
| + resp = api("fp=" + old.FP) |
| + if len(resp.Records) != 2 || resp.Records[0].Path == resp.Records[1].Path { |
| + t.Fatalf("expanded history = %+v", resp.Records) |
| + } |
| + |
| + fx.setUser("paul@keymail.dev") |
| + c := newBrowser(t) |
| + signUp(t, fx, c, "code-avail-signin") |
| + status, _, body := get(t, c, fx.u("/apps/titogo/availability")) |
| + if status != http.StatusOK { |
| + t.Fatalf("availability page: %d %s", status, body) |
| + } |
| + for _, want := range []string{"refused the connection", "cause not recorded", "wake failed", ">platform", "carlos availability --app titogo", |
| + `<h2 class="visually-hidden">Availability</h2>`, `/availability" aria-current="page">Availability</a>`} { |
| + if !strings.Contains(body, want) { |
| + t.Errorf("page missing %q", want) |
| + } |
| + } |
| + if strings.Contains(body, "/checkout") { |
| + t.Error("availability page shows the app's own 500") |
| + } |
| +} |
diff --git a/internal/console/authcli.go b/internal/console/authcli.go| index 80c626ff9..2c77fb0c0 100644 |
| --- a/internal/console/authcli.go |
| +++ b/internal/console/authcli.go |
| @@ -598,6 +598,7 @@ func (s *Server) registerCLIRoutes() { |
| // and an app token may read its own app's errors for the same reason it |
| // may read its own logs. |
| s.cliRoute("GET /api/cli/apps/{account}/{app}/errors", svcrole.Allow(svcrole.Instance, svcrole.Operate), s.handleAPIErrors) |
| + s.cliRoute("GET /api/cli/apps/{account}/{app}/availability", svcrole.Allow(svcrole.Instance, svcrole.Operate), s.handleAPIAvailability) |
| // The analytics read (`carlos analytics`, apianalytics.go) — the third of |
| // the same family, on the same footing: an app token that may read every |
| // line of its own logs is not learning anything new from a count of its |
diff --git a/internal/console/errorsview.go b/internal/console/errorsview.go| index 4eea044a0..a47fe8917 100644 |
| --- a/internal/console/errorsview.go |
| +++ b/internal/console/errorsview.go |
| @@ -30,10 +30,37 @@ type errorsWindow struct { |
| Sources []string // sources with at least one record in the window |
| } |
| -// readErrors assembles the window. level overrides the resolved setting |
| -// for THIS read only ("" = use the setting); it is what lets a member on |
| -// error widen one look to warn without changing anything (§8 ruling 3). |
| +// errView picks which half of the edge's and platform's failures a read |
| +// keeps (spec 2026-09-07 §3.2, amendment 2026-09-24): the app's own errors, |
| +// or Availability — whether an instance could answer at all. |
| +type errView int |
| + |
| +const ( |
| + viewErrors errView = iota |
| + viewAvailability |
| +) |
| + |
| +// availabilityLogLevel is the platform-event threshold Availability reads |
| +// at, whatever the app's errors setting: an instance that exited on a |
| +// signal is filed at warn (backend_exec.go), and it is exactly the event |
| +// an outage row needs beside it. |
| +const availabilityLogLevel = "warn" |
| + |
| +// readErrors assembles the Errors window. level overrides the resolved |
| +// setting for THIS read only ("" = use the setting); it is what lets a |
| +// member on error widen one look to warn without changing anything (§8 |
| +// ruling 3). |
| func (s *Server) readErrors(ctx context.Context, sqid, app string, f apperr.Filter, level string) (errorsWindow, error) { |
| + return s.readErrorView(ctx, sqid, app, f, level, viewErrors) |
| +} |
| + |
| +// readAvailability assembles the Availability window: edge 502/503/504 |
| +// grouped by cause, and platform events about the instance's lifecycle. |
| +func (s *Server) readAvailability(ctx context.Context, sqid, app string, f apperr.Filter) (errorsWindow, error) { |
| + return s.readErrorView(ctx, sqid, app, f, "", viewAvailability) |
| +} |
| + |
| +func (s *Server) readErrorView(ctx context.Context, sqid, app string, f apperr.Filter, level string, view errView) (errorsWindow, error) { |
| var out errorsWindow |
| res, err := telemetry.Resolve(ctx, s.cfg.Releases, sqid, app) |
| if err != nil { |
| @@ -43,28 +70,59 @@ func (s *Server) readErrors(ctx context.Context, sqid, app string, f apperr.Filt |
| if level == "" { |
| level = res.LogLevel |
| } |
| - stored, trimmed, err := apperr.ReadWindow(ctx, s.cfg.Releases, sqid, app, f, logsCollectCap) |
| + // A record the edge wrote before the amendment is regrouped on read |
| + // (apperr.AsAvailability), so its stored FP is not the one the tab |
| + // shows: the FP filter applies after the regroup, never in the read. |
| + // Every stored Availability record is the edge's, so the stored read |
| + // takes only the edge's: server and browser errors must not spend the |
| + // collect cap an expanded cause needs. (The edge's own 500s still pass |
| + // the read and are dropped by keep below.) |
| + rf, fp := f, f.FP |
| + if view == viewAvailability { |
| + rf.FP, rf.Source = "", apperr.SourceEdge |
| + level = availabilityLogLevel |
| + } |
| + stored, trimmed, err := apperr.ReadWindow(ctx, s.cfg.Releases, sqid, app, rf, logsCollectCap) |
| if err != nil { |
| return out, err |
| } |
| out.Truncated = trimmed |
| + keep := func(r apperr.Record) bool { return apperr.IsAvailability(r) == (view == viewAvailability) } |
| + var kept []apperr.Record |
| + for _, r := range stored { |
| + if !keep(r) { |
| + continue |
| + } |
| + if view == viewAvailability { |
| + if r = apperr.AsAvailability(r); fp != "" && r.FP != fp { |
| + continue |
| + } |
| + } |
| + kept = append(kept, r) |
| + } |
| var derived []apperr.Record |
| if f.Source == "" || f.Source == apperr.SourceLog { |
| lf := applog.Filter{Since: f.Since, Until: f.Until, Host: f.Host, Level: level} |
| + if view == viewAvailability { |
| + lf.Stream = applog.StreamPlatform |
| + } |
| lines, ltrim, err := applog.ReadWindow(ctx, s.cfg.Releases, sqid, app, lf, logsCollectCap) |
| if err != nil { |
| return out, err |
| } |
| out.Truncated = out.Truncated || ltrim |
| for _, l := range lines { |
| + if applog.IsDropNotice(l) { |
| + continue |
| + } |
| r := apperr.FromLog(l) |
| - if (f.FP == "" || r.FP == f.FP) && (f.Release == "" || r.Release == f.Release) { |
| + if keep(r) && (fp == "" || r.FP == fp) && (f.Release == "" || r.Release == f.Release) { |
| derived = append(derived, r) |
| } |
| } |
| } |
| - all := apperr.MergeRecords(stored, derived) |
| - if f.FP != "" { |
| + all := apperr.MergeRecords(kept, derived) |
| + if fp != "" { |
| out.Records = all |
| } |
| out.Groups = apperr.GroupRecords(all) |
| @@ -157,7 +215,22 @@ const accountAuditSubject = "account" |
| // handleAppErrors renders the Errors tab: the same read the API answers, |
| // with the filters as GET params so a view is a URL. |
| func (s *Server) handleAppErrors(w http.ResponseWriter, r *http.Request) { |
| - p, acct, ok := s.appPage(w, r, "errors") |
| + s.serveErrorView(w, r, viewErrors) |
| +} |
| + |
| +// handleAppAvailability renders the Availability tab: the Errors tab's |
| +// read and page shape over the other half of the records — no source or |
| +// level to choose, since what it holds is fixed (errView). |
| +func (s *Server) handleAppAvailability(w http.ResponseWriter, r *http.Request) { |
| + s.serveErrorView(w, r, viewAvailability) |
| +} |
| + |
| +func (s *Server) serveErrorView(w http.ResponseWriter, r *http.Request, view errView) { |
| + tab, tmpl := "errors", "app_errors" |
| + if view == viewAvailability { |
| + tab, tmpl = "availability", "app_availability" |
| + } |
| + p, acct, ok := s.appPage(w, r, tab) |
| if !ok { |
| return |
| } |
| @@ -178,14 +251,20 @@ func (s *Server) handleAppErrors(w http.ResponseWriter, r *http.Request) { |
| // who mistyped a URL wants to see. |
| f, level, _ = errorsFilterFromQuery(map[string][]string{"since": {p.ErrSince}}, now) |
| } |
| + if view == viewAvailability { |
| + f.Source, level = "", "" |
| + } |
| p.ErrSource, p.ErrLevel, p.ErrHost, p.ErrFP = f.Source, level, f.Host, f.FP |
| if s.cfg.Releases != nil { |
| ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| defer cancel() |
| - win, err := s.readErrors(ctx, acct.Sqid, p.AppName, f, level) |
| + win, err := s.readErrorView(ctx, acct.Sqid, p.AppName, f, level, view) |
| if err != nil { |
| - log.Printf("console: app errors %s/%s: %v", acct.Sqid, p.AppName, err) |
| + log.Printf("console: app %s %s/%s: %v", tab, acct.Sqid, p.AppName, err) |
| p.ErrsErr = "The errors could not be read just now — try again in a moment." |
| + if view == viewAvailability { |
| + p.ErrsErr = "Availability could not be read just now — try again in a moment." |
| + } |
| // A failed read leaves no resolved setting, and the filter's |
| // "setting (…)" option must still name a level rather than an |
| // empty pair of brackets. |
| @@ -210,7 +289,7 @@ func (s *Server) handleAppErrors(w http.ResponseWriter, r *http.Request) { |
| // rendering "the setting ()" over an empty window. |
| p.ErrWin.LogLevel, p.ErrWin.LogLevelFrom = telemetry.LogLevelError, "default" |
| } |
| - s.render(w, http.StatusOK, "app_errors", p) |
| + s.render(w, http.StatusOK, tmpl, p) |
| } |
| // handleAppErrorLevel is the Settings tab's form: owner, CSRF, one key. |
diff --git a/internal/console/server.go b/internal/console/server.go| index b5d5a2706..58f4c09f9 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -1161,6 +1161,11 @@ func NewServer(cfg Config, st *Store) (*Server, error) { |
| // (CSRF, then owner, then the byte-identical 404). |
| s.mux.HandleFunc("GET /a/{sqid}/apps/{app}/errors", s.handleAppErrors) |
| s.mux.HandleFunc("POST /a/{sqid}/apps/{app}/errors/level", s.handleAppErrorLevel) |
| + // Availability (errorsview.go): the Errors tab's sibling for the |
| + // failures that are the instance's, not the request's — edge 502/503/504 |
| + // by cause and the platform's lifecycle events. Member-readable, no |
| + // setting of its own. |
| + s.mux.HandleFunc("GET /a/{sqid}/apps/{app}/availability", s.handleAppAvailability) |
| // The Analytics tab (analyticsview.go) and the switch beside it. The tab |
| // is member-readable like every other app tab; the switch rides the same |
| // gate the errors level does at both levels, and bumps the serial where |
diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go| index c5efc7d15..44765af64 100644 |
| --- a/internal/console/svcroutes_test.go |
| +++ b/internal/console/svcroutes_test.go |
| @@ -335,9 +335,16 @@ func TestNoRoleReachesAMintFleetOrPolicyDoor(t *testing.T) { |
| // views. The telemetry PUT that decides whether those counts are collected |
| // at all is Operate-only and no instance credential reaches it — |
| // TestAPITelemetryPutSetsAnalytics pins that half. |
| +// |
| +// GET .../availability is the tenth (spec 2026-09-07 §11), and it stands on |
| +// the errors door's footing exactly: it is the other half of the same |
| +// records, split off 2026-09-24. Before the split an instance credential |
| +// read these very rows through GET .../errors, so granting it here widens |
| +// nothing — refusing it would be the change. |
| var exemptFromLegacyPin = map[string]bool{ |
| "GET /api/cli/apps": true, |
| "GET /api/cli/apps/{account}/{app}/errors": true, |
| + "GET /api/cli/apps/{account}/{app}/availability": true, |
| "GET /api/cli/apps/{account}/{app}/analytics": true, |
| "GET /api/cli/apps/{account}/{app}/domains/{host}": true, |
| "PUT /api/cli/apps/{account}/{app}/domains/{host}/paths": true, |
diff --git a/internal/console/templates/app_analytics.html b/internal/console/templates/app_analytics.html| index dbb3b841b..b5f15b06b 100644 |
| --- a/internal/console/templates/app_analytics.html |
| +++ b/internal/console/templates/app_analytics.html |
| @@ -4,7 +4,7 @@ |
| {{template "appnav" .}} |
| {{template "lognav" .}} |
| -<h2>Analytics</h2> |
| +<h2 class="visually-hidden">Analytics</h2> |
| {{if .AnalyticsErr}}<p class="banner error">{{.AnalyticsErr}}</p>{{end}} |
| {{if not .AnalyticsWin.On}} |
| <p><strong>Analytics is off for this app</strong> (from the {{.AnalyticsWin.Setting.CountFrom}}). |
diff --git a/internal/console/templates/app_availability.html b/internal/console/templates/app_availability.html| new file mode 100644 |
| index 000000000..e5fd5ef47 |
| --- /dev/null |
| +++ b/internal/console/templates/app_availability.html |
| @@ -0,0 +1,61 @@ |
| +{{define "app_availability"}}{{template "top" .}} |
| +{{if .Flash}}<p class="banner">{{.Flash}}</p>{{end}} |
| +{{if .FlashErr}}<p class="banner error">{{.FlashErr}}</p>{{end}} |
| +{{template "appnav" .}} |
| +{{template "lognav" .}} |
| + |
| +<h2 class="visually-hidden">Availability</h2> |
| +{{if .ErrsErr}}<p class="banner error">{{.ErrsErr}}</p>{{end}} |
| +<form method="get" class="small"> |
| + <label>from the last |
| + <select name="since"> |
| + <option value="1h"{{if eq .ErrSince "1h"}} selected{{end}}>hour</option> |
| + <option value="24h"{{if eq .ErrSince "24h"}} selected{{end}}>day</option> |
| + <option value="168h"{{if eq .ErrSince "168h"}} selected{{end}}>week</option> |
| + </select> |
| + </label> |
| + <label>host <input name="host" value="{{.ErrHost}}" size="24"></label> |
| + <button type="submit">Update</button> |
| +</form> |
| +<p class="small">Requests an instance could not answer — a 502, 503 or 504 — grouped by cause rather than by page, and the platform's own events about the instance: failed wakes and exits. |
| +A 500 is the app answering, and stays on Errors. A visitor who left within five seconds is not counted; one who gave up after waiting longer is. Counts are to the nearest flush.</p> |
| + |
| +{{if .ErrFP}} |
| +{{with .ErrWin.Records}} |
| +<h3>Occurrences</h3> |
| +<p class="small"><a href="?since={{$.ErrSince}}{{if $.ErrHost}}&host={{$.ErrHost}}{{end}}">All causes</a></p> |
| +{{if $.ErrWin.Truncated}}<p class="small">Showing the newest {{len $.ErrWin.Records}} occurrences of a larger window — narrow the window, or use <code>carlos availability</code> for all of them.</p>{{end}} |
| +{{range .}} |
| +<div class="tablewrap"><table> |
| + <tr><th>Last</th><td>{{.Last.UTC.Format "2006-01-02 15:04:05"}}</td></tr> |
| + <tr><th>Count</th><td>{{.Count}}</td></tr> |
| + <tr><th>Host</th><td>{{.Host}}</td></tr> |
| + {{if .Release}}<tr><th>Release</th><td>{{.Release}}</td></tr>{{end}} |
| + {{if .Path}}<tr><th>Path</th><td><code>{{.Path}}</code>{{if gt .Count 1}} <span class="small">(the first of these {{.Count}})</span>{{end}}</td></tr>{{end}} |
| + <tr><th>What happened</th><td>{{.Name}}</td></tr> |
| + <tr><th>Logs</th><td><a href="{{$.Nav}}/apps/{{$.AppName}}/logs?since=1h&host={{.Host}}">around this time</a></td></tr> |
| +</table></div> |
| +{{end}} |
| +{{else}}<p>No occurrences of this cause in the window.</p>{{end}} |
| +{{else if .ErrWin.Groups}} |
| +{{if .ErrWin.Truncated}}<p class="small">Showing {{len .ErrWin.Groups}} causes of a larger window — counts are within what was read. Narrow the window, or use <code>carlos availability</code> for all of them.</p>{{end}} |
| +<div class="tablewrap"> |
| +<table> |
| + <tr><th>Last seen</th><th>First seen</th><th>Count</th><th>From</th><th>What happened</th><th>Hosts</th><th>Releases</th></tr> |
| + {{range .ErrWin.Groups}}<tr> |
| + <td>{{.Last.UTC.Format "2006-01-02 15:04"}}</td> |
| + <td>{{.First.UTC.Format "2006-01-02 15:04"}}</td> |
| + <td>{{.Count}}{{if eq .FP "_overflow"}} <span class="small" title="occurrences beyond the hourly cap on distinct errors">(overflow)</span>{{end}}</td> |
| + <td>{{if eq .Source "log"}}platform{{if .Level}} <span class="small">{{.Level}}</span>{{end}}{{else}}{{.Source}}{{end}}</td> |
| + <td><a href="?since={{$.ErrSince}}&fp={{.FP}}{{if $.ErrHost}}&host={{$.ErrHost}}{{end}}">{{.Name}}</a></td> |
| + <td>{{range $i, $h := .Hosts}}{{if $i}}, {{end}}{{$h}}{{end}}</td> |
| + <td>{{range $i, $r := .Releases}}{{if $i}}, {{end}}<code>{{$r}}</code>{{end}}</td> |
| + </tr>{{end}} |
| +</table> |
| +</div> |
| +<p class="small">Times are UTC. The same view from a terminal: <code>carlos availability --app {{.AppName}} --since {{.ErrSince}}</code>.</p> |
| +{{else if not .ErrsErr}} |
| +<p>No availability problems in this window.</p> |
| +<p class="small">The same view from a terminal: <code>carlos availability --app {{.AppName}}</code>.</p> |
| +{{end}} |
| +{{template "bottom" .}}{{end}} |
diff --git a/internal/console/templates/app_errors.html b/internal/console/templates/app_errors.html| index 36210b847..0525cd51a 100644 |
| --- a/internal/console/templates/app_errors.html |
| +++ b/internal/console/templates/app_errors.html |
| @@ -4,7 +4,7 @@ |
| {{template "appnav" .}} |
| {{template "lognav" .}} |
| -<h2>Errors</h2> |
| +<h2 class="visually-hidden">Errors</h2> |
| {{if .ErrsErr}}<p class="banner error">{{.ErrsErr}}</p>{{end}} |
| <form method="get" class="small"> |
| <label>from the last |
| @@ -75,6 +75,6 @@ Counts are to the nearest flush. Messages are redacted for addresses and credent |
| <p class="small">Times are UTC. The same view from a terminal: <code>carlos errors --app {{.AppName}} --since {{.ErrSince}}</code>.</p> |
| {{else if not .ErrsErr}} |
| <p>No errors in this window.</p> |
| -<p class="small">Log lines at <code>{{.ErrWin.LogLevel}}</code> and above and responses of 500 or higher appear here on their own; a JavaScript or server-side error needs the hook (coming in a later step). The level is set on the Settings tab. The same view from a terminal: <code>carlos errors --app {{.AppName}}</code>.</p> |
| +<p class="small">Log lines at <code>{{.ErrWin.LogLevel}}</code> and above and the app's own 500s appear here on their own (a 502, 503 or 504 is on <a href="{{.Nav}}/apps/{{.AppName}}/availability">Availability</a>); a JavaScript or server-side error needs the hook (coming in a later step). The level is set on the Settings tab. The same view from a terminal: <code>carlos errors --app {{.AppName}}</code>.</p> |
| {{end}} |
| {{template "bottom" .}}{{end}} |
diff --git a/internal/console/templates/app_logs.html b/internal/console/templates/app_logs.html| index fdfe6ed66..6ef6e785f 100644 |
| --- a/internal/console/templates/app_logs.html |
| +++ b/internal/console/templates/app_logs.html |
| @@ -4,7 +4,7 @@ |
| {{template "appnav" .}} |
| {{template "lognav" .}} |
| -<h2>Logs</h2> |
| +<h2 class="visually-hidden">Logs</h2> |
| {{if .LogsErr}}<p class="banner error">{{.LogsErr}}</p>{{end}} |
| <form method="get" class="small" id="logs-filter" data-url="{{.Base}}/apps/{{.AppName}}/logs/data" data-cursor="{{.LogCursor}}" data-follow-error="{{.LogCopy.FollowError}}" data-refreshed="{{.LogCopy.Refreshed}}"> |
| <label>{{.LogCopy.Instance}} |
diff --git a/internal/console/templates/appnav.html b/internal/console/templates/appnav.html| index 9f6da22b1..18b548fa5 100644 |
| --- a/internal/console/templates/appnav.html |
| +++ b/internal/console/templates/appnav.html |
| @@ -7,7 +7,7 @@ |
| <a href="{{.Nav}}/apps/{{.AppName}}"{{if eq .ActiveTab "overview"}} aria-current="page"{{end}}>Overview</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/instances"{{if eq .ActiveTab "instances"}} aria-current="page"{{end}}>Running</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/deployments"{{if eq .ActiveTab "releases"}} aria-current="page"{{end}}>{{.DeploymentsCopy.Title}}</a> |
| - <a href="{{.Nav}}/apps/{{.AppName}}/logs"{{if or (eq .ActiveTab "logs") (eq .ActiveTab "errors") (eq .ActiveTab "analytics")}} aria-current="page"{{end}}>Logs</a> |
| + <a href="{{.Nav}}/apps/{{.AppName}}/logs"{{if or (eq .ActiveTab "logs") (eq .ActiveTab "errors") (eq .ActiveTab "availability") (eq .ActiveTab "analytics")}} aria-current="page"{{end}}>Logs</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/activity"{{if eq .ActiveTab "activity"}} aria-current="page"{{end}}>Activity</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/settings"{{if eq .ActiveTab "settings"}} aria-current="page"{{end}}>Settings</a> |
| </nav> |
| @@ -17,6 +17,7 @@ |
| <nav class="tabs" aria-label="Logs"> |
| <a href="{{.Nav}}/apps/{{.AppName}}/logs"{{if eq .ActiveTab "logs"}} aria-current="page"{{end}}>Logs</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/errors"{{if eq .ActiveTab "errors"}} aria-current="page"{{end}}>Errors</a> |
| + <a href="{{.Nav}}/apps/{{.AppName}}/availability"{{if eq .ActiveTab "availability"}} aria-current="page"{{end}}>Availability</a> |
| <a href="{{.Nav}}/apps/{{.AppName}}/analytics"{{if eq .ActiveTab "analytics"}} aria-current="page"{{end}}>Analytics</a> |
| </nav> |
| {{end}} |
diff --git a/internal/edge/answered.go b/internal/edge/answered.go| index c74fbd578..2df6e81d9 100644 |
| --- a/internal/edge/answered.go |
| +++ b/internal/edge/answered.go |
| @@ -3,6 +3,7 @@ package edge |
| import ( |
| "context" |
| "net/http" |
| + "time" |
| ) |
| // answeredKey carries the flag the edge attaches so it can tell a 503 the |
| @@ -21,17 +22,64 @@ import ( |
| // the activator instead would be an import cycle. |
| type answeredKey struct{} |
| +// outcome is what the serving path learns about a request that its status |
| +// alone does not say: whether the activator answered it itself, and, for a |
| +// 502/503/504 the edge produced, why (an apperr.Cause*, or causeClientGone). |
| +// The observing defer in ServeHTTP reads both once the response is written. |
| +type outcome struct { |
| + answered bool |
| + cause string |
| + start time.Time // when the edge began serving, so a proxy failure knows how long the client waited |
| +} |
| + |
| +// causeClientGone is the cause noted when a proxy fails because the CLIENT |
| +// went away (its request context is done) soon after asking: ReverseProxy |
| +// still writes a 502, but to nobody, and a visitor navigating away mid-load |
| +// is not an outage. It is never filed. |
| +const causeClientGone = "client_gone" |
| + |
| +// clientGoneGrace is how long a client may wait before its leaving stops |
| +// being its own business. Past it the wait is the finding: a dead unix |
| +// socket is redialled for 13s (dialUnixWithRetry) and a wedged instance is |
| +// never timed out at all, so a monitor or a visitor that gives up first is |
| +// the only way those outages end — and filing nothing for them would hide |
| +// exactly the worst ones (apperr.CauseAbandoned). |
| +var clientGoneGrace = 5 * time.Second // a var only so a test can shorten it |
| + |
| +// withOutcome returns r carrying a fresh outcome and the outcome itself. |
| +func withOutcome(r *http.Request) (*http.Request, *outcome) { |
| + o := &outcome{start: time.Now()} |
| + return r.WithContext(context.WithValue(r.Context(), answeredKey{}, o)), o |
| +} |
| + |
| // WithAnswerFlag returns r carrying a fresh flag and the flag itself. |
| func WithAnswerFlag(r *http.Request) (*http.Request, *bool) { |
| - f := new(bool) |
| - return r.WithContext(context.WithValue(r.Context(), answeredKey{}, f)), f |
| + r, o := withOutcome(r) |
| + return r, &o.answered |
| } |
| // MarkAnswered sets the flag when r carries one; a no-op otherwise — so the |
| // activator can call it unconditionally, including on the paths (a |
| // non-hibernating route, a direct unit test) where the edge attached none. |
| func MarkAnswered(r *http.Request) { |
| - if f, ok := r.Context().Value(answeredKey{}).(*bool); ok { |
| - *f = true |
| + if o, ok := r.Context().Value(answeredKey{}).(*outcome); ok { |
| + o.answered = true |
| + } |
| +} |
| + |
| +// waited reports how long r has been served for, when the edge attached an |
| +// outcome; zero otherwise. |
| +func waited(r *http.Request) time.Duration { |
| + if o, ok := r.Context().Value(answeredKey{}).(*outcome); ok { |
| + return time.Since(o.start) |
| + } |
| + return 0 |
| +} |
| + |
| +// noteCause records why the edge could not get an answer for r; a no-op |
| +// when r carries no outcome, like MarkAnswered. |
| +func noteCause(r *http.Request, cause string) { |
| + if o, ok := r.Context().Value(answeredKey{}).(*outcome); ok { |
| + o.cause = cause |
| } |
| } |
diff --git a/internal/edge/edge.go b/internal/edge/edge.go| index f47dd053e..3a5cfba56 100644 |
| --- a/internal/edge/edge.go |
| +++ b/internal/edge/edge.go |
| @@ -28,6 +28,7 @@ import ( |
| "errors" |
| "flag" |
| "fmt" |
| + "io" |
| "log" |
| "maps" |
| "net" |
| @@ -376,14 +377,30 @@ func (e *Edge) EnableErrors(s ErrorSink) { |
| } |
| // emitEdgeError files one edge-observed 5xx for rt. The path is sanitised |
| -// (query and token segments off) before it becomes the record's name. |
| -func (e *Edge) emitEdgeError(rt registry.Route, r *http.Request, status int) { |
| +// (query and token segments off) before it goes on the record. A 502, 503 |
| +// or 504 is filed under Availability (apperr/availability.go): named and |
| +// so fingerprinted by status and cause, never by path, with the path kept |
| +// only as the occurrence's sample. cause is what a proxy ErrorHandler |
| +// noted; none means the upstream wrote the status itself. A failure whose |
| +// client had already gone is not filed at all. |
| +// |
| +// On a tunnel-backed or fleet-placed route the upstream is not the app: it |
| +// is another box's edge (the agent's, the only tunnel client), which files |
| +// what it observes serving the app — including deciding NOT to file a wake |
| +// 503 its activator answered (MarkAnswered). So this box files only what it |
| +// noted itself, the hop failing; a status that merely passed through would |
| +// be the same failure filed twice, and a 502/503 labelled "the app answered |
| +// it itself" when the app never saw the request. |
| +func (e *Edge) emitEdgeError(rt registry.Route, r *http.Request, status int, cause string) { |
| ref := e.errSink.Load() |
| - if ref == nil { |
| + if ref == nil || cause == causeClientGone { |
| + return |
| + } |
| + if _, fleetPlaced := registry.FleetPlacement(rt.Region); cause == "" && (rt.Tunnel != "" || fleetPlaced) { |
| return |
| } |
| path := apperr.SanitizePath(r.URL.Path) |
| - ref.Add(release.RouteAccount(rt), rt.App, apperr.Record{ |
| + rec := apperr.Record{ |
| Host: rt.Host, |
| Source: apperr.SourceEdge, |
| Kind: apperr.KindHTTP, |
| @@ -391,7 +408,60 @@ func (e *Edge) emitEdgeError(rt registry.Route, r *http.Request, status int) { |
| Path: path, |
| Status: status, |
| Release: rt.Version, |
| - }) |
| + } |
| + if apperr.UnavailableStatus(status) { |
| + if cause == "" { |
| + cause = apperr.CauseUpstream |
| + } |
| + rec.Kind, rec.Cause, rec.Name = apperr.KindUnavailable, cause, apperr.AvailabilityName(status, cause) |
| + } |
| + ref.Add(release.RouteAccount(rt), rt.App, rec) |
| +} |
| + |
| +// proxyCause classifies why a proxy could not get an answer for r. The |
| +// client's own context is checked before the error itself: a client that |
| +// left surfaces as a canceled dial or read whatever the instance was doing. |
| +// Leaving soon after asking is the client's business and is not filed; |
| +// leaving after clientGoneGrace means the instance kept it waiting, and is. |
| +func proxyCause(r *http.Request, err error) string { |
| + if r.Context().Err() != nil { |
| + if waited(r) >= clientGoneGrace { |
| + return apperr.CauseAbandoned |
| + } |
| + return causeClientGone |
| + } |
| + var ne net.Error |
| + switch { |
| + case errors.Is(err, syscall.ECONNREFUSED), errors.Is(err, syscall.ENOENT): |
| + return apperr.CauseRefused |
| + case errors.Is(err, context.DeadlineExceeded), errors.As(err, &ne) && ne.Timeout(): |
| + return apperr.CauseTimeout |
| + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF), |
| + errors.Is(err, syscall.ECONNRESET), errors.Is(err, syscall.EPIPE): |
| + return apperr.CauseReset |
| + } |
| + return apperr.CauseProxy |
| +} |
| + |
| +// tunnelCause is proxyCause for a proxy whose far side is a fleet box's |
| +// tunnel: every failure but the client's own leaving is the hop's, not |
| +// the instance's. |
| +func tunnelCause(r *http.Request, err error) string { |
| + if c := proxyCause(r, err); c == causeClientGone || c == apperr.CauseAbandoned { |
| + return c |
| + } |
| + return apperr.CauseTunnel |
| +} |
| + |
| +// defaultProxyError is httputil.ReverseProxy's own default ErrorHandler — |
| +// the same log line, the same bare 502 — plus the cause note, for the |
| +// proxies that had no handler of their own. |
| +func defaultProxyError(classify func(*http.Request, error) string) func(http.ResponseWriter, *http.Request, error) { |
| + return func(w http.ResponseWriter, r *http.Request, err error) { |
| + noteCause(r, classify(r, err)) |
| + log.Printf("http: proxy error: %v", err) |
| + w.WriteHeader(http.StatusBadGateway) |
| + } |
| } |
| // armStoreDoor builds (or clears) the fleet store door from whatever is |
| @@ -1399,21 +1469,21 @@ func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| var bytes int64 |
| ob := &countingWriter{ResponseWriter: w, n: &bytes} |
| w = ob |
| - var answered *bool |
| - if up.hibernating { |
| - // The activator marks the wake-path 503s it writes itself |
| - // (MarkAnswered, answered.go); those are already the member's |
| - // platform log error and are not filed again as edge errors |
| - // (spec §3.2, "exactly once"). |
| - r, answered = WithAnswerFlag(r) |
| - } |
| if up.route.Kind != "status" { |
| + // The outcome rides the request so the two things only the inner |
| + // layers know reach this defer: the activator marks the wake-path |
| + // 503s it writes itself (MarkAnswered, answered.go) — already the |
| + // member's platform event, so not filed again (spec §3.2, "exactly |
| + // once") — and a proxy ErrorHandler notes WHY it answered 502/503 |
| + // (noteCause), which is what groups Availability by cause. |
| + var out *outcome |
| + r, out = withOutcome(r) |
| defer func() { |
| if m := e.usageMeter.Load(); m != nil { |
| m.Record(up.route, 1, bytes, hopped) |
| } |
| - if ob.status >= 500 && (answered == nil || !*answered) { |
| - e.emitEdgeError(up.route, r, ob.status) |
| + if ob.status >= 500 && !out.answered { |
| + e.emitEdgeError(up.route, r, ob.status, out.cause) |
| } |
| e.observeAnalytics(up.route, r, ob, bytes) |
| }() |
| @@ -2097,6 +2167,7 @@ func unixProxy(socketPath string) *httputil.ReverseProxy { |
| MaxIdleConns: 32, |
| IdleConnTimeout: 90 * time.Second, |
| }, |
| + ErrorHandler: defaultProxyError(proxyCause), |
| } |
| } |
| @@ -2116,6 +2187,7 @@ func tcpProxy(addr string) *httputil.ReverseProxy { |
| MaxIdleConns: 32, |
| IdleConnTimeout: 90 * time.Second, |
| }, |
| + ErrorHandler: defaultProxyError(proxyCause), |
| } |
| } |
| @@ -2254,6 +2326,7 @@ func multiProxy(host string, p *proberSet) *httputil.ReverseProxy { |
| }, |
| ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| if errors.Is(err, errNoLiveUpstream) { |
| + noteCause(r, apperr.CauseNoUpstream) |
| w.Header().Set("Retry-After", "2") |
| http.Error(w, "no healthy upstream", http.StatusServiceUnavailable) |
| return |
| @@ -2261,6 +2334,7 @@ func multiProxy(host string, p *proberSet) *httputil.ReverseProxy { |
| // Anything else is a real proxy failure (upstream hung up |
| // mid-response, malformed reply): same shape as |
| // httputil.ReverseProxy's own default handler, log and 502. |
| + noteCause(r, proxyCause(r, err)) |
| log.Printf("edge: multi-upstream %s proxy error: %v", host, err) |
| w.WriteHeader(http.StatusBadGateway) |
| }, |
| @@ -2284,7 +2358,8 @@ func tunnelProxy(name string, hub *TunnelHub) *httputil.ReverseProxy { |
| pr.Out.Host = pr.In.Host |
| scrubInboundPurgeHeader(pr) |
| }, |
| - Transport: hub.transportFor(name), |
| + Transport: hub.transportFor(name), |
| + ErrorHandler: defaultProxyError(tunnelCause), |
| } |
| } |
| @@ -2347,6 +2422,7 @@ func fleetProxy(fleet, boxLabel string, hub *TunnelHub) *httputil.ReverseProxy { |
| }), |
| ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| if errors.Is(err, errFleetDark) { |
| + noteCause(r, apperr.CauseFleetDark) |
| w.Header().Set("Retry-After", "2") |
| http.Error(w, "fleet unreachable: no connected box", http.StatusServiceUnavailable) |
| return |
| @@ -2354,6 +2430,7 @@ func fleetProxy(fleet, boxLabel string, hub *TunnelHub) *httputil.ReverseProxy { |
| // Any other transport failure (e.g. the picked channel died between |
| // fleetChannelFor's check and the RoundTrip call) — same shape as |
| // httputil.ReverseProxy's own default handler: log and 502. |
| + noteCause(r, tunnelCause(r, err)) |
| log.Printf("edge: fleet %s proxy error: %v", fleet, err) |
| w.WriteHeader(http.StatusBadGateway) |
| }, |
diff --git a/internal/edge/edgeerrors_test.go b/internal/edge/edgeerrors_test.go| index e267de7d1..d7c9b3067 100644 |
| --- a/internal/edge/edgeerrors_test.go |
| +++ b/internal/edge/edgeerrors_test.go |
| @@ -1,11 +1,17 @@ |
| package edge |
| import ( |
| + "context" |
| + "errors" |
| + "io" |
| + "net" |
| "net/http" |
| "net/http/httptest" |
| "strings" |
| "sync" |
| + "syscall" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/apperr" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -170,3 +176,176 @@ func TestEdgeEmitsUnmarked503OnHibernatingRoute(t *testing.T) { |
| t.Fatalf("record = %+v", recs[0]) |
| } |
| } |
| + |
| +// TestEdgeFilesUnavailableByCause: a 502/503/504 goes under Availability, |
| +// named by status and cause rather than path, so one dead instance probed |
| +// on two paths is one fingerprint — and a 503 the app wrote itself says so. |
| +func TestEdgeFilesUnavailableByCause(t *testing.T) { |
| + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + http.Error(w, "busy", http.StatusServiceUnavailable) |
| + })) |
| + t.Cleanup(backend.Close) |
| + dead := httptest.NewServer(http.NotFoundHandler()) |
| + deadAddr := dead.Listener.Addr().String() |
| + dead.Close() // nothing listens here now: every dial is refused |
| + |
| + reg := openReg(t) |
| + for _, rt := range []registry.Route{ |
| + {Host: "busy.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web", Version: "v7"}, |
| + {Host: "dead.example.com", Kind: "service", Addr: deadAddr, Status: "ready", App: "web", Version: "v7"}, |
| + } { |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + e, err := New(reg, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + sink := &memErrSink{} |
| + e.EnableErrors(sink) |
| + srv := httptest.NewServer(e) |
| + t.Cleanup(srv.Close) |
| + do := func(host, path string) int { |
| + req, _ := http.NewRequest("GET", srv.URL+path, nil) |
| + req.Host = host |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + resp.Body.Close() |
| + return resp.StatusCode |
| + } |
| + if got := do("busy.example.com", "/sandbox"); got != 503 { |
| + t.Fatalf("busy status = %d", got) |
| + } |
| + if do("dead.example.com", "/.env") != 502 || do("dead.example.com", "/phpinfo.php") != 502 { |
| + t.Fatal("a refused dial must answer 502") |
| + } |
| + recs := sink.snap() |
| + if len(recs) != 3 { |
| + t.Fatalf("records = %d, want 3", len(recs)) |
| + } |
| + busy := recs[0] |
| + if busy.Kind != apperr.KindUnavailable || busy.Cause != apperr.CauseUpstream || busy.Name != "503 · the app answered it itself" || busy.Path != "/sandbox" { |
| + t.Fatalf("busy record = %+v", busy) |
| + } |
| + for _, r := range recs[1:] { |
| + if r.Kind != apperr.KindUnavailable || r.Cause != apperr.CauseRefused || r.Status != 502 || strings.Contains(r.Name, "/") { |
| + t.Fatalf("dead record = %+v", r) |
| + } |
| + } |
| + if recs[1].Name != recs[2].Name || recs[1].Path == recs[2].Path { |
| + t.Fatalf("two paths of one refused instance must share a name and keep their own paths: %+v", recs[1:]) |
| + } |
| + if !apperr.IsAvailability(busy) { |
| + t.Fatal("IsAvailability(busy) = false") |
| + } |
| +} |
| + |
| +// TestProxyCauseClientGone: a proxy failure after the client left is the |
| +// client's doing, whatever the error says, and is never filed. |
| +func TestProxyCauseClientGone(t *testing.T) { |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + cancel() |
| + r := httptest.NewRequest("GET", "http://x/", nil).WithContext(ctx) |
| + if got := proxyCause(r, syscall.ECONNRESET); got != causeClientGone { |
| + t.Fatalf("proxyCause = %q, want %q", got, causeClientGone) |
| + } |
| + r, out := withOutcome(httptest.NewRequest("GET", "http://x/", nil)) |
| + for _, c := range []struct { |
| + err error |
| + want string |
| + }{ |
| + {&net.OpError{Op: "dial", Err: syscall.ECONNREFUSED}, apperr.CauseRefused}, |
| + {io.ErrUnexpectedEOF, apperr.CauseReset}, |
| + {context.DeadlineExceeded, apperr.CauseTimeout}, |
| + {errors.New("malformed"), apperr.CauseProxy}, |
| + } { |
| + if got := proxyCause(r, c.err); got != c.want { |
| + t.Errorf("proxyCause(%v) = %q, want %q", c.err, got, c.want) |
| + } |
| + } |
| + // The same leaving after a long wait is the instance's doing: a dead |
| + // socket redialled for 13s, or a wedged instance nothing times out. |
| + gone, o := withOutcome(httptest.NewRequest("GET", "http://x/", nil).WithContext(ctx)) |
| + o.start = time.Now().Add(-clientGoneGrace - time.Second) |
| + if got := proxyCause(gone, syscall.ECONNREFUSED); got != apperr.CauseAbandoned { |
| + t.Fatalf("long wait then gone: proxyCause = %q, want %q", got, apperr.CauseAbandoned) |
| + } |
| + if got := tunnelCause(gone, syscall.ECONNREFUSED); got != apperr.CauseAbandoned { |
| + t.Fatalf("long wait then gone via tunnel: %q", got) |
| + } |
| + noteCause(r, apperr.CauseReset) |
| + if out.cause != apperr.CauseReset { |
| + t.Fatalf("noteCause did not reach the outcome: %+v", out) |
| + } |
| + sink := &memErrSink{} |
| + e := &Edge{} |
| + e.EnableErrors(sink) |
| + e.emitEdgeError(registry.Route{Host: "h", App: "web"}, r, 502, causeClientGone) |
| + if n := len(sink.snap()); n != 0 { |
| + t.Fatalf("a client-gone 502 was filed (%d records)", n) |
| + } |
| +} |
| + |
| +// TestEdgeFilesOnlyItsOwnFailuresThroughATunnel: behind a tunnel or a fleet |
| +// placement the upstream is another box's edge, which files what it saw |
| +// serving the app. This box files the hop failing, never a status that |
| +// merely passed through it. |
| +func TestEdgeFilesOnlyItsOwnFailuresThroughATunnel(t *testing.T) { |
| + sink := &memErrSink{} |
| + e := &Edge{} |
| + e.EnableErrors(sink) |
| + r, _ := withOutcome(httptest.NewRequest("GET", "http://x/admin", nil)) |
| + tunneled := registry.Route{Host: "t.example.com", App: "web", Tunnel: "box1"} |
| + fleeted := registry.Route{Host: "f.example.com", App: "web", Region: registry.PlacementTarget("eleven")} |
| + for _, rt := range []registry.Route{tunneled, fleeted} { |
| + e.emitEdgeError(rt, r, 503, "") // the serving box's own 503, passed through |
| + e.emitEdgeError(rt, r, 500, "") // the app's 500, filed on the serving box |
| + } |
| + if n := len(sink.snap()); n != 0 { |
| + t.Fatalf("pass-through statuses were filed on the front box: %+v", sink.snap()) |
| + } |
| + e.emitEdgeError(tunneled, r, 502, apperr.CauseTunnel) |
| + e.emitEdgeError(fleeted, r, 503, apperr.CauseFleetDark) |
| + if recs := sink.snap(); len(recs) != 2 || recs[0].Cause != apperr.CauseTunnel || recs[1].Cause != apperr.CauseFleetDark { |
| + t.Fatalf("the hop's own failures must be filed: %+v", recs) |
| + } |
| +} |
| + |
| +// TestEdgeFilesAbandonedWaitEndToEnd drives a real request at an instance |
| +// that never answers, through ServeHTTP and the proxy, with a client that |
| +// gives up after the grace: the wait is measured from where the edge |
| +// attaches the outcome, and the outage is filed rather than hidden. |
| +func TestEdgeFilesAbandonedWaitEndToEnd(t *testing.T) { |
| + old := clientGoneGrace |
| + clientGoneGrace = 50 * time.Millisecond |
| + t.Cleanup(func() { clientGoneGrace = old }) |
| + release := make(chan struct{}) |
| + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + select { |
| + case <-release: |
| + case <-r.Context().Done(): |
| + } |
| + })) |
| + t.Cleanup(func() { close(release); backend.Close() }) |
| + reg := openReg(t) |
| + if err := reg.Upsert(registry.Route{Host: "wedged.example.com", Kind: "service", Addr: backend.Listener.Addr().String(), Status: "ready", App: "web"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + e, err := New(reg, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + sink := &memErrSink{} |
| + e.EnableErrors(sink) |
| + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) |
| + defer cancel() |
| + req := httptest.NewRequest("GET", "http://wedged.example.com/slow", nil).WithContext(ctx) |
| + e.ServeHTTP(httptest.NewRecorder(), req) |
| + recs := sink.snap() |
| + if len(recs) != 1 || recs[0].Cause != apperr.CauseAbandoned || recs[0].Status != 502 { |
| + t.Fatalf("records = %+v, want one abandoned 502", recs) |
| + } |
| +} |
diff --git a/internal/edge/static.go b/internal/edge/static.go| index 39fa9c4df..ea43c68ba 100644 |
| --- a/internal/edge/static.go |
| +++ b/internal/edge/static.go |
| @@ -29,6 +29,7 @@ import ( |
| "sync" |
| "time" |
| + "github.com/carlosframework/platform/internal/apperr" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -147,6 +148,7 @@ func (s *staticSite) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| // this 502 stays unstamped rather than claiming a build it isn't |
| // serving. |
| log.Printf("edge: static %s/%s: %v", s.app, s.channel, err) |
| + noteCause(r, apperr.CauseStatic) |
| http.Error(w, "static site unavailable", http.StatusBadGateway) |
| return |
| } |
diff --git a/main.go b/main.go| index ebec0f470..b3b52dff7 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -89,6 +89,8 @@ func main() { |
| runLogs(args) |
| case "errors": |
| runErrors(args) |
| + case "availability": |
| + runAvailability(args) |
| case "analytics": |
| runAnalytics(args) |
| case "channels": |
| @@ -226,7 +228,9 @@ Your apps — ship and run software on a CARLOS deployment |
| logs an app's merged log timeline: its own output plus platform |
| events (wakes, restarts, failures); -f follows live |
| errors error groups for an app: log lines at the set level plus |
| - 5xx the edge saw |
| + 500s the edge saw |
| + availability requests an app's instances could not answer (502/503/504), |
| + grouped by cause, plus failed wakes and exits |
| analytics an app's page views and visitors from a terminal, counted at |
| the edge as it serves your app |
| ledger open ledgers: append | head | list | publish | blob | verify |