amadan

carlos / platform Public

apperr: error record, sanitisation and fingerprinting (spec §3.1–3.2)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev 185b01901c11ead5c6f966e6b5c7b3feb12edfb3 parent ae19d84
5 files changed, +320 −0
  • internal/apperr/fingerprint.go +76 −0
  • internal/apperr/fingerprint_test.go +62 −0
  • internal/apperr/record.go +77 −0
  • internal/apperr/sanitize.go +64 −0
  • internal/apperr/sanitize_test.go +41 −0
diff --git a/internal/apperr/fingerprint.go b/internal/apperr/fingerprint.go
new file mode 100644
index 000000000..dadb5cce6
--- /dev/null
+++ b/internal/apperr/fingerprint.go
@@ -0,0 +1,76 @@
+package apperr
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+var (
+ reQuoted = regexp.MustCompile(`"[^"]*"|'[^']*'`)
+ reUUIDAny = regexp.MustCompile(`\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`)
+ reHexRun = regexp.MustCompile(`\b[0-9a-fA-F]{8,}\b`)
+ reDigits4 = regexp.MustCompile(`\b\d{4,}\b`)
+ reLineCol = regexp.MustCompile(`:\d+(:\d+)?`)
+ reBundleJS = regexp.MustCompile(`\.[0-9a-fA-F]{8,}\.js`)
+)
+
+// Normalize folds the parts of a message that vary per occurrence into
+// placeholders on token boundaries (spec §3.2): quoted strings, UUIDs, hex
+// runs of 8+, digit runs of 4+. Tokens of 1–3 digits and mixed tokens
+// (v2, E404) survive, so "HTTP 503" and "retry 3" still separate groups
+// while "user 4123" and "user 9911" do not. Small ids under 1,000 will
+// split groups; that is the stated trade-off, and a caller-supplied FP
+// overrides it.
+func Normalize(s string) string {
+ s = reQuoted.ReplaceAllString(s, `"?"`)
+ s = reUUIDAny.ReplaceAllString(s, "<uuid>")
+ s = reHexRun.ReplaceAllStringFunc(s, func(m string) string {
+ if _, err := strconv.ParseUint(m, 10, 64); err == nil {
+ return "<n>" // all digits: a digit run, not a hex run
+ }
+ return "<hex>"
+ })
+ s = reDigits4.ReplaceAllString(s, "<n>")
+ return s
+}
+
+// TopFrame is the first non-empty line of a stack with its line, column
+// and bundle hash removed, so a redeploy does not split every group.
+func TopFrame(stack string) string {
+ for _, line := range strings.Split(stack, "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+ line = reBundleJS.ReplaceAllString(line, ".js")
+ line = reLineCol.ReplaceAllString(line, "")
+ return line
+ }
+ return ""
+}
+
+// Fingerprint groups occurrences: sha256 over length-delimited (source,
+// kind, Normalize(name), Normalize(TopFrame(stack)))[:16] as hex. For a
+// stack with a message line first (JS "TypeError: x\n at …"), the top
+// frame is the second line; callers pass the message as name and the
+// whole stack here.
+func Fingerprint(source, kind, name, stack string) string {
+ h := sha256.New()
+ frame := TopFrame(stack)
+ if strings.HasPrefix(frame, name) || strings.Contains(frame, ": "+name) {
+ // The stack's first line restates the message; take the next.
+ rest := strings.TrimSpace(stack)
+ if i := strings.IndexByte(rest, '\n'); i >= 0 {
+ frame = TopFrame(rest[i+1:])
+ }
+ }
+ for _, part := range []string{source, kind, Normalize(name), Normalize(frame)} {
+ h.Write([]byte(strconv.Itoa(len(part))))
+ h.Write([]byte{':'})
+ h.Write([]byte(part))
+ }
+ return hex.EncodeToString(h.Sum(nil))[:16]
+}
diff --git a/internal/apperr/fingerprint_test.go b/internal/apperr/fingerprint_test.go
new file mode 100644
index 000000000..756fa87ae
--- /dev/null
+++ b/internal/apperr/fingerprint_test.go
@@ -0,0 +1,62 @@
+package apperr
+
+import "testing"
+
+func TestNormalize(t *testing.T) {
+ cases := map[string]string{
+ "user 4123 not found": "user <n> not found",
+ "user 9911 not found": "user <n> not found",
+ "HTTP 503 from upstream": "HTTP 503 from upstream",
+ "retry 3 failed": "retry 3 failed",
+ "v2 handler panicked": "v2 handler panicked",
+ "E404 for id 550e8400-e29b-41d4-a716-446655440000": "E404 for id <uuid>",
+ `missing key "stripe_secret"`: `missing key "?"`,
+ "trace 0123456789abcdef bad": "trace <hex> bad",
+ "deadbeef cafe": "<hex> cafe",
+ }
+ for in, want := range cases {
+ if got := Normalize(in); got != want {
+ t.Errorf("Normalize(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+func TestTopFrame(t *testing.T) {
+ cases := map[string]string{
+ "TypeError: x\n at render (https://app.example/assets/main.3f2a9c1b.js:12:34)\n at y": "TypeError: x",
+ "\n app/models/user.rb:42:in `find'\n app/x.rb:1": "app/models/user.rb:in `find'",
+ "": "",
+ "single line only": "single line only",
+ }
+ for in, want := range cases {
+ if got := TopFrame(in); got != want {
+ t.Errorf("TopFrame(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+func TestFingerprint(t *testing.T) {
+ a := Fingerprint(SourceLog, KindLog, "user 4123 not found", "")
+ b := Fingerprint(SourceLog, KindLog, "user 9911 not found", "")
+ c := Fingerprint(SourceLog, KindLog, "user gone", "")
+ d := Fingerprint(SourceEdge, KindLog, "user 4123 not found", "")
+ if a != b {
+ t.Fatalf("digit runs must group: %s != %s", a, b)
+ }
+ if a == c || a == d {
+ t.Fatalf("different names or sources must not group: %s %s %s", a, c, d)
+ }
+ if len(a) != 16 {
+ t.Fatalf("fingerprint length %d, want 16 hex", len(a))
+ }
+ // Length-delimited: ("ab","c") and ("a","bc") differ.
+ if Fingerprint("ab", "c", "n", "") == Fingerprint("a", "bc", "n", "") {
+ t.Fatal("components must be length-delimited")
+ }
+ // A redeploy's bundle hash and line numbers do not split a browser group.
+ s1 := "TypeError: x\n at render (https://a/main.3f2a9c1b.js:12:34)"
+ s2 := "TypeError: x\n at render (https://a/main.9b8c7d6e.js:99:1)"
+ if Fingerprint(SourceBrowser, "TypeError", "x", s1) != Fingerprint(SourceBrowser, "TypeError", "x", s2) {
+ t.Fatal("frame normalisation must drop bundle hash and line:col")
+ }
+}
diff --git a/internal/apperr/record.go b/internal/apperr/record.go
new file mode 100644
index 000000000..5dcfb19cd
--- /dev/null
+++ b/internal/apperr/record.go
@@ -0,0 +1,77 @@
+// Package apperr is the pure half of the Errors tab (spec
+// 2026-09-07-errors-and-analytics-design.md §3.1–3.2): the wire record,
+// sanitisation, fingerprinting, the box-side coalescing sink, the chunk
+// read and the grouping fold. No I/O and no clock of its own — the same
+// split internal/applog keeps: internal/host ships, internal/edge and the
+// activator produce, internal/console reads.
+package apperr
+
+import (
+ "time"
+
+ "github.com/carlosframework/platform/internal/applog"
+)
+
+// Sources an error can come from (spec §3.1). log is never stored in the
+// errors/ family — it is derived on read from logs/ (§3.2).
+const (
+ SourceLog = "log"
+ SourceEdge = "edge"
+ SourceServer = "server"
+ SourceBrowser = "browser"
+)
+
+// Kinds the platform itself produces. A server or browser caller sets its
+// own kind (an exception class, typically).
+const (
+ KindHTTP = "http" // an edge-observed 5xx
+ KindLog = "log" // a log line at or above the configured level
+)
+
+// FPVersion stamps the fingerprint algorithm so it can change without
+// merging old groups into new ones.
+const FPVersion = 1
+
+// Record is one coalesced error on the wire — one JSONL line inside an
+// errors/ chunk. Additive shape (AGENTS.md rule): every field but TS,
+// Source, Name and FP is omitempty, unknown keys are ignored on read.
+// Count, First and Last are the coalescing state: one record stands for
+// Count occurrences of FP on Host under Release between First and Last;
+// the other fields are the first occurrence's sample.
+type Record struct {
+ TS time.Time `json:"ts"`
+ Box string `json:"box,omitempty"`
+ Host string `json:"host,omitempty"`
+ Source string `json:"source"`
+ Kind string `json:"kind,omitempty"`
+ Name string `json:"name"`
+ Level string `json:"level,omitempty"` // log source: warn | error
+ Path string `json:"path,omitempty"`
+ Status int `json:"status,omitempty"`
+ Release string `json:"release,omitempty"`
+ Stack string `json:"stack,omitempty"`
+ Fields map[string]string `json:"fields,omitempty"`
+ FP string `json:"fp"`
+ FPV int `json:"fpv,omitempty"`
+ Count int `json:"count,omitempty"`
+ First time.Time `json:"first,omitempty"`
+ Last time.Time `json:"last,omitempty"`
+ Seq uint64 `json:"seq,omitempty"`
+}
+
+// MaxStack bounds a stored stack (spec §3.1).
+const MaxStack = 8 << 10
+
+// FromLog derives the log-source record for one applog line (spec §3.2:
+// read-derived, never stored twice). Fingerprinted on the normalised
+// message alone — a log line has no frame.
+func FromLog(r applog.Record) Record {
+ name := Sanitize(r.Msg)
+ return Record{
+ TS: r.TS, Box: r.Box, Host: r.Host,
+ Source: SourceLog, Kind: KindLog, Name: name, Level: r.Level,
+ Fields: r.Fields,
+ FP: Fingerprint(SourceLog, KindLog, name, ""), FPV: FPVersion,
+ Count: 1, First: r.TS, Last: r.TS, Seq: r.Seq,
+ }
+}
diff --git a/internal/apperr/sanitize.go b/internal/apperr/sanitize.go
new file mode 100644
index 000000000..c0510dbc7
--- /dev/null
+++ b/internal/apperr/sanitize.go
@@ -0,0 +1,64 @@
+package apperr
+
+import (
+ "regexp"
+ "strings"
+)
+
+// Sanitisation is a floor, not a guarantee (spec §3.1): the patterns below
+// catch what secrets look like in practice — addresses, bearer tokens,
+// key=value credentials, long opaque runs — before anything is buffered.
+var (
+ reEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}`)
+ reAuth = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]+`)
+ // key=value and "key": "value" shapes, value up to the next delimiter.
+ reKV = regexp.MustCompile(`(?i)\b(token|key|secret|password|passwd|pwd|api_key|apikey|access_token|auth)("?\s*[=:]\s*"?)[^\s&"',;]+`)
+ // A hex or base64 run of 32+ characters: a digest, a session id, a key.
+ reLong = regexp.MustCompile(`\b[A-Za-z0-9+/_-]{32,}={0,2}`)
+)
+
+// Sanitize redacts secrets in free text: messages, stacks, field values.
+func Sanitize(s string) string {
+ s = reEmail.ReplaceAllString(s, "<email>")
+ s = reAuth.ReplaceAllStringFunc(s, func(m string) string {
+ word := m[:strings.IndexAny(m, " \t")]
+ return word + " <redacted>"
+ })
+ s = reKV.ReplaceAllString(s, "${1}${2}<redacted>")
+ s = reLong.ReplaceAllString(s, "<hex>")
+ return s
+}
+
+var (
+ reUUID = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
+ reHex16 = regexp.MustCompile(`^[0-9a-fA-F]{16,}$`)
+ reB64ish = regexp.MustCompile(`^[A-Za-z0-9+/_-]{20,}={0,2}$`)
+ reEmailNW = regexp.MustCompile(`^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$`)
+)
+
+// SanitizePath strips query and fragment and collapses token-shaped
+// segments to ":token" (spec §3.3): a UUID, a hex run of 16+, a base64-
+// looking run of 20+, or an email. "/blog/2024-review" and "/orders/42"
+// survive; "/reset/8f3a…" does not. Empty is "/".
+//
+// The base64 segment rule requires both a digit and an upper-case letter
+// so that ordinary long slugs ("some-long-page-name-here") survive.
+func SanitizePath(p string) string {
+ if i := strings.IndexAny(p, "?#"); i >= 0 {
+ p = p[:i]
+ }
+ if p == "" {
+ return "/"
+ }
+ segs := strings.Split(p, "/")
+ for i, s := range segs {
+ if s == "" {
+ continue
+ }
+ if reUUID.MatchString(s) || reHex16.MatchString(s) || reEmailNW.MatchString(s) ||
+ (reB64ish.MatchString(s) && strings.ContainsAny(s, "0123456789") && strings.ContainsAny(s, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")) {
+ segs[i] = ":token"
+ }
+ }
+ return strings.Join(segs, "/")
+}
diff --git a/internal/apperr/sanitize_test.go b/internal/apperr/sanitize_test.go
new file mode 100644
index 000000000..6553c6aaa
--- /dev/null
+++ b/internal/apperr/sanitize_test.go
@@ -0,0 +1,41 @@
+package apperr
+
+import "testing"
+
+func TestSanitize(t *testing.T) {
+ cases := map[string]string{
+ "user paul@tito.io not found": "user <email> not found",
+ "Authorization: Bearer abc.def-ghi": "Authorization: Bearer <redacted>",
+ "basic dXNlcjpwYXNz failed": "basic <redacted> failed",
+ "GET /x?token=abc123&y=1": "GET /x?token=<redacted>&y=1",
+ `{"password": "hunter2", "n": 3}`: `{"password": "<redacted>", "n": 3}`,
+ "api_key=sk-live-1234 ok": "api_key=<redacted> ok",
+ "digest 0123456789abcdef0123456789abcdef done": "digest <hex> done",
+ "b64 QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo0MTIzNA== ok": "b64 <hex> ok",
+ "short 4123 stays": "short 4123 stays",
+ "nothing here": "nothing here",
+ }
+ for in, want := range cases {
+ if got := Sanitize(in); got != want {
+ t.Errorf("Sanitize(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+func TestSanitizePath(t *testing.T) {
+ cases := map[string]string{
+ "/reset/8f3a9c1e2b4d6f708a9b0c1d2e3f4a5b?x=1#frag": "/reset/:token",
+ "/users/paul@tito.io/edit": "/users/:token/edit",
+ "/items/550e8400-e29b-41d4-a716-446655440000": "/items/:token",
+ "/s/QUJDREVGR0hJSktMTU5PUFFSU1RV": "/s/:token",
+ "/blog/2024-review": "/blog/2024-review",
+ "/api/v2/orders/42": "/api/v2/orders/42",
+ "/": "/",
+ "": "/",
+ }
+ for in, want := range cases {
+ if got := SanitizePath(in); got != want {
+ t.Errorf("SanitizePath(%q) = %q, want %q", in, got, want)
+ }
+ }
+}