carlos / platform Public

apperr, analytics: per-source error budget, sanitised field keys, Dropped.Total

Four fixes from the whole-branch review, all in the record path.

The fingerprint budget is now per app per source rather than per app. The
browser door admits 600 events a minute and every one of them may carry a
fresh fingerprint, so a single shared thousand was gone in two minutes and
every genuine edge 5xx for the rest of that hour folded into _overflow.
Each source now gets its own thousand and its own overflow row, drained
after the groups in sorted source order so the batch stays reproducible.
The wire shape is untouched: Record already carries Source, and the
overflow record is a copy of the record that triggered it.

The Errors tab follows: GroupRecords keys the overflow row by (FP, Source)
so two sources' overflows are two rows wearing their own badges, instead of
one merged row that says nothing about who blew the budget.

Field keys go through Sanitize too, and are capped at MaxFieldKey. A caller
that counts per user by address was leaking it through the key while the
value beside it was scrubbed.

Path "/" is no longer blanked for browser records. That rule exists because
a log line rarely knows the request path; a JS error on the homepage knows
exactly where it happened.

And Dropped.Total sums the nine drop counters in one place — the console
view, the CLI headline and the CLI breakdown were each spelling it out.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Paul Campbell pushed by paul@keymail.dev 1 month ago 44bab238c7d11bf62109c4e252af72d7909c9495 parent 65960d5

9 files changed, +252 −35

analytics_cmd.go +2 −2
internal/analytics/day.go +9 −0
internal/analytics/day_test.go +16 −0
internal/apperr/group.go +15 −5
internal/apperr/group_test.go +29 −0
internal/apperr/record.go +4 −0
internal/apperr/sink.go +64 −27
internal/apperr/sink_test.go +112 −0
internal/console/analyticsview.go +1 −1

diff --git a/analytics_cmd.go b/analytics_cmd.go

index f8f4ded07..1c27202ab 100644

--- a/analytics_cmd.go

+++ b/analytics_cmd.go

@@ -99,12 +99,12 @@ func printAnalytics(w io.Writer, res apiAnalyticsResponse) {

c.head(fmt.Sprintf("visitors %d (daily visitors, summed)", res.Visitors)),

c.muted(fmt.Sprintf("bytes %d", res.Bytes)),

c.muted(fmt.Sprintf("bots %d", res.Bots)),

- c.muted(fmt.Sprintf("dropped %d", res.Dropped.E.Rate+res.Dropped.E.Size+res.Dropped.E.Origin+res.Dropped.E.Malformed+res.Dropped.A.Rate+res.Dropped.A.Size+res.Dropped.A.Origin+res.Dropped.A.Malformed+res.Dropped.Flush)))

+ c.muted(fmt.Sprintf("dropped %d", res.Dropped.Total())))

// A breakdown only when something was actually dropped: a silent app

// must not print a line of zeros.

d := res.Dropped

- if total := d.E.Rate + d.E.Size + d.E.Origin + d.E.Malformed + d.A.Rate + d.A.Size + d.A.Origin + d.A.Malformed + d.Flush; total > 0 {

+ if d.Total() > 0 {

mutedf(w, "dropped by reason — errors: rate %d, size %d, origin %d, malformed %d; events: rate %d, size %d, origin %d, malformed %d; flush %d",

d.E.Rate, d.E.Size, d.E.Origin, d.E.Malformed, d.A.Rate, d.A.Size, d.A.Origin, d.A.Malformed, d.Flush)

}

diff --git a/internal/analytics/day.go b/internal/analytics/day.go

index d016afc8f..546ad5c03 100644

--- a/internal/analytics/day.go

+++ b/internal/analytics/day.go

@@ -33,6 +33,15 @@ type Dropped struct {

Flush int64 `json:"flush"`

}

+// Total sums the nine drop counters — the honesty headline. A template

+// cannot add a struct up and three callers were spelling the sum out by

+// hand; one of them is enough.

+func (d Dropped) Total() int64 {

+ e, a := d.E, d.A

+ return e.Rate + e.Size + e.Origin + e.Malformed +

+ a.Rate + a.Size + a.Origin + a.Malformed + d.Flush

// Day is apps/<sqid>/<app>/analytics/<box>/days/<date>.json (spec §3.3):

// one box's counting for one app for one UTC day. Additive shape.

type Day struct {

diff --git a/internal/analytics/day_test.go b/internal/analytics/day_test.go

index f702d4000..6d286743a 100644

--- a/internal/analytics/day_test.go

+++ b/internal/analytics/day_test.go

@@ -92,3 +92,19 @@ func TestUpsertDayCASAcrossBoxes(t *testing.T) {

}

func contains(s, sub string) bool { return strings.Contains(s, sub) }

+// TestDroppedTotal pins the honesty headline: all nine counters, once.

+func TestDroppedTotal(t *testing.T) {

+ var d Dropped

+ if d.Total() != 0 {

+ t.Fatalf("zero Dropped totals %d", d.Total())

+ }

+ d = Dropped{

+ E: DropCounts{Rate: 1, Size: 2, Origin: 4, Malformed: 8},

+ A: DropCounts{Rate: 16, Size: 32, Origin: 64, Malformed: 128},

+ Flush: 256,

+ }

+ if got := d.Total(); got != 511 {

+ t.Fatalf("Total = %d, want 511 (each counter counted exactly once)", got)

+ }

diff --git a/internal/apperr/group.go b/internal/apperr/group.go

index d74cf81e2..c4eaee892 100644

--- a/internal/apperr/group.go

+++ b/internal/apperr/group.go

@@ -24,10 +24,20 @@ type Group struct {

// GroupRecords folds records into groups by FP, newest Last first. The

// sample is the earliest record of the group (its stack and fields are

// the first occurrence's).

+//

+// The _overflow row is the exception: the sink's fingerprint budget is per

+// app per source, so one window can hold a browser overflow and an edge

+// overflow, and folding both into one row would hide which source blew its

+// budget. Those group by (FP, Source) instead, so each source's overflow is

+// its own row wearing its own badge.

func GroupRecords(recs []Record) []Group {

byFP := make(map[string]*Group)

var order []string

for _, r := range recs {

+ key := r.FP

+ if r.FP == OverflowFP {

+ key = r.FP + "\x00" + r.Source

+ }

first, last := r.First, r.Last

if first.IsZero() {

first = r.TS

@@ -39,11 +49,11 @@ func GroupRecords(recs []Record) []Group {

if count == 0 {

count = 1

}

- g := byFP[r.FP]

+ g := byFP[key]

if g == nil {

g = &Group{FP: r.FP, Source: r.Source, Kind: r.Kind, Name: r.Name, Level: r.Level, First: first, Last: last, Sample: r}

- byFP[r.FP] = g

- order = append(order, r.FP)

+ byFP[key] = g

+ order = append(order, key)

}

g.Count += count

if first.Before(g.First) {

@@ -57,8 +67,8 @@ func GroupRecords(recs []Record) []Group {

g.Releases = addUnique(g.Releases, r.Release)

}

out := make([]Group, 0, len(order))

- for _, fp := range order {

- g := byFP[fp]

+ for _, key := range order {

+ g := byFP[key]

sort.Strings(g.Hosts)

sort.Strings(g.Releases)

out = append(out, *g)

diff --git a/internal/apperr/group_test.go b/internal/apperr/group_test.go

index 354f0418e..573b72d7f 100644

--- a/internal/apperr/group_test.go

+++ b/internal/apperr/group_test.go

@@ -31,3 +31,32 @@ func TestGroupRecords(t *testing.T) {

t.Fatal("sample must be the earliest record")

}

+// TestGroupRecordsOverflowPerSource: the fingerprint budget is per source,

+// so a window can hold one overflow row per source and the tab must show

+// both — each with its own badge — rather than one merged mystery row.

+func TestGroupRecordsOverflowPerSource(t *testing.T) {

+ now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC)

+ recs := []Record{

+ {FP: OverflowFP, Source: SourceBrowser, Name: "TypeError x", Host: "h1", Count: 12, First: now, Last: now},

+ {FP: OverflowFP, Source: SourceEdge, Name: "500 GET /z", Host: "h1", Count: 40, First: now, Last: now},

+ {FP: OverflowFP, Source: SourceBrowser, Name: "TypeError y", Host: "h2", Count: 3, First: now, Last: now},

+ }

+ gs := GroupRecords(recs)

+ if len(gs) != 2 {

+ t.Fatalf("groups = %d, want one overflow row per source: %+v", len(gs), gs)

+ }

+ bySource := map[string]Group{}

+ for _, g := range gs {

+ if g.FP != OverflowFP {

+ t.Fatalf("group FP = %q, want the overflow fingerprint kept on the row", g.FP)

+ }

+ bySource[g.Source] = g

+ }

+ if b := bySource[SourceBrowser]; b.Count != 15 {

+ t.Fatalf("browser overflow count = %d, want 15", b.Count)

+ }

+ if e := bySource[SourceEdge]; e.Count != 40 {

+ t.Fatalf("edge overflow count = %d, want 40", e.Count)

+ }

diff --git a/internal/apperr/record.go b/internal/apperr/record.go

index 5dcfb19cd..fc74147a3 100644

--- a/internal/apperr/record.go

+++ b/internal/apperr/record.go

@@ -62,6 +62,10 @@ type Record struct {

// MaxStack bounds a stored stack (spec §3.1).

const MaxStack = 8 << 10

+// MaxFieldKey bounds a stored field key, matching the door's own cap in

+// carlos.js so a key that survives one end survives the other.

+const MaxFieldKey = 64

// FromLog derives the log-source record for one applog line (spec §3.2:

// read-derived, never stored twice). Fingerprinted on the normalised

// message alone — a log line has no frame.

diff --git a/internal/apperr/sink.go b/internal/apperr/sink.go

index 55e80a85e..fa7d3be1f 100644

--- a/internal/apperr/sink.go

+++ b/internal/apperr/sink.go

@@ -1,18 +1,23 @@

package apperr

import (

+ "sort"

"sync"

"time"

"github.com/carlosframework/platform/internal/applog"

)

-// MaxGroupsPerAppHour caps coalesced records per app per UTC hour, counting

-// distinct (fp, host, release) combinations admitted within the hour, across

-// every drain (spec §3.2) — not admissions since the last drain. A key once

-// admitted this hour stays admitted; only genuinely new keys count against

-// the cap. Per app, not per box: one tenant's storm must not suppress a

-// neighbour's.

+// MaxGroupsPerAppHour caps coalesced records per app per source per UTC

+// hour, counting distinct (fp, host, release) combinations admitted within

+// the hour, across every drain (spec §3.2) — not admissions since the last

+// drain. A key once admitted this hour stays admitted; only genuinely new

+// keys count against the cap. Per app, not per box: one tenant's storm must

+// not suppress a neighbour's. Per source too: the browser door admits up to

+// 600 events a minute, each free to carry a fresh fingerprint, so a single

+// shared budget would be gone in two minutes and every genuine edge 5xx for

+// the rest of the hour would fold into _overflow. Each source gets its own

+// thousand and its own overflow row.

const MaxGroupsPerAppHour = 1000

// OverflowFP names the one row the excess beyond the cap folds into. Its

@@ -37,12 +42,15 @@ type groupKey struct{ fp, host, release string }

type appBuf struct {

groups map[groupKey]*Record

- order []groupKey // insertion order, for a deterministic drain

- overflow *Record

- hour string // the UTC hour `seen` counts for

- seen map[groupKey]bool // keys admitted this hour, across every drain — the cap counter; Drain must not touch it

+ order []groupKey // insertion order, for a deterministic drain

+ overflow map[string]*Record // source -> that source's overflow row

+ hour string // the UTC hour `seen` counts for

+ seen map[string]seenKeys // source -> keys admitted this hour, across every drain — the cap counter; Drain must not touch it

}

+// seenKeys is one source's admitted-this-hour set.

+type seenKeys map[groupKey]bool

// NewSink returns a Sink stamping records with box and timestamps from now.

// seq starts at UnixMilli for the reason applog.NewSink gives: chunk keys

// embed the first record's seq, and a restart must sort after last boot.

@@ -56,7 +64,11 @@ func (s *Sink) Add(acct, app string, r Record) {

now := s.now().UTC()

r.Name = Sanitize(r.Name)

r.Path = SanitizePath(r.Path)

- if r.Path == "/" && r.Status == 0 && r.Source != SourceEdge {

+ // A log line rarely knows the request path, so a bare "/" from a log

+ // (or a server) record is noise and gets blanked. Edge and browser

+ // records do know: a 404 on "/" and a JS error on the homepage both

+ // happened at "/" and must keep it.

+ if r.Path == "/" && r.Status == 0 && r.Source != SourceEdge && r.Source != SourceBrowser {

r.Path = ""

}

if len(r.Stack) > MaxStack {

@@ -66,6 +78,14 @@ func (s *Sink) Add(acct, app string, r Record) {

if len(r.Fields) > 0 {

f := make(map[string]string, len(r.Fields))

for k, v := range r.Fields {

+ // Keys carry secrets as readily as values — a caller that

+ // logs an address as the key of a per-user counter leaks it

+ // otherwise — so both ends go through Sanitize, and the key

+ // is capped like the other short strings.

+ k = Sanitize(k)

+ if len(k) > MaxFieldKey {

+ k = k[:MaxFieldKey]

+ }

f[k] = Sanitize(v)

}

r.Fields = f

@@ -86,7 +106,12 @@ func (s *Sink) Add(acct, app string, r Record) {

s.apps[applog.AppKey{Acct: acct, App: app}] = b

}

if h := applog.HourOf(now); h != b.hour {

- b.hour, b.seen = h, make(map[groupKey]bool)

+ b.hour, b.seen = h, make(map[string]seenKeys)

+ }

+ seen := b.seen[r.Source]

+ if seen == nil {

+ seen = make(seenKeys)

+ b.seen[r.Source] = seen

}

k := groupKey{r.FP, r.Host, r.Release}

if g := b.groups[k]; g != nil {

@@ -94,21 +119,26 @@ func (s *Sink) Add(acct, app string, r Record) {

g.Last = now

return

}

- if !b.seen[k] && len(b.seen) >= MaxGroupsPerAppHour {

- if b.overflow == nil {

- o := r

- o.FP = OverflowFP

- o.TS, o.First, o.Last, o.Count = now, now, now, 0

- o.Box = s.box

+ if !seen[k] && len(seen) >= MaxGroupsPerAppHour {

+ o := b.overflow[r.Source]

+ if o == nil {

+ over := r

+ over.FP = OverflowFP

+ over.TS, over.First, over.Last, over.Count = now, now, now, 0

+ over.Box = s.box

s.seq++

- o.Seq = s.seq

- b.overflow = &o

+ over.Seq = s.seq

+ o = &over

+ if b.overflow == nil {

+ b.overflow = make(map[string]*Record)

+ }

+ b.overflow[r.Source] = o

}

- b.overflow.Count++

- b.overflow.Last = now

+ o.Count++

+ o.Last = now

return

}

- b.seen[k] = true

+ seen[k] = true

s.seq++

r.TS, r.First, r.Last, r.Count = now, now, now, 1

r.Box = s.box

@@ -127,15 +157,22 @@ func (s *Sink) Drain() map[applog.AppKey][]Record {

defer s.mu.Unlock()

out := make(map[applog.AppKey][]Record)

for k, b := range s.apps {

- if len(b.order) == 0 && b.overflow == nil {

+ if len(b.order) == 0 && len(b.overflow) == 0 {

continue

}

- recs := make([]Record, 0, len(b.order)+1)

+ recs := make([]Record, 0, len(b.order)+len(b.overflow))

for _, gk := range b.order {

recs = append(recs, *b.groups[gk])

}

- if b.overflow != nil {

- recs = append(recs, *b.overflow)

+ // Overflow rows have no insertion order to follow, so sort them by

+ // source: a drain must be reproducible whatever the map walk does.

+ srcs := make([]string, 0, len(b.overflow))

+ for src := range b.overflow {

+ srcs = append(srcs, src)

+ }

+ sort.Strings(srcs)

+ for _, src := range srcs {

+ recs = append(recs, *b.overflow[src])

}

out[k] = recs

b.groups = make(map[groupKey]*Record)

diff --git a/internal/apperr/sink_test.go b/internal/apperr/sink_test.go

index 8fa9a3bcf..fa2831dcd 100644

--- a/internal/apperr/sink_test.go

+++ b/internal/apperr/sink_test.go

@@ -2,6 +2,7 @@ package apperr

import (

"fmt"

+ "strings"

"testing"

"time"

@@ -170,3 +171,114 @@ func TestSinkCapCountsDistinctKeysAcrossDrains(t *testing.T) {

t.Fatalf("next hour = %+v", recs)

}

+// TestSinkSanitisesFieldKeys: a key carries a secret as readily as a value.

+func TestSinkSanitisesFieldKeys(t *testing.T) {

+ now := time.Date(2026, 9, 7, 10, 0, 0, 0, time.UTC)

+ s := NewSink("box1", func() time.Time { return now })

+ longKey := strings.Repeat("k", MaxFieldKey+20)

+ s.Add("bab", "web", Record{Source: SourceServer, Kind: "RuntimeError", Name: "boom",

+ Fields: map[string]string{"paul@tito.io": "1", longKey: "2"}})

+ r := s.Drain()[applog.AppKey{Acct: "bab", App: "web"}][0]

+ if r.Fields["<email>"] != "1" {

+ t.Fatalf("field keys = %v, want the email key stored as <email>", r.Fields)

+ }

+ if _, ok := r.Fields["paul@tito.io"]; ok {

+ t.Fatalf("raw email key survived: %v", r.Fields)

+ }

+ if r.Fields[strings.Repeat("k", MaxFieldKey)] != "2" {

+ t.Fatalf("long key not capped at %d: %v", MaxFieldKey, r.Fields)

+ }

+// TestSinkKeepsBrowserRootPath: the "/" blanking rule is for log records,

+// which rarely know a path. A JS error on the homepage happened at "/".

+func TestSinkKeepsBrowserRootPath(t *testing.T) {

+ now := time.Date(2026, 9, 7, 10, 0, 0, 0, time.UTC)

+ s := NewSink("box1", func() time.Time { return now })

+ s.Add("bab", "web", Record{Source: SourceBrowser, Kind: "TypeError", Name: "x is not a function", Path: "/"})

+ s.Add("bab", "web", Record{Source: SourceLog, Kind: KindLog, Name: "some log line", Path: "/"})

+ var browser, logrec Record

+ for _, r := range s.Drain()[applog.AppKey{Acct: "bab", App: "web"}] {

+ switch r.Source {

+ case SourceBrowser:

+ browser = r

+ case SourceLog:

+ logrec = r

+ }

+ if browser.Path != "/" {

+ t.Fatalf("browser path = %q, want /", browser.Path)

+ }

+ if logrec.Path != "" {

+ t.Fatalf("log path = %q, want blank", logrec.Path)

+ }

+// TestSinkCapIsPerSource is the flood the browser door makes possible: 1,001

+// distinct browser fingerprints in one hour must not cost the edge its own

+// budget. The browser gets its 1,000 rows plus its own overflow; a genuine

+// edge 5xx arriving afterwards is still admitted as itself.

+func TestSinkCapIsPerSource(t *testing.T) {

+ now := time.Date(2026, 9, 7, 10, 0, 0, 0, time.UTC)

+ s := NewSink("box1", func() time.Time { return now })

+ for i := 0; i <= MaxGroupsPerAppHour; i++ {

+ s.Add("bab", "web", Record{Source: SourceBrowser, Kind: "TypeError", Name: "boom", FP: fmt.Sprintf("b%d", i)})

+ }

+ s.Add("bab", "web", Record{Source: SourceEdge, Kind: KindHTTP, Name: "500 GET /x", FP: "edge1", Status: 500})

+ var browsers, overflows, edges int

+ var edge Record

+ for _, r := range s.Drain()[applog.AppKey{Acct: "bab", App: "web"}] {

+ switch {

+ case r.FP == OverflowFP:

+ overflows++

+ if r.Source != SourceBrowser {

+ t.Fatalf("overflow source = %q, want browser", r.Source)

+ }

+ if r.Count != 1 {

+ t.Fatalf("browser overflow count = %d, want 1", r.Count)

+ }

+ case r.Source == SourceBrowser:

+ browsers++

+ case r.Source == SourceEdge:

+ edges++

+ edge = r

+ }

+ if browsers != MaxGroupsPerAppHour || overflows != 1 {

+ t.Fatalf("browser rows = %d, overflow rows = %d, want %d and 1", browsers, overflows, MaxGroupsPerAppHour)

+ }

+ if edges != 1 || edge.FP != "edge1" {

+ t.Fatalf("edge rows = %d (%+v), want the 5xx admitted normally", edges, edge)

+ }

+// TestSinkOverflowRowPerSource: two sources both over budget drain as two

+// overflow rows, in a deterministic source order.

+func TestSinkOverflowRowPerSource(t *testing.T) {

+ now := time.Date(2026, 9, 7, 10, 0, 0, 0, time.UTC)

+ s := NewSink("box1", func() time.Time { return now })

+ for _, src := range []string{SourceBrowser, SourceEdge} {

+ for i := 0; i < MaxGroupsPerAppHour+3; i++ {

+ s.Add("bab", "web", Record{Source: src, Kind: KindHTTP, Name: "boom", FP: fmt.Sprintf("%s-%d", src, i)})

+ }

+ var over []Record

+ for _, r := range s.Drain()[applog.AppKey{Acct: "bab", App: "web"}] {

+ if r.FP == OverflowFP {

+ over = append(over, r)

+ }

+ if len(over) != 2 {

+ t.Fatalf("overflow rows = %d, want one per source", len(over))

+ }

+ if over[0].Source != SourceBrowser || over[1].Source != SourceEdge {

+ t.Fatalf("overflow order = %q, %q; want browser then edge (sorted)", over[0].Source, over[1].Source)

+ }

+ for _, r := range over {

+ if r.Count != 3 {

+ t.Fatalf("%s overflow count = %d, want 3", r.Source, r.Count)

+ }

diff --git a/internal/console/analyticsview.go b/internal/console/analyticsview.go

index 19c35da86..a4676ce39 100644

--- a/internal/console/analyticsview.go

+++ b/internal/console/analyticsview.go

@@ -282,7 +282,7 @@ func (s *Server) readAnalytics(ctx context.Context, sqid, app, rng string, now t

// nobody has pointed the snippet at yet must not claim a source it has

// not seen (ruling 7).

d := out.Sum.Dropped

- out.DroppedTotal = d.E.Rate + d.E.Size + d.E.Origin + d.E.Malformed + d.A.Rate + d.A.Size + d.A.Origin + d.A.Malformed + d.Flush

+ out.DroppedTotal = d.Total()

if len(out.Sum.Events) > 0 || out.DroppedTotal-d.Flush > 0 {

out.Sources = append(out.Sources, "browser")

}