amadan
analytics: HyperLogLog and Misra–Gries sketches (spec §3.3)
Adds internal/analytics, a new leaf package (nothing imports it yet) holding the two cardinality/frequency sketches the Analytics tab's in-memory meter will use: HLL for distinct-visitor counting and TopK for the pages/referers/events top-keys summaries. HLL is a 2^11-register HyperLogLog over 64-bit hashes, with the Flajolet et al. 2007 estimator plus the small-range linear-counting correction, register-max Merge, and an Encode/DecodeHLL wire format (base64 of a version byte plus the raw registers; "" decodes to an empty sketch, an unknown version or size errors). TopK is a Misra–Gries summary: exact below TopKSize counters, a lower bound within the summary's error above it, with evicted mass folded into OtherKey so Fold's total is always conserved. Merge re-applies Add so two full summaries merge back down to the cap. TopKFrom is Fold's inverse for the day-file's wire shape. Deviation from the brief: Add mixes the incoming hash through MurmurHash3's fmix64 finalizer before slicing out the register index and run bits. Without it, TestHLLEstimatesWithinThreeSigma failed at n=100000 (91517 vs 100000, ~8.5% off, versus a 3σ bound of ~6.9%). The register index is the hash's top 11 bits, and the test's own hash function (FNV-1a, given verbatim in the brief) does not mix entropy reliably into its high bits for short, sequential, common-prefixed inputs like "v0".."v99999": only ~23 of 2048 registers ever got touched by 1000 such elements, starving Count's estimate. The small-range correction itself checks out fine (n=1000 already used linear counting and matched theory); the failure was hash-quality, not formula. Adding a finalizer is the standard fix and is transparent to callers already passing a well-mixed hash. Also replaced hll_test.go's brief-supplied crude itoa (reversed-digit encoding) with strconv.Itoa: itoa(99) produced "990000", not "99", which broke TestTopKExactBelowCapAndBoundedAbove's f["/p99"] lookup. Test semantics (unique keys per i) are unchanged. go test ./internal/analytics/ -count=1 -v: PASS (5/5 tests, 5 repeat runs). gofmt and go vet clean. go build ./... succeeds.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 files changed,
+350
−0
internal/analytics/hll.go+116 −0internal/analytics/hll_test.go+71 −0internal/analytics/topk.go+106 −0internal/analytics/topk_test.go+57 −0
diff --git a/internal/analytics/hll.go b/internal/analytics/hll.go| new file mode 100644 |
| index 000000000..43827c342 |
| --- /dev/null |
| +++ b/internal/analytics/hll.go |
| @@ -0,0 +1,116 @@ |
| +// Package analytics is the pure half of the Analytics tab (spec |
| +// 2026-09-07-errors-and-analytics-design.md §3.3): the visitor and top-key |
| +// sketches, the in-memory meter the edge feeds, the day-file wire shape |
| +// and its idempotent CAS merge, the daily salt, the bot list and the |
| +// cross-box range read. internal/edge observes, internal/host flushes, |
| +// internal/console reads — the applog/apperr split. |
| +package analytics |
| + |
| +import ( |
| + "encoding/base64" |
| + "errors" |
| + "math" |
| + "math/bits" |
| +) |
| + |
| +// HLLPrecision is the register-index width: 2^11 registers, 2 KB, ~2.3% |
| +// standard error (spec §3.3). One sketch per app per day per box. |
| +const HLLPrecision = 11 |
| +const hllRegisters = 1 << HLLPrecision |
| + |
| +// HLL is a HyperLogLog cardinality sketch over 64-bit hashes. |
| +type HLL struct{ reg [hllRegisters]uint8 } |
| + |
| +// mix64 is MurmurHash3's fmix64 finalizer. Add uses it to avalanche the |
| +// incoming hash before slicing out the register index and run bits: the |
| +// register index is the hash's top HLLPrecision bits, and callers' hash |
| +// functions (FNV-1a included — see h64 in hll_test.go) do not reliably |
| +// mix entropy into their high bits, especially for short, sequential |
| +// inputs sharing a common prefix ("v0".."v99999"), which otherwise |
| +// starves most registers and skews Count. |
| +func mix64(x uint64) uint64 { |
| + x ^= x >> 33 |
| + x *= 0xff51afd7ed558ccd |
| + x ^= x >> 33 |
| + x *= 0xc4ceb9fe1a85ec53 |
| + x ^= x >> 33 |
| + return x |
| +} |
| + |
| +// Add records one hashed element. |
| +func (h *HLL) Add(hash uint64) { |
| + hash = mix64(hash) |
| + idx := hash >> (64 - HLLPrecision) |
| + rest := hash<<HLLPrecision | 1<<(HLLPrecision-1) // guard bit bounds the run |
| + rank := uint8(bits.LeadingZeros64(rest)) + 1 |
| + if rank > h.reg[idx] { |
| + h.reg[idx] = rank |
| + } |
| +} |
| + |
| +// Merge folds o into h by register max — the union of the two sets. |
| +func (h *HLL) Merge(o *HLL) { |
| + for i := range h.reg { |
| + if o.reg[i] > h.reg[i] { |
| + h.reg[i] = o.reg[i] |
| + } |
| + } |
| +} |
| + |
| +// Empty reports whether nothing was added. |
| +func (h *HLL) Empty() bool { |
| + for _, r := range h.reg { |
| + if r != 0 { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| +// Count estimates the cardinality (Flajolet et al. 2007 with the small- |
| +// range linear-counting correction). |
| +func (h *HLL) Count() float64 { |
| + m := float64(hllRegisters) |
| + alpha := 0.7213 / (1 + 1.079/m) |
| + var sum float64 |
| + zeros := 0 |
| + for _, r := range h.reg { |
| + sum += math.Pow(2, -float64(r)) |
| + if r == 0 { |
| + zeros++ |
| + } |
| + } |
| + est := alpha * m * m / sum |
| + if est <= 2.5*m && zeros > 0 { |
| + est = m * math.Log(m/float64(zeros)) |
| + } |
| + return est |
| +} |
| + |
| +const hllVersion = 1 |
| + |
| +// Encode renders the sketch as base64 of a version byte plus the registers. |
| +func (h *HLL) Encode() string { |
| + b := make([]byte, 1+hllRegisters) |
| + b[0] = hllVersion |
| + copy(b[1:], h.reg[:]) |
| + return base64.StdEncoding.EncodeToString(b) |
| +} |
| + |
| +// DecodeHLL parses Encode's output; "" is an empty sketch, an unknown |
| +// version is an error (a future shape this build cannot read). |
| +func DecodeHLL(s string) (*HLL, error) { |
| + h := &HLL{} |
| + if s == "" { |
| + return h, nil |
| + } |
| + b, err := base64.StdEncoding.DecodeString(s) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if len(b) != 1+hllRegisters || b[0] != hllVersion { |
| + return nil, errors.New("analytics: unknown visitors sketch version or size") |
| + } |
| + copy(h.reg[:], b[1:]) |
| + return h, nil |
| +} |
diff --git a/internal/analytics/hll_test.go b/internal/analytics/hll_test.go| new file mode 100644 |
| index 000000000..8cb2cbe82 |
| --- /dev/null |
| +++ b/internal/analytics/hll_test.go |
| @@ -0,0 +1,71 @@ |
| +package analytics |
| + |
| +import ( |
| + "hash/fnv" |
| + "math" |
| + "strconv" |
| + "testing" |
| +) |
| + |
| +func h64(s string) uint64 { f := fnv.New64a(); f.Write([]byte(s)); return f.Sum64() } |
| + |
| +func TestHLLEstimatesWithinThreeSigma(t *testing.T) { |
| + for _, n := range []int{1000, 10000, 100000} { |
| + var h HLL |
| + for i := 0; i < n; i++ { |
| + h.Add(h64("v" + itoa(i))) |
| + } |
| + got := h.Count() |
| + sigma := 1.04 / math.Sqrt(hllRegisters) * float64(n) |
| + if math.Abs(got-float64(n)) > 3*sigma { |
| + t.Errorf("n=%d: estimate %.0f off by more than 3σ (%.0f)", n, got, 3*sigma) |
| + } |
| + } |
| +} |
| + |
| +func TestHLLMergeIsUnionAndIdempotent(t *testing.T) { |
| + var a, b HLL |
| + for i := 0; i < 5000; i++ { |
| + a.Add(h64("a" + itoa(i))) |
| + b.Add(h64("b" + itoa(i))) |
| + } |
| + var ab HLL |
| + ab.Merge(&a) |
| + ab.Merge(&b) |
| + ab.Merge(&b) // idempotent |
| + if got := ab.Count(); math.Abs(got-10000) > 3*(1.04/math.Sqrt(hllRegisters))*10000 { |
| + t.Fatalf("union estimate %.0f", got) |
| + } |
| + var ba HLL |
| + ba.Merge(&b) |
| + ba.Merge(&a) |
| + if ab.reg != ba.reg { |
| + t.Fatal("merge must be commutative") |
| + } |
| +} |
| + |
| +func TestHLLEncodeRoundTripAndVersion(t *testing.T) { |
| + var h HLL |
| + h.Add(h64("x")) |
| + enc := h.Encode() |
| + if enc[:2] != "AQ" { // base64 of a leading 0x01 byte |
| + t.Fatalf("version prefix missing: %q", enc[:4]) |
| + } |
| + d, err := DecodeHLL(enc) |
| + if err != nil || d.reg != h.reg { |
| + t.Fatalf("round trip: %v", err) |
| + } |
| + if e, err := DecodeHLL(""); err != nil || !e.Empty() { |
| + t.Fatal("empty string must decode to an empty sketch") |
| + } |
| + if _, err := DecodeHLL("AgAA"); err == nil { |
| + t.Fatal("unknown version must error") |
| + } |
| +} |
| + |
| +// itoa is the brief's crude helper, replaced with strconv.Itoa: the |
| +// original digit-reversed encoding broke TestTopKExactBelowCapAndBoundedAbove's |
| +// key lookups (itoa(99) produced "990000", not "99") and had weaker |
| +// diversity in leading digits than strconv's ordinary decimal, which |
| +// stressed the HLL register-index entropy (see hll.go's mix64 note). |
| +func itoa(i int) string { return strconv.Itoa(i) } |
diff --git a/internal/analytics/topk.go b/internal/analytics/topk.go| new file mode 100644 |
| index 000000000..71745ee7c |
| --- /dev/null |
| +++ b/internal/analytics/topk.go |
| @@ -0,0 +1,106 @@ |
| +package analytics |
| + |
| +// TopKSize bounds a top-keys summary (pages, referers, events) per app |
| +// per day per box (spec §3.3). Misra–Gries: exact below the cap, a lower |
| +// bound within the summary's error above it, mergeable, with the evicted |
| +// mass kept in OtherKey so totals are conserved. |
| +const TopKSize = 500 |
| + |
| +// OtherKey is the fold-over row for keys beyond the cap. |
| +const OtherKey = "_other" |
| + |
| +// MaxKeyLen caps a key's bytes. |
| +const MaxKeyLen = 256 |
| + |
| +// TopK is a Misra–Gries summary with TopKSize counters. |
| +type TopK struct { |
| + counts map[string]int64 |
| + other int64 |
| +} |
| + |
| +// NewTopK returns an empty summary. |
| +func NewTopK() *TopK { return &TopK{counts: make(map[string]int64)} } |
| + |
| +// Add counts n occurrences of key. When the summary is full and key is |
| +// new, every counter is decremented by the smallest amount that frees a |
| +// slot, and the decremented mass moves to other. |
| +func (t *TopK) Add(key string, n int64) { |
| + if len(key) > MaxKeyLen { |
| + key = key[:MaxKeyLen] |
| + } |
| + if n <= 0 { |
| + return |
| + } |
| + if _, ok := t.counts[key]; ok || len(t.counts) < TopKSize { |
| + t.counts[key] += n |
| + return |
| + } |
| + // Full and new: take the smallest counter as the decrement. |
| + min := int64(-1) |
| + for _, v := range t.counts { |
| + if min < 0 || v < min { |
| + min = v |
| + } |
| + } |
| + if n <= min { |
| + t.other += n |
| + return |
| + } |
| + for k, v := range t.counts { |
| + if v <= min { |
| + delete(t.counts, k) |
| + t.other += v |
| + } else { |
| + t.counts[k] = v - min |
| + t.other += min |
| + } |
| + } |
| + t.counts[key] = n - min |
| + t.other += min |
| +} |
| + |
| +// Merge folds o into t: counters add, other adds, then t re-folds to the |
| +// cap so a merge of two full summaries stays bounded. |
| +func (t *TopK) Merge(o *TopK) { |
| + merged := NewTopK() |
| + merged.other = t.other + o.other |
| + sum := make(map[string]int64, len(t.counts)+len(o.counts)) |
| + for k, v := range t.counts { |
| + sum[k] += v |
| + } |
| + for k, v := range o.counts { |
| + sum[k] += v |
| + } |
| + for k, v := range sum { // Add re-applies the cap |
| + merged.Add(k, v) |
| + } |
| + t.counts, t.other = merged.counts, merged.other |
| +} |
| + |
| +// Fold returns the counters plus OtherKey when any mass fell over. |
| +func (t *TopK) Fold() map[string]int64 { |
| + out := make(map[string]int64, len(t.counts)+1) |
| + for k, v := range t.counts { |
| + out[k] = v |
| + } |
| + if t.other > 0 { |
| + out[OtherKey] = t.other |
| + } |
| + return out |
| +} |
| + |
| +// TopKFrom rebuilds a summary from a folded map (the day file's shape). |
| +func TopKFrom(m map[string]int64) *TopK { |
| + t := NewTopK() |
| + for k, v := range m { |
| + if k == OtherKey { |
| + t.other += v |
| + continue |
| + } |
| + t.Add(k, v) |
| + } |
| + return t |
| +} |
| + |
| +// Approximate reports whether counters are lower bounds (mass fell over). |
| +func (t *TopK) Approximate() bool { return t.other > 0 } |
diff --git a/internal/analytics/topk_test.go b/internal/analytics/topk_test.go| new file mode 100644 |
| index 000000000..516c78d92 |
| --- /dev/null |
| +++ b/internal/analytics/topk_test.go |
| @@ -0,0 +1,57 @@ |
| +package analytics |
| + |
| +import "testing" |
| + |
| +func TestTopKExactBelowCapAndBoundedAbove(t *testing.T) { |
| + k := NewTopK() |
| + for i := 0; i < 100; i++ { |
| + k.Add("/p"+itoa(i), int64(i+1)) |
| + } |
| + f := k.Fold() |
| + if len(f) != 100 || f["/p99"] != 100 || k.Approximate() { |
| + t.Fatalf("below cap must be exact: len=%d p99=%d approx=%v", len(f), f["/p99"], k.Approximate()) |
| + } |
| + for i := 0; i < 2000; i++ { |
| + k.Add("/q"+itoa(i), 1) |
| + } |
| + f = k.Fold() |
| + if len(f) > TopKSize+1 || !k.Approximate() || f[OtherKey] == 0 { |
| + t.Fatalf("above cap: len=%d approx=%v other=%d", len(f), k.Approximate(), f[OtherKey]) |
| + } |
| + // Total mass is conserved between counters and _other. |
| + var total int64 |
| + for _, v := range f { |
| + total += v |
| + } |
| + if total != 100*101/2+2000 { |
| + t.Fatalf("mass %d", total) |
| + } |
| + // A heavy hitter survives. |
| + if f["/p99"] < 100-2000/TopKSize { |
| + t.Fatalf("heavy hitter undercounted: %d", f["/p99"]) |
| + } |
| +} |
| + |
| +func TestTopKMergeRefolds(t *testing.T) { |
| + a, b := NewTopK(), NewTopK() |
| + for i := 0; i < 400; i++ { |
| + a.Add("/a"+itoa(i), 2) |
| + b.Add("/b"+itoa(i), 3) |
| + } |
| + a.Merge(b) |
| + f := a.Fold() |
| + if len(f) > TopKSize+1 || f[OtherKey] == 0 { |
| + t.Fatalf("merged 800 keys must fold to ≤501: %d", len(f)) |
| + } |
| + r := TopKFrom(f) |
| + if r.other != f[OtherKey] || len(r.counts) != len(f)-1 { |
| + t.Fatal("TopKFrom must invert Fold") |
| + } |
| + long := NewTopK() |
| + long.Add(string(make([]byte, 300)), 1) |
| + for key := range long.Fold() { |
| + if len(key) > MaxKeyLen { |
| + t.Fatal("keys must be capped at 256 bytes") |
| + } |
| + } |
| +} |