amadan

carlos / platform Public

analytics: bot list, day salt, day file with seq-idempotent CAS merge (spec §3.3)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev f763a2b051b2095de9563b9d3d5604967807ed10 parent 4200c32
6 files changed, +381 −0
  • internal/analytics/bots.go +32 −0
  • internal/analytics/bots_test.go +27 −0
  • internal/analytics/day.go +165 −0
  • internal/analytics/day_test.go +76 −0
  • internal/analytics/salt.go +48 −0
  • internal/analytics/salt_test.go +33 −0
diff --git a/internal/analytics/bots.go b/internal/analytics/bots.go
new file mode 100644
index 000000000..195bfca0c
--- /dev/null
+++ b/internal/analytics/bots.go
@@ -0,0 +1,32 @@
+package analytics
+
+import "strings"
+
+// botMarkers is the vendored user-agent list (spec §3.3): incomplete by
+// nature, so the tab shows the bots count rather than pretending. Matched
+// case-insensitively as substrings. An empty UA is a bot: no browser sends
+// none.
+var botMarkers = []string{
+ "bot", "crawler", "spider", "slurp", "curl/", "wget/", "python-requests",
+ "go-http-client", "headlesschrome", "phantomjs", "facebookexternalhit",
+ "ahrefs", "semrush", "mj12", "dotbot", "petalbot", "bytespider", "gptbot",
+ "claudebot", "ccbot", "applebot", "yandex", "baiduspider", "duckduckbot",
+ "linkedinbot", "twitterbot", "discordbot", "telegrambot", "whatsapp",
+ "pingdom", "uptimerobot", "statuscake", "site24x7", "newrelicpinger",
+ "lighthouse", "pagespeed", "gtmetrix", "httpclient", "okhttp", "java/",
+ "libwww", "scrapy", "node-fetch", "axios/", "postmanruntime", "insomnia",
+}
+
+// IsBot reports whether ua looks like an automated client.
+func IsBot(ua string) bool {
+ if ua == "" {
+ return true
+ }
+ l := strings.ToLower(ua)
+ for _, m := range botMarkers {
+ if strings.Contains(l, m) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/analytics/bots_test.go b/internal/analytics/bots_test.go
new file mode 100644
index 000000000..b01378a63
--- /dev/null
+++ b/internal/analytics/bots_test.go
@@ -0,0 +1,27 @@
+package analytics
+
+import "testing"
+
+func TestIsBot(t *testing.T) {
+ yes := []string{
+ "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+ "Mozilla/5.0 (compatible; bingbot/2.0)", "curl/8.4.0", "Wget/1.21",
+ "python-requests/2.31", "Go-http-client/1.1", "Mozilla/5.0 HeadlessChrome/120",
+ "facebookexternalhit/1.1", "Slackbot-LinkExpanding", "AhrefsBot/7.0", "",
+ }
+ no := []string{
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148",
+ }
+ for _, ua := range yes {
+ if !IsBot(ua) {
+ t.Errorf("IsBot(%q) = false", ua)
+ }
+ }
+ for _, ua := range no {
+ if IsBot(ua) {
+ t.Errorf("IsBot(%q) = true", ua)
+ }
+ }
+}
diff --git a/internal/analytics/day.go b/internal/analytics/day.go
new file mode 100644
index 000000000..14614b3d0
--- /dev/null
+++ b/internal/analytics/day.go
@@ -0,0 +1,165 @@
+package analytics
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+
+ "github.com/carlosframework/platform/internal/applog"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// HourRow is one UTC hour's counters.
+type HourRow struct {
+ PV int64 `json:"pv"`
+ Bots int64 `json:"bots"`
+ Bytes int64 `json:"bytes"`
+}
+
+// DropCounts is why events were dropped at a door (steps 3–4 fill them).
+type DropCounts struct {
+ Rate int64 `json:"rate"`
+ Size int64 `json:"size"`
+ Origin int64 `json:"origin"`
+ Malformed int64 `json:"malformed"`
+}
+
+// Dropped is the honesty block (spec §3.3).
+type Dropped struct {
+ E DropCounts `json:"e"`
+ A DropCounts `json:"a"`
+ Flush int64 `json:"flush"`
+}
+
+// Day is apps/<sqid>/<app>/analytics/<box>/days/<date>.json (spec §3.3):
+// one box's counting for one app for one UTC day. Additive shape.
+type Day struct {
+ V int `json:"v"`
+ Date string `json:"date"`
+ Hours map[string]HourRow `json:"hours"`
+ Visitors string `json:"visitors"`
+ Pages map[string]int64 `json:"pages"`
+ Referers map[string]int64 `json:"referers"`
+ Events map[string]int64 `json:"events"`
+ Seq map[string]int64 `json:"seq"`
+ Dropped Dropped `json:"dropped"`
+ Countries map[string]int64 `json:"countries"`
+ Restarted bool `json:"restarted,omitempty"`
+}
+
+// Delta is one flush batch from one box for one app-day.
+type Delta struct {
+ Date string
+ Box string
+ Seq int64
+ Hours map[string]HourRow
+ Visitors *HLL
+ Pages *TopK
+ Referers *TopK
+ Events *TopK
+ Dropped Dropped
+ Restarted bool
+}
+
+// Prefix is the app's analytics/ subtree.
+func Prefix(sqid, app string) string { return applog.FamilyPrefix(applog.FamilyAnalytics, sqid, app) }
+
+// BoxPrefix is one box's day files.
+func BoxPrefix(sqid, app, box string) string { return Prefix(sqid, app) + box + "/days/" }
+
+// DayKey is one day file.
+func DayKey(sqid, app, box, date string) string { return BoxPrefix(sqid, app, box) + date + ".json" }
+
+// Merge folds x into d unless x.Seq is not above the recorded seq for
+// x.Box — the idempotency rule that makes a retried conditional put safe.
+func (d *Day) Merge(x Delta) (bool, error) {
+ if d.Seq == nil {
+ d.Seq = map[string]int64{}
+ }
+ if x.Seq <= d.Seq[x.Box] {
+ return false, nil
+ }
+ d.V, d.Date = 1, x.Date
+ if d.Hours == nil {
+ d.Hours = map[string]HourRow{}
+ }
+ for h, row := range x.Hours {
+ cur := d.Hours[h]
+ cur.PV += row.PV
+ cur.Bots += row.Bots
+ cur.Bytes += row.Bytes
+ d.Hours[h] = cur
+ }
+ vis, err := DecodeHLL(d.Visitors)
+ if err != nil {
+ return false, err
+ }
+ if x.Visitors != nil {
+ vis.Merge(x.Visitors)
+ }
+ d.Visitors = vis.Encode()
+ d.Pages = mergeTop(d.Pages, x.Pages)
+ d.Referers = mergeTop(d.Referers, x.Referers)
+ d.Events = mergeTop(d.Events, x.Events)
+ d.Dropped.E = addDrops(d.Dropped.E, x.Dropped.E)
+ d.Dropped.A = addDrops(d.Dropped.A, x.Dropped.A)
+ d.Dropped.Flush += x.Dropped.Flush
+ d.Restarted = d.Restarted || x.Restarted
+ if d.Countries == nil {
+ d.Countries = map[string]int64{}
+ }
+ d.Seq[x.Box] = x.Seq
+ return true, nil
+}
+
+func mergeTop(m map[string]int64, t *TopK) map[string]int64 {
+ k := TopKFrom(m)
+ if t != nil {
+ k.Merge(t)
+ }
+ return k.Fold()
+}
+
+func addDrops(a, b DropCounts) DropCounts {
+ return DropCounts{a.Rate + b.Rate, a.Size + b.Size, a.Origin + b.Origin, a.Malformed + b.Malformed}
+}
+
+// UpsertDay CAS-merges x into the app-day object for x.Box: read (etag),
+// Merge, conditional put; on ErrCASConflict re-read and retry, three
+// attempts. A no-op merge (seq already recorded) writes nothing.
+func UpsertDay(ctx context.Context, s release.Store, sqid, app string, x Delta) error {
+ key := DayKey(sqid, app, x.Box, x.Date)
+ for attempt := 0; attempt < 3; attempt++ {
+ var day Day
+ raw, etag, err := s.Get(ctx, key)
+ switch {
+ case err == nil:
+ if uerr := json.Unmarshal(raw, &day); uerr != nil {
+ return fmt.Errorf("analytics: parse %s: %w", key, uerr)
+ }
+ case errors.Is(err, os.ErrNotExist):
+ etag = ""
+ default:
+ return err
+ }
+ applied, err := day.Merge(x)
+ if err != nil {
+ return err
+ }
+ if !applied {
+ return nil
+ }
+ out, err := json.Marshal(day)
+ if err != nil {
+ return err
+ }
+ if _, err := s.Put(ctx, key, out, etag); err == nil {
+ return nil
+ } else if !errors.Is(err, release.ErrCASConflict) {
+ return err
+ }
+ }
+ return fmt.Errorf("analytics: %s: lost CAS race three times", key)
+}
diff --git a/internal/analytics/day_test.go b/internal/analytics/day_test.go
new file mode 100644
index 000000000..2990a0479
--- /dev/null
+++ b/internal/analytics/day_test.go
@@ -0,0 +1,76 @@
+package analytics
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func delta(box string, seq int64, pv int64) Delta {
+ h := &HLL{}
+ h.Add(h64(box + itoa(int(seq))))
+ p := NewTopK()
+ p.Add("/", pv)
+ return Delta{Date: "2026-09-07", Box: box, Seq: seq, Hours: map[string]HourRow{"09": {PV: pv, Bytes: pv * 100}},
+ Visitors: h, Pages: p, Referers: NewTopK(), Events: NewTopK()}
+}
+
+func TestDayMergeIsSeqIdempotent(t *testing.T) {
+ var d Day
+ if ok, err := d.Merge(delta("box-a", 5, 10)); !ok || err != nil {
+ t.Fatalf("first merge: %v %v", ok, err)
+ }
+ if ok, _ := d.Merge(delta("box-a", 5, 10)); ok {
+ t.Fatal("replaying the same seq must be a no-op")
+ }
+ if ok, _ := d.Merge(delta("box-a", 4, 10)); ok {
+ t.Fatal("an older seq must be a no-op")
+ }
+ if ok, _ := d.Merge(delta("box-b", 1, 7)); !ok {
+ t.Fatal("another box's seq is independent")
+ }
+ if d.Hours["09"].PV != 17 || d.Pages["/"] != 17 || d.Seq["box-a"] != 5 || d.Seq["box-b"] != 1 || d.V != 1 || d.Date != "2026-09-07" {
+ t.Fatalf("merged = %+v", d)
+ }
+ vis, _ := DecodeHLL(d.Visitors)
+ if c := vis.Count(); c < 1.5 || c > 2.5 {
+ t.Fatalf("visitors ≈ 2, got %.2f", c)
+ }
+ b, _ := json.Marshal(d)
+ for _, want := range []string{`"v":1`, `"hours"`, `"visitors"`, `"pages"`, `"referers"`, `"events"`, `"seq"`, `"dropped"`, `"countries"`} {
+ if !contains(string(b), want) {
+ t.Errorf("wire missing %s in %s", want, b)
+ }
+ }
+}
+
+func TestUpsertDayCASAcrossBoxes(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ if err := UpsertDay(ctx, s, "bab", "web", delta("box-a", 1, 3)); err != nil {
+ t.Fatal(err)
+ }
+ if err := UpsertDay(ctx, s, "bab", "web", delta("box-a", 2, 4)); err != nil {
+ t.Fatal(err)
+ }
+ if err := UpsertDay(ctx, s, "bab", "web", delta("box-a", 2, 4)); err != nil { // retry of an ambiguous put
+ t.Fatal(err)
+ }
+ raw, _, err := s.Get(ctx, DayKey("bab", "web", "box-a", "2026-09-07"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var d Day
+ json.Unmarshal(raw, &d)
+ if d.Hours["09"].PV != 7 || d.Seq["box-a"] != 2 {
+ t.Fatalf("day = %+v", d)
+ }
+ if DayKey("bab", "web", "box-a", "2026-09-07") != "apps/bab/web/analytics/box-a/days/2026-09-07.json" {
+ t.Fatal(DayKey("bab", "web", "box-a", "2026-09-07"))
+ }
+}
+
+func contains(s, sub string) bool { return strings.Contains(s, sub) }
diff --git a/internal/analytics/salt.go b/internal/analytics/salt.go
new file mode 100644
index 000000000..8272958fe
--- /dev/null
+++ b/internal/analytics/salt.go
@@ -0,0 +1,48 @@
+package analytics
+
+import (
+ "crypto/rand"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+const saltPrefix = "analytics-salt-"
+
+// DaySalt returns the box's visitor salt for date (YYYY-MM-DD), creating
+// <dir>/analytics-salt-<date> with 32 random bytes at 0600 when absent and
+// deleting any other day's file (spec §3.3: restart-stable within the day,
+// gone on rotation, never in the bucket). dir is the box data directory.
+func DaySalt(dir, date string) ([]byte, error) {
+ path := filepath.Join(dir, saltPrefix+date)
+ if b, err := os.ReadFile(path); err == nil && len(b) == 32 {
+ return b, nil
+ }
+ b := make([]byte, 32)
+ if _, err := rand.Read(b); err != nil {
+ return nil, err
+ }
+ f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ if errors.Is(err, os.ErrExist) { // a sibling process won the race
+ return os.ReadFile(path)
+ }
+ return nil, err
+ }
+ _, werr := f.Write(b)
+ cerr := f.Close()
+ if werr != nil || cerr != nil {
+ os.Remove(path)
+ return nil, errors.Join(werr, cerr)
+ }
+ // Rotation: every other day's salt goes, so nothing on disk can join a
+ // hash across days.
+ entries, _ := os.ReadDir(dir)
+ for _, e := range entries {
+ if n := e.Name(); strings.HasPrefix(n, saltPrefix) && n != saltPrefix+date {
+ os.Remove(filepath.Join(dir, n))
+ }
+ }
+ return b, nil
+}
diff --git a/internal/analytics/salt_test.go b/internal/analytics/salt_test.go
new file mode 100644
index 000000000..d4f4a5ea9
--- /dev/null
+++ b/internal/analytics/salt_test.go
@@ -0,0 +1,33 @@
+package analytics
+
+import (
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+)
+
+func TestDaySaltIsStableAndRotates(t *testing.T) {
+ dir := t.TempDir()
+ a, err := DaySalt(dir, "2026-09-07")
+ if err != nil || len(a) != 32 {
+ t.Fatalf("salt: %v len=%d", err, len(a))
+ }
+ b, _ := DaySalt(dir, "2026-09-07")
+ if string(a) != string(b) {
+ t.Fatal("same day must return the same salt (restart-stable)")
+ }
+ if runtime.GOOS != "windows" {
+ st, _ := os.Stat(filepath.Join(dir, "analytics-salt-2026-09-07"))
+ if st.Mode().Perm() != 0o600 {
+ t.Fatalf("mode %v", st.Mode().Perm())
+ }
+ }
+ c, _ := DaySalt(dir, "2026-09-08")
+ if string(c) == string(a) {
+ t.Fatal("a new day must salt afresh")
+ }
+ if _, err := os.Stat(filepath.Join(dir, "analytics-salt-2026-09-07")); !os.IsNotExist(err) {
+ t.Fatal("rotation must delete the previous day's salt")
+ }
+}