| 1 | package analytics |
| 2 | |
| 3 | import ( |
| 4 | "path/filepath" |
| 5 | "testing" |
| 6 | "time" |
| 7 | ) |
| 8 | |
| 9 | func obs(method string, status int, ctype string) Obs { |
| 10 | return Obs{Acct: "bab", App: "web", Host: "h1", Method: method, Status: status, ContentType: ctype, |
| 11 | Path: "/pricing?x=1", RefererHost: "news.ycombinator.com", UA: "Mozilla/5.0 Safari", ClientIP: "203.0.113.5", |
| 12 | Bytes: 1200, When: time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC)} |
| 13 | } |
| 14 | |
| 15 | func TestIsPageviewRule(t *testing.T) { |
| 16 | cases := []struct { |
| 17 | name string |
| 18 | o Obs |
| 19 | pv bool |
| 20 | bot bool |
| 21 | }{ |
| 22 | {"html 200", obs("GET", 200, "text/html; charset=utf-8"), true, false}, |
| 23 | {"json", obs("GET", 200, "application/json"), false, false}, |
| 24 | {"404", obs("GET", 404, "text/html"), false, false}, |
| 25 | {"redirect", obs("GET", 302, "text/html"), false, false}, |
| 26 | {"304", obs("GET", 304, ""), false, false}, |
| 27 | {"head", obs("HEAD", 200, "text/html"), false, false}, |
| 28 | {"post", obs("POST", 200, "text/html"), false, false}, |
| 29 | {"prefetch", func() Obs { o := obs("GET", 200, "text/html"); o.Prefetch = true; return o }(), false, false}, |
| 30 | {"fetch dest empty", func() Obs { o := obs("GET", 200, "text/html"); o.FetchDest = "empty"; return o }(), false, false}, |
| 31 | {"fetch dest document", func() Obs { o := obs("GET", 200, "text/html"); o.FetchDest = "document"; return o }(), true, false}, |
| 32 | {"carlos prefix", func() Obs { o := obs("GET", 200, "text/html"); o.Path = "/__carlos/a"; return o }(), false, false}, |
| 33 | {"bot", func() Obs { o := obs("GET", 200, "text/html"); o.UA = "Googlebot/2.1"; return o }(), false, true}, |
| 34 | } |
| 35 | for _, c := range cases { |
| 36 | pv, bot := IsPageview(c.o) |
| 37 | if pv != c.pv || bot != c.bot { |
| 38 | t.Errorf("%s: pv=%v bot=%v, want %v %v", c.name, pv, bot, c.pv, c.bot) |
| 39 | } |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func TestMeterCountsAndDrains(t *testing.T) { |
| 44 | now := time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC) |
| 45 | m := NewMeter(t.TempDir(), "box1", func() time.Time { return now }) |
| 46 | m.Observe(obs("GET", 200, "text/html")) |
| 47 | m.Observe(obs("GET", 200, "text/html")) |
| 48 | o := obs("GET", 200, "text/html") |
| 49 | o.ClientIP = "198.51.100.9" |
| 50 | m.Observe(o) |
| 51 | m.Observe(obs("GET", 200, "application/json")) // bytes only |
| 52 | b := obs("GET", 200, "text/html") |
| 53 | b.UA = "curl/8" |
| 54 | m.Observe(b) |
| 55 | m.Track("bab", "web", "Signup", now) |
| 56 | m.Track("bab", "web", "bad name!", now) // rejected |
| 57 | out := m.Drain() |
| 58 | ds := out[AppKey{Acct: "bab", App: "web"}] |
| 59 | if len(ds) != 1 { |
| 60 | t.Fatalf("deltas = %d", len(ds)) |
| 61 | } |
| 62 | d := ds[0] |
| 63 | if d.Date != "2026-09-07" || d.Box != "box1" || d.Seq == 0 { |
| 64 | t.Fatalf("delta = %+v", d) |
| 65 | } |
| 66 | if h := d.Hours["09"]; h.PV != 3 || h.Bots != 1 || h.Bytes != 6000 { |
| 67 | t.Fatalf("hour = %+v", h) |
| 68 | } |
| 69 | if c := d.Visitors.Count(); c < 1.5 || c > 2.5 { |
| 70 | t.Fatalf("visitors ≈ 2, got %.2f", c) |
| 71 | } |
| 72 | if p := d.Pages.Fold(); p["/pricing"] != 3 { |
| 73 | t.Fatalf("pages = %v", p) |
| 74 | } |
| 75 | if r := d.Referers.Fold(); r["news.ycombinator.com"] != 3 { |
| 76 | t.Fatalf("referers = %v", r) |
| 77 | } |
| 78 | if e := d.Events.Fold(); e["signup"] != 1 || len(e) != 1 { |
| 79 | t.Fatalf("events = %v", e) |
| 80 | } |
| 81 | if d.Dropped.A.Malformed != 1 { |
| 82 | t.Fatalf("bad event name must count Dropped.A.Malformed, not vanish: %+v", d.Dropped) |
| 83 | } |
| 84 | if len(m.Drain()) != 0 { |
| 85 | t.Fatal("second drain must be empty") |
| 86 | } |
| 87 | // Event-time bucketing: an observation at 09:59 flushed at 10:00 lands in 09. |
| 88 | m.Observe(func() Obs { o := obs("GET", 200, "text/html"); o.When = now.Add(29 * time.Minute); return o }()) |
| 89 | now = now.Add(31 * time.Minute) |
| 90 | if d := m.Drain()[AppKey{Acct: "bab", App: "web"}][0]; d.Hours["09"].PV != 1 || d.Hours["10"].PV != 0 { |
| 91 | t.Fatalf("event-time bucket: %+v", d.Hours) |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | // TestMeterRequeueSurvivesOnce exercises the production caller's actual |
| 96 | // sequence — Drain, flush, Requeue on failure, repeat — where pending is |
| 97 | // always empty at the moment Requeue is called (the prior Drain just |
| 98 | // cleared it). The "already re-queued once" state has to live on the |
| 99 | // delta itself, not on pending's emptiness, or nothing is ever dropped |
| 100 | // and pending grows without bound across repeated failures. Each delta |
| 101 | // may survive exactly one failed flush: a second failure for the same |
| 102 | // delta drops it and counts the drop (once the delta is marked, a second |
| 103 | // Requeue drops it) on the next delta that reaches the app's bucket — |
| 104 | // bounded forever at two windows per app no matter how many flushes in a |
| 105 | // row fail. |
| 106 | func TestMeterRequeueSurvivesOnce(t *testing.T) { |
| 107 | now := time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC) |
| 108 | m := NewMeter(t.TempDir(), "box1", func() time.Time { return now }) |
| 109 | k := AppKey{Acct: "bab", App: "web"} |
| 110 | |
| 111 | m.Observe(obs("GET", 200, "text/html")) |
| 112 | d1 := m.Drain() |
| 113 | m.Requeue(d1) // flush failed: d1's delta is marked and held |
| 114 | |
| 115 | m.Observe(obs("GET", 200, "text/html")) |
| 116 | d2 := m.Drain() // = d1's delta (marked, own seq) + the new one (fresh seq) |
| 117 | if ds := d2[k]; len(ds) != 2 || ds[0].Seq >= ds[1].Seq { |
| 118 | t.Fatalf("d2 must be the requeued delta then the new one, in seq order: %+v", ds) |
| 119 | } |
| 120 | m.Requeue(d2) // failed again: d1's delta is already marked, so it drops; d2's new delta is marked and held |
| 121 | |
| 122 | m.Observe(obs("GET", 200, "text/html")) |
| 123 | d3 := m.Drain()[k] |
| 124 | if len(d3) != 2 { |
| 125 | t.Fatalf("d1's window must be gone, leaving d2's surviving delta plus the new one: %+v", d3) |
| 126 | } |
| 127 | if d3[0].Dropped.Flush != 1 { |
| 128 | t.Fatalf("the drop must be reported on the delta that reaches the bucket: %+v", d3) |
| 129 | } |
| 130 | if d3[0].Seq != d2[k][1].Seq { |
| 131 | t.Fatalf("the surviving delta must keep the seq it had in d2, not be re-stamped: got %d, want %d", d3[0].Seq, d2[k][1].Seq) |
| 132 | } |
| 133 | if d3[1].Seq <= d3[0].Seq { |
| 134 | t.Fatalf("the new delta must get a fresh, higher seq: %+v", d3) |
| 135 | } |
| 136 | |
| 137 | // A third failure in a row: bounded forever at two windows per app. |
| 138 | m.Requeue(map[AppKey][]Delta{k: d3}) |
| 139 | m.Observe(obs("GET", 200, "text/html")) |
| 140 | d4 := m.Drain()[k] |
| 141 | if len(d4) != 2 || d4[0].Dropped.Flush != 1 { |
| 142 | t.Fatalf("repeated failures must stay bounded at two windows, each drop reported once: %+v", d4) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | // TestMeterTrackMalformedCountsDropped: an invalid event name must not |
| 147 | // vanish silently — it counts against the app's current-day |
| 148 | // Dropped.A.Malformed. |
| 149 | func TestMeterTrackMalformedCountsDropped(t *testing.T) { |
| 150 | now := time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC) |
| 151 | m := NewMeter(t.TempDir(), "box1", func() time.Time { return now }) |
| 152 | m.Track("bab", "web", "not a valid name!", now) |
| 153 | m.Track("bab", "web", "also/not valid", now) |
| 154 | d := m.Drain()[AppKey{Acct: "bab", App: "web"}][0] |
| 155 | if d.Dropped.A.Malformed != 2 { |
| 156 | t.Fatalf("Dropped.A.Malformed = %d, want 2", d.Dropped.A.Malformed) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // TestMeterSaltFailureIsThrottled: a persistently failing DaySalt must not |
| 161 | // be retried on every request — pageviews still count, but visitors stay |
| 162 | // uncounted while it fails, without blocking or panicking on the |
| 163 | // serving path. |
| 164 | func TestMeterSaltFailureIsThrottled(t *testing.T) { |
| 165 | now := time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC) |
| 166 | saltDir := filepath.Join(t.TempDir(), "missing", "nested") // DaySalt can't create its file here |
| 167 | m := NewMeter(saltDir, "box1", func() time.Time { return now }) |
| 168 | m.Observe(obs("GET", 200, "text/html")) // salt lookup fails |
| 169 | m.Observe(obs("GET", 200, "text/html")) // within the backoff window: not retried |
| 170 | now = now.Add(31 * time.Second) |
| 171 | m.Observe(obs("GET", 200, "text/html")) // backoff elapsed: retried (and fails again, saltDir still missing) |
| 172 | d := m.Drain()[AppKey{Acct: "bab", App: "web"}][0] |
| 173 | if d.Hours["09"].PV != 3 { |
| 174 | t.Fatalf("pageviews must still count while the salt fails: %+v", d.Hours) |
| 175 | } |
| 176 | if c := d.Visitors.Count(); c != 0 { |
| 177 | t.Fatalf("visitors must stay uncounted while the salt fails, got %.2f", c) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // TestMeterRestartedPerAppOnStartDay: Restarted is per (app, day), not per |
| 182 | // process. Every app observed on the start day gets its own first-delta |
| 183 | // flag (the "counts since the last flush may be missing" note is true for |
| 184 | // each app independently); a later drain of the same start day must not |
| 185 | // repeat it; and it never fires once the wall clock has moved past the |
| 186 | // start day. |
| 187 | func TestMeterRestartedPerAppOnStartDay(t *testing.T) { |
| 188 | now := time.Date(2026, 9, 7, 9, 30, 0, 0, time.UTC) |
| 189 | m := NewMeter(t.TempDir(), "box1", func() time.Time { return now }) |
| 190 | |
| 191 | appA := AppKey{Acct: "bab", App: "web"} |
| 192 | appB := AppKey{Acct: "other", App: "app2"} |
| 193 | |
| 194 | m.Observe(obs("GET", 200, "text/html")) // app A's first delta of the start day |
| 195 | o := obs("GET", 200, "text/html") |
| 196 | o.Acct, o.App = appB.Acct, appB.App |
| 197 | m.Observe(o) // app B's first delta of the start day |
| 198 | |
| 199 | out := m.Drain() |
| 200 | if !out[appA][0].Restarted { |
| 201 | t.Fatal("app A's first delta of the start day must carry Restarted") |
| 202 | } |
| 203 | if !out[appB][0].Restarted { |
| 204 | t.Fatal("app B's first delta of the start day must also carry Restarted: the flag is per app-day, not per process") |
| 205 | } |
| 206 | |
| 207 | // A second drain of the same start day for app A must not repeat the note. |
| 208 | m.Observe(obs("GET", 200, "text/html")) |
| 209 | out = m.Drain() |
| 210 | if out[appA][0].Restarted { |
| 211 | t.Fatal("app A's second delta of the start day must not be flagged again") |
| 212 | } |
| 213 | |
| 214 | // The next day, nothing is flagged: the note belongs to the start day. |
| 215 | now = time.Date(2026, 9, 8, 9, 30, 0, 0, time.UTC) |
| 216 | next := obs("GET", 200, "text/html") |
| 217 | next.When = now |
| 218 | m.Observe(next) |
| 219 | out = m.Drain() |
| 220 | if out[appA][0].Restarted { |
| 221 | t.Fatal("a day after the start day must never carry Restarted") |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | func TestVisitorHashAndEventName(t *testing.T) { |
| 226 | s1, s2 := []byte("salt-a"), []byte("salt-b") |
| 227 | if VisitorHash(s1, "h", "ip", "ua") != VisitorHash(s1, "h", "ip", "ua") { |
| 228 | t.Fatal("deterministic") |
| 229 | } |
| 230 | if VisitorHash(s1, "h", "ip", "ua") == VisitorHash(s2, "h", "ip", "ua") { |
| 231 | t.Fatal("salt must change the hash") |
| 232 | } |
| 233 | for in, want := range map[string]string{"Signup": "signup", "checkout.done": "checkout.done", "a b": "", "": "", "x/y": ""} { |
| 234 | got, ok := EventName(in) |
| 235 | if (want == "") == ok || got != want { |
| 236 | t.Errorf("EventName(%q) = %q,%v", in, got, ok) |
| 237 | } |
| 238 | } |
| 239 | } |
| 240 | |
amadan