| 1 | package analytics |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "math" |
| 8 | "os" |
| 9 | "sort" |
| 10 | "strconv" |
| 11 | "strings" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | "github.com/carlosframework/platform/internal/usage" |
| 16 | ) |
| 17 | |
| 18 | // DaySummary is one UTC day across every box. |
| 19 | type DaySummary struct { |
| 20 | Date string `json:"date"` |
| 21 | PV int64 `json:"pv"` |
| 22 | Bots int64 `json:"bots"` |
| 23 | Bytes int64 `json:"bytes"` |
| 24 | Visitors int64 `json:"visitors"` |
| 25 | Hours [24]int64 `json:"hours"` |
| 26 | Restarted bool `json:"restarted"` |
| 27 | } |
| 28 | |
| 29 | // Summary is what the tab and the API render: per-day rows plus totals. |
| 30 | // Visitors is the SUM of daily estimates (spec §3.3: "daily visitors, |
| 31 | // summed"), never a cross-day union. |
| 32 | type Summary struct { |
| 33 | From, To string |
| 34 | Days []DaySummary |
| 35 | PV, Bots int64 |
| 36 | Bytes int64 |
| 37 | Visitors int64 |
| 38 | Pages map[string]int64 |
| 39 | Referers map[string]int64 |
| 40 | Events map[string]int64 |
| 41 | Countries map[string]int64 |
| 42 | GeoSources map[string]int64 |
| 43 | PagesApprox bool |
| 44 | ReferersApprox bool |
| 45 | EventsApprox bool |
| 46 | Dropped Dropped |
| 47 | Boxes []string |
| 48 | } |
| 49 | |
| 50 | // Dates lists the UTC dates from from to to inclusive. |
| 51 | func Dates(from, to time.Time) []string { |
| 52 | var out []string |
| 53 | for d := from.UTC().Truncate(24 * time.Hour); !d.After(to.UTC()); d = d.Add(24 * time.Hour) { |
| 54 | out = append(out, usage.DayOf(d)) |
| 55 | } |
| 56 | return out |
| 57 | } |
| 58 | |
| 59 | // ReadRange lists the app's boxes and reads each day file in the range, |
| 60 | // merging across boxes: counters sum, visitor sketches union per day, |
| 61 | // top-key summaries merge and re-fold. A missing day file is an empty |
| 62 | // day; a store error fails the read. |
| 63 | func ReadRange(ctx context.Context, s release.Store, sqid, app string, from, to time.Time) (Summary, error) { |
| 64 | sum := Summary{From: usage.DayOf(from), To: usage.DayOf(to)} |
| 65 | prefix, err := Prefix(sqid, app) |
| 66 | if err != nil { |
| 67 | return sum, err |
| 68 | } |
| 69 | boxDirs, err := s.ListDirs(ctx, prefix) |
| 70 | if err != nil { |
| 71 | return sum, err |
| 72 | } |
| 73 | for _, dir := range boxDirs { |
| 74 | sum.Boxes = append(sum.Boxes, boxName(dir)) |
| 75 | } |
| 76 | sort.Strings(sum.Boxes) |
| 77 | pages, refs, events := NewTopK(), NewTopK(), NewTopK() |
| 78 | countries, sources := map[string]int64{}, map[string]int64{} |
| 79 | for _, date := range Dates(from, to) { |
| 80 | ds := DaySummary{Date: date} |
| 81 | vis := &HLL{} |
| 82 | for _, box := range sum.Boxes { |
| 83 | key, err := DayKey(sqid, app, box, date) |
| 84 | if err != nil { |
| 85 | return sum, err |
| 86 | } |
| 87 | raw, _, err := s.Get(ctx, key) |
| 88 | if errors.Is(err, os.ErrNotExist) { |
| 89 | continue |
| 90 | } |
| 91 | if err != nil { |
| 92 | return sum, err |
| 93 | } |
| 94 | var day Day |
| 95 | if err := json.Unmarshal(raw, &day); err != nil { |
| 96 | return sum, err |
| 97 | } |
| 98 | for h, row := range day.Hours { |
| 99 | if i, err := strconv.Atoi(h); err == nil && i >= 0 && i < 24 { |
| 100 | ds.Hours[i] += row.PV |
| 101 | } |
| 102 | ds.PV += row.PV |
| 103 | ds.Bots += row.Bots |
| 104 | ds.Bytes += row.Bytes |
| 105 | } |
| 106 | if v, err := DecodeHLL(day.Visitors); err == nil { |
| 107 | vis.Merge(v) |
| 108 | } |
| 109 | pages.Merge(TopKFrom(day.Pages)) |
| 110 | refs.Merge(TopKFrom(day.Referers)) |
| 111 | events.Merge(TopKFrom(day.Events)) |
| 112 | mergeCounts(countries, day.Countries, maxCountryKeys, nil) |
| 113 | mergeCounts(sources, day.GeoSources, maxGeoSourceKeys, attributed) |
| 114 | sum.Dropped.E = addDrops(sum.Dropped.E, day.Dropped.E) |
| 115 | sum.Dropped.A = addDrops(sum.Dropped.A, day.Dropped.A) |
| 116 | sum.Dropped.Flush += day.Dropped.Flush |
| 117 | ds.Restarted = ds.Restarted || day.Restarted |
| 118 | } |
| 119 | if !vis.Empty() { |
| 120 | ds.Visitors = int64(math.Round(vis.Count())) |
| 121 | } |
| 122 | sum.Days = append(sum.Days, ds) |
| 123 | sum.PV += ds.PV |
| 124 | sum.Bots += ds.Bots |
| 125 | sum.Bytes += ds.Bytes |
| 126 | sum.Visitors += ds.Visitors |
| 127 | } |
| 128 | sum.Pages, sum.PagesApprox = pages.Fold(), pages.Approximate() |
| 129 | sum.Referers, sum.ReferersApprox = refs.Fold(), refs.Approximate() |
| 130 | sum.Events, sum.EventsApprox = events.Fold(), events.Approximate() |
| 131 | sum.Countries, sum.GeoSources = countries, sources |
| 132 | return sum, nil |
| 133 | } |
| 134 | |
| 135 | func boxName(dir string) string { |
| 136 | t := strings.TrimSuffix(dir, "/") |
| 137 | if i := strings.LastIndexByte(t, '/'); i >= 0 { |
| 138 | return t[i+1:] |
| 139 | } |
| 140 | return t |
| 141 | } |
| 142 | |
amadan