| new file mode 100644 |
| index 000000000..f614637e8 |
| --- /dev/null |
| +++ b/internal/analytics/read.go |
| @@ -0,0 +1,127 @@ |
| +package analytics |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "math" |
| + "os" |
| + "sort" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/usage" |
| +) |
| + |
| +// DaySummary is one UTC day across every box. |
| +type DaySummary struct { |
| + Date string |
| + PV int64 |
| + Bots int64 |
| + Bytes int64 |
| + Visitors int64 // HLL estimate, rounded |
| + Hours [24]int64 |
| + Restarted bool |
| +} |
| + |
| +// Summary is what the tab and the API render: per-day rows plus totals. |
| +// Visitors is the SUM of daily estimates (spec §3.3: "daily visitors, |
| +// summed"), never a cross-day union. |
| +type Summary struct { |
| + From, To string |
| + Days []DaySummary |
| + PV, Bots int64 |
| + Bytes int64 |
| + Visitors int64 |
| + Pages map[string]int64 |
| + Referers map[string]int64 |
| + Events map[string]int64 |
| + PagesApprox bool |
| + ReferersApprox bool |
| + EventsApprox bool |
| + Dropped Dropped |
| + Boxes []string |
| +} |
| + |
| +// Dates lists the UTC dates from from to to inclusive. |
| +func Dates(from, to time.Time) []string { |
| + var out []string |
| + for d := from.UTC().Truncate(24 * time.Hour); !d.After(to.UTC()); d = d.Add(24 * time.Hour) { |
| + out = append(out, usage.DayOf(d)) |
| + } |
| + return out |
| +} |
| + |
| +// ReadRange lists the app's boxes and reads each day file in the range, |
| +// merging across boxes: counters sum, visitor sketches union per day, |
| +// top-key summaries merge and re-fold. A missing day file is an empty |
| +// day; a store error fails the read. |
| +func ReadRange(ctx context.Context, s release.Store, sqid, app string, from, to time.Time) (Summary, error) { |
| + sum := Summary{From: usage.DayOf(from), To: usage.DayOf(to)} |
| + boxDirs, err := s.ListDirs(ctx, Prefix(sqid, app)) |
| + if err != nil { |
| + return sum, err |
| + } |
| + for _, dir := range boxDirs { |
| + sum.Boxes = append(sum.Boxes, boxName(dir)) |
| + } |
| + sort.Strings(sum.Boxes) |
| + pages, refs, events := NewTopK(), NewTopK(), NewTopK() |
| + for _, date := range Dates(from, to) { |
| + ds := DaySummary{Date: date} |
| + vis := &HLL{} |
| + for _, box := range sum.Boxes { |
| + raw, _, err := s.Get(ctx, DayKey(sqid, app, box, date)) |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue |
| + } |
| + if err != nil { |
| + return sum, err |
| + } |
| + var day Day |
| + if err := json.Unmarshal(raw, &day); err != nil { |
| + return sum, err |
| + } |
| + for h, row := range day.Hours { |
| + if i, err := strconv.Atoi(h); err == nil && i >= 0 && i < 24 { |
| + ds.Hours[i] += row.PV |
| + } |
| + ds.PV += row.PV |
| + ds.Bots += row.Bots |
| + ds.Bytes += row.Bytes |
| + } |
| + if v, err := DecodeHLL(day.Visitors); err == nil { |
| + vis.Merge(v) |
| + } |
| + pages.Merge(TopKFrom(day.Pages)) |
| + refs.Merge(TopKFrom(day.Referers)) |
| + events.Merge(TopKFrom(day.Events)) |
| + sum.Dropped.E = addDrops(sum.Dropped.E, day.Dropped.E) |
| + sum.Dropped.A = addDrops(sum.Dropped.A, day.Dropped.A) |
| + sum.Dropped.Flush += day.Dropped.Flush |
| + ds.Restarted = ds.Restarted || day.Restarted |
| + } |
| + if !vis.Empty() { |
| + ds.Visitors = int64(math.Round(vis.Count())) |
| + } |
| + sum.Days = append(sum.Days, ds) |
| + sum.PV += ds.PV |
| + sum.Bots += ds.Bots |
| + sum.Bytes += ds.Bytes |
| + sum.Visitors += ds.Visitors |
| + } |
| + sum.Pages, sum.PagesApprox = pages.Fold(), pages.Approximate() |
| + sum.Referers, sum.ReferersApprox = refs.Fold(), refs.Approximate() |
| + sum.Events, sum.EventsApprox = events.Fold(), events.Approximate() |
| + return sum, nil |
| +} |
| + |
| +func boxName(dir string) string { |
| + t := strings.TrimSuffix(dir, "/") |
| + if i := strings.LastIndexByte(t, '/'); i >= 0 { |
| + return t[i+1:] |
| + } |
| + return t |
| +} |
amadan