amadan

carlos / platform Public

geoip: capped archive readers and the feed vocabulary (geoip spec §10)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 62b80bb0b08c17fd7e3adc743e487b3d56b9b187 parent 419d897
3 files changed, +259 −0
  • internal/geoip/archive.go +71 −0
  • internal/geoip/archive_test.go +111 −0
  • internal/geoip/feed.go +77 −0
diff --git a/internal/geoip/archive.go b/internal/geoip/archive.go
new file mode 100644
index 000000000..316355696
--- /dev/null
+++ b/internal/geoip/archive.go
@@ -0,0 +1,71 @@
+package geoip
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "strings"
+)
+
+// ErrTooLarge is a capped reader refusing to read past its cap (spec §10.3).
+var ErrTooLarge = errors.New("geoip archive: larger than its cap")
+
+type cappedReader struct {
+ r io.Reader
+ left int64
+}
+
+func (c *cappedReader) Read(p []byte) (int, error) {
+ if c.left < 0 {
+ return 0, ErrTooLarge
+ }
+ if int64(len(p)) > c.left+1 {
+ p = p[:c.left+1]
+ }
+ n, err := c.r.Read(p)
+ c.left -= int64(n)
+ if c.left < 0 {
+ return n, ErrTooLarge
+ }
+ return n, err
+}
+
+// CappedReader reads r until more than max bytes have come through, then
+// fails with ErrTooLarge. Exactly max bytes read cleanly.
+func CappedReader(r io.Reader, max int64) io.Reader { return &cappedReader{r: r, left: max} }
+
+// GunzipReader expands a gzip stream, refusing to expand past maxExpanded:
+// a decompression bomb stops at the cap, never at the end of its expansion.
+func GunzipReader(r io.Reader, maxExpanded int64) (io.Reader, error) {
+ zr, err := gzip.NewReader(r)
+ if err != nil {
+ return nil, fmt.Errorf("geoip archive: not gzip: %w", err)
+ }
+ return CappedReader(zr, maxExpanded), nil
+}
+
+// TarGzMMDB returns the first regular *.mmdb member of a .tar.gz (MaxMind's
+// download format). Every expanded byte counts toward maxExpanded,
+// including the members walked past, so junk padding cannot fill a disk.
+func TarGzMMDB(r io.Reader, maxExpanded int64) (io.Reader, error) {
+ expanded, err := GunzipReader(r, maxExpanded)
+ if err != nil {
+ return nil, err
+ }
+ tr := tar.NewReader(expanded)
+ for {
+ h, err := tr.Next()
+ if err == io.EOF {
+ return nil, errors.New("geoip archive: no .mmdb member")
+ }
+ if err != nil {
+ return nil, err
+ }
+ if h.Typeflag == tar.TypeReg && strings.HasSuffix(path.Base(h.Name), ".mmdb") {
+ return tr, nil
+ }
+ }
+}
diff --git a/internal/geoip/archive_test.go b/internal/geoip/archive_test.go
new file mode 100644
index 000000000..030a3903d
--- /dev/null
+++ b/internal/geoip/archive_test.go
@@ -0,0 +1,111 @@
+package geoip
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "errors"
+ "io"
+ "testing"
+ "time"
+)
+
+func gz(t *testing.T, b []byte) []byte {
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ w.Write(b)
+ w.Close()
+ return buf.Bytes()
+}
+
+func targz(t *testing.T, members map[string][]byte, order []string) []byte {
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(zw)
+ for _, name := range order {
+ body := members[name]
+ tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
+ tw.Write(body)
+ }
+ tw.Close()
+ zw.Close()
+ return buf.Bytes()
+}
+
+func TestCappedReader(t *testing.T) {
+ if b, err := io.ReadAll(CappedReader(bytes.NewReader(make([]byte, 100)), 100)); err != nil || len(b) != 100 {
+ t.Fatalf("exactly max = %d %v", len(b), err)
+ }
+ if _, err := io.ReadAll(CappedReader(bytes.NewReader(make([]byte, 101)), 100)); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("max+1 = %v", err)
+ }
+}
+
+func TestGunzipStopsAtTheCap(t *testing.T) {
+ r, err := GunzipReader(bytes.NewReader(gz(t, make([]byte, 10<<20))), 1<<20)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n, err := io.Copy(io.Discard, r)
+ if !errors.Is(err, ErrTooLarge) || n > 1<<20+1 {
+ t.Fatalf("bomb read %d bytes, err %v", n, err)
+ }
+}
+
+func TestTarGzMMDBCountsSkippedMembers(t *testing.T) {
+ arc := targz(t, map[string][]byte{"GeoLite2-City_20261006/README.txt": make([]byte, 2<<20), "GeoLite2-City_20261006/GeoLite2-City.mmdb": []byte("the database")},
+ []string{"GeoLite2-City_20261006/README.txt", "GeoLite2-City_20261006/GeoLite2-City.mmdb"})
+ r, err := TarGzMMDB(bytes.NewReader(arc), 4<<20)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if b, _ := io.ReadAll(r); string(b) != "the database" {
+ t.Fatalf("member = %q", b)
+ }
+ if _, err := TarGzMMDB(bytes.NewReader(arc), 1<<20); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("padding past the cap = %v, want ErrTooLarge", err)
+ }
+ none := targz(t, map[string][]byte{"x.txt": []byte("x")}, []string{"x.txt"})
+ if _, err := TarGzMMDB(bytes.NewReader(none), 1<<20); err == nil {
+ t.Fatal("an archive with no .mmdb was accepted")
+ }
+}
+
+func TestFeedSpecsAndURLs(t *testing.T) {
+ s, ok := ParseFeed("dbip-city-lite")
+ if !ok || s.Type != "DBIP-City-Lite" || s.Provider != "dbip" {
+ t.Fatalf("spec = %+v %v", s, ok)
+ }
+ if got := DBIPURL("", s, time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)); got != "https://download.db-ip.com/free/dbip-city-lite-2026-10.mmdb.gz" {
+ t.Fatalf("url = %s", got)
+ }
+ c, _ := ParseFeed("dbip-country-lite")
+ if got := DBIPURL("", c, time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)); got != "https://download.db-ip.com/free/dbip-country-lite-2026-01.mmdb.gz" {
+ t.Fatalf("country url = %s", got)
+ }
+ g, _ := ParseFeed("geolite2-city")
+ if got := MaxMindURL("", g, "tar.gz.sha256"); got != "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz.sha256" {
+ t.Fatalf("maxmind url = %s", got)
+ }
+ if _, ok := ParseFeed("geoip2-city"); ok {
+ t.Fatal("unknown feed accepted")
+ }
+}
+
+func TestFeedStatusFailing(t *testing.T) {
+ s := FeedStatus{V: 1, ConsecutiveFailures: 6}
+ if s.Failing() {
+ t.Fatal("6 is not failing")
+ }
+ s.ConsecutiveFailures = 7
+ if !s.Failing() {
+ t.Fatal("7 is failing")
+ }
+ if !(FeedStatus{ConsecutiveNotFound: 7}).Failing() {
+ t.Fatal("7 not-found is failing")
+ }
+ raw := []byte(`{"v":1,"feed":"dbip-city-lite","box":"b","consecutive_failures":2}`)
+ if got, err := ParseFeedStatus(raw); err != nil || got.Box != "b" {
+ t.Fatalf("parse = %+v %v", got, err)
+ }
+}
diff --git a/internal/geoip/feed.go b/internal/geoip/feed.go
new file mode 100644
index 000000000..9c176a006
--- /dev/null
+++ b/internal/geoip/feed.go
@@ -0,0 +1,77 @@
+package geoip
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+)
+
+// FeedSpec is one supported feed (spec §10.1): the env name, the database
+// type it must deliver, and its provider.
+type FeedSpec struct{ Name, Type, Provider string }
+
+var feeds = map[string]FeedSpec{
+ "dbip-city-lite": {"dbip-city-lite", "DBIP-City-Lite", "dbip"},
+ "dbip-country-lite": {"dbip-country-lite", "DBIP-Country-Lite", "dbip"},
+ "geolite2-city": {"geolite2-city", "GeoLite2-City", "maxmind"},
+ "geolite2-country": {"geolite2-country", "GeoLite2-Country", "maxmind"},
+}
+
+// ParseFeed resolves CONSOLE_GEOIP_FEED.
+func ParseFeed(name string) (FeedSpec, bool) {
+ s, ok := feeds[strings.TrimSpace(name)]
+ return s, ok
+}
+
+// DBIPURL is the month's DB-IP Lite download (spec §10.2).
+func DBIPURL(base string, spec FeedSpec, month time.Time) string {
+ if base == "" {
+ base = "https://download.db-ip.com"
+ }
+ kind := strings.TrimSuffix(strings.TrimPrefix(spec.Name, "dbip-"), "-lite") // city | country
+ return fmt.Sprintf("%s/free/dbip-%s-lite-%s.mmdb.gz", base, kind, month.UTC().Format("2006-01"))
+}
+
+// MaxMindURL is the GeoLite2 permalink; suffix is "tar.gz" or "tar.gz.sha256".
+func MaxMindURL(base string, spec FeedSpec, suffix string) string {
+ if base == "" {
+ base = "https://download.maxmind.com"
+ }
+ return fmt.Sprintf("%s/geoip/databases/%s/download?suffix=%s", base, spec.Type, suffix)
+}
+
+// FeedStatus is control/geoip/feed.json, written by the console after every
+// check (spec §10.4). Box names the console's box: the one box whose mirror
+// raises the feed reason, so it fires once.
+type FeedStatus struct {
+ V int `json:"v"`
+ Feed string `json:"feed"`
+ Box string `json:"box"`
+ LastCheck time.Time `json:"last_check"`
+ LastSuccess time.Time `json:"last_success,omitempty"`
+ LastEdition string `json:"last_edition,omitempty"`
+ ConsecutiveFailures int `json:"consecutive_failures"`
+ LastFailure string `json:"last_failure,omitempty"` // a fixed category, never an error text
+ ConsecutiveNotFound int `json:"consecutive_not_found"`
+}
+
+// FeedNotFoundAlert is how many failing checks in a row raise the alert.
+const FeedNotFoundAlert = 7
+
+// ParseFeedStatus decodes feed.json.
+func ParseFeedStatus(raw []byte) (FeedStatus, error) {
+ var s FeedStatus
+ if err := json.Unmarshal(raw, &s); err != nil {
+ return FeedStatus{}, fmt.Errorf("geoip feed status: %w", err)
+ }
+ if s.V != 1 {
+ return FeedStatus{}, fmt.Errorf("geoip feed status: version %d: %w", s.V, ErrUnknownVersion)
+ }
+ return s, nil
+}
+
+// Failing is the alert's feed reason: 7 failed or not-found checks in a row.
+func (s FeedStatus) Failing() bool {
+ return s.ConsecutiveFailures >= FeedNotFoundAlert || s.ConsecutiveNotFound >= FeedNotFoundAlert
+}