amadan

carlos / platform Public

geo: Lookup with country, subdivision and point; metadata accessors; OpenMMDBTrusted (geoip spec §6)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude ecbbf2e1eb74aa6159555d90259878683dc73c69 parent 9413294
2 files changed, +301 −26
  • internal/geo/mmdb.go +155 −26
  • internal/geo/mmdb_test.go +146 −0
diff --git a/internal/geo/mmdb.go b/internal/geo/mmdb.go
index c9cc6250b..e085d1eda 100644
--- a/internal/geo/mmdb.go
+++ b/internal/geo/mmdb.go
@@ -4,6 +4,7 @@ import (
"log"
"math"
"net"
+ "strings"
"sync"
"time"
@@ -16,12 +17,16 @@ type Locator interface {
Locate(ip net.IP) (Point, bool)
}
-// MMDB is an operator-supplied MaxMind-format database
-// (CONSOLE_GEOIP_DB). lookup is the reader's Lookup, a field so a test can
-// prove which addresses ever reach it.
+// MMDB is an operator-supplied MaxMind-format database. lookup is the
+// reader's Lookup, a field so a test can prove which addresses ever reach
+// it; source, build and nodes are copied from the metadata at open so a
+// test fake can set them too.
type MMDB struct {
r *maxminddb.Reader
lookup func(ip net.IP, result any) error
+ source string
+ build time.Time
+ nodes uint64
}
// OpenMMDB opens and verifies the file. An error means "IP step off"; the
@@ -35,15 +40,68 @@ func OpenMMDB(path string) (*MMDB, error) {
r.Close()
return nil, err
}
- return &MMDB{r: r, lookup: r.Lookup}, nil
+ return newMMDB(r), nil
}
-// Close releases the mapping.
-func (m *MMDB) Close() error {
- if m == nil || m.r == nil {
- return nil
+// OpenMMDBTrusted opens without Verify. Only for a file whose bytes were
+// verified elsewhere and proven identical by digest (geoip spec §5.3):
+// Verify decodes the whole data section, which costs seconds and tens of
+// MiB on every edge for no new information.
+func OpenMMDBTrusted(path string) (*MMDB, error) {
+ r, err := maxminddb.Open(path)
+ if err != nil {
+ return nil, err
}
- return m.r.Close()
+ return newMMDB(r), nil
+}
+
+func newMMDB(r *maxminddb.Reader) *MMDB {
+ md := r.Metadata
+ return &MMDB{
+ r: r, lookup: r.Lookup,
+ source: sanitizeSource(md.DatabaseType),
+ build: time.Unix(int64(md.BuildEpoch), 0).UTC(),
+ nodes: uint64(md.NodeCount),
+ }
+}
+
+// sanitizeSource reduces a metadata database_type to [A-Za-z0-9._-],
+// at most 64 bytes: it is echoed into a request header and a log line.
+func sanitizeSource(s string) string {
+ var b strings.Builder
+ for _, c := range s {
+ if b.Len() >= 64 {
+ break
+ }
+ if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '.' || c == '_' || c == '-' {
+ b.WriteRune(c)
+ }
+ }
+ return b.String()
+}
+
+// Source is the database's type ("DBIP-City-Lite", "GeoLite2-City"), sanitised.
+func (m *MMDB) Source() string {
+ if m == nil {
+ return ""
+ }
+ return m.source
+}
+
+// BuildEpoch is when the database was built (UTC); zero for a nil MMDB.
+func (m *MMDB) BuildEpoch() time.Time {
+ if m == nil {
+ return time.Time{}
+ }
+ return m.build
+}
+
+// NodeCount is the search tree's size, the shrink check's yardstick.
+func (m *MMDB) NodeCount() uint64 {
+ if m == nil {
+ return 0
+ }
+ return m.nodes
}
type mmdbRecord struct {
@@ -57,6 +115,9 @@ type mmdbRecord struct {
RegisteredCountry struct {
ISOCode string `maxminddb:"iso_code"`
} `maxminddb:"registered_country"`
+ Subdivisions []struct {
+ ISOCode string `maxminddb:"iso_code"`
+ } `maxminddb:"subdivisions"`
}
var (
@@ -64,18 +125,17 @@ var (
lookupErrLast time.Time
)
-// Locate returns the record's location (City databases), else its
-// country's point, else its registered country's. A lookup error is a
-// miss, logged as a FIXED category at most once a minute: the reader's
-// own error text can contain the address (spec §6.5).
-func (m *MMDB) Locate(ip net.IP) (Point, bool) {
+// decode is the one place a record is read and judged (codex R1-9): the
+// Locatable filter, IPv4 normalisation, the fixed-category error log, and
+// the rule that a malformed location makes the WHOLE record a miss.
+func (m *MMDB) decode(ip net.IP) (mmdbRecord, bool) {
+ var rec mmdbRecord
if m == nil || m.lookup == nil || !Locatable(ip) {
- return Point{}, false
+ return rec, false
}
if v4 := ip.To4(); v4 != nil {
ip = v4
}
- var rec mmdbRecord
if err := m.lookup(ip, &rec); err != nil {
lookupErrMu.Lock()
if time.Since(lookupErrLast) > time.Minute {
@@ -83,19 +143,28 @@ func (m *MMDB) Locate(ip net.IP) (Point, bool) {
log.Print("geo: a GeoIP lookup failed and was treated as a miss")
}
lookupErrMu.Unlock()
+ return mmdbRecord{}, false
+ }
+ if rec.Location.Latitude != nil && rec.Location.Longitude != nil &&
+ !onTheGlobe(Point{*rec.Location.Latitude, *rec.Location.Longitude}) {
+ // A structurally valid database can still carry a malformed
+ // location (final review). A record wrong in one field is not
+ // trusted on the next.
+ return mmdbRecord{}, false
+ }
+ return rec, true
+}
+
+// Locate returns the record's location (City databases), else its
+// country's point, else its registered country's. Behaviour is exactly
+// what it was before Lookup existed (codex R1-9).
+func (m *MMDB) Locate(ip net.IP) (Point, bool) {
+ rec, ok := m.decode(ip)
+ if !ok {
return Point{}, false
}
if rec.Location.Latitude != nil && rec.Location.Longitude != nil {
- p := Point{*rec.Location.Latitude, *rec.Location.Longitude}
- if !onTheGlobe(p) {
- // A structurally valid database can still carry a malformed
- // location (final review). It is a miss, not a point that
- // ranks regions from nowhere, and not a fall-through to the
- // record's country: a record wrong in one field is not
- // trusted on the next.
- return Point{}, false
- }
- return p, true
+ return Point{*rec.Location.Latitude, *rec.Location.Longitude}, true
}
if p, ok := CountryPoint(rec.Country.ISOCode); ok {
return p, true
@@ -103,6 +172,66 @@ func (m *MMDB) Locate(ip net.IP) (Point, bool) {
return CountryPoint(rec.RegisteredCountry.ISOCode)
}
+// Result is what Lookup found. Country is ISO 3166-1 alpha-2 (the
+// record's country, else its registered country); Subdivision is
+// "CC-XXX" only when the database carries an ISO code for it (GeoLite2
+// does, DB-IP Lite does not, FACTS.md); HasPoint is true only for a real
+// location record, never a country centroid.
+type Result struct {
+ Country string
+ Subdivision string
+ Point Point
+ HasPoint bool
+}
+
+// Lookup answers with whatever the record holds. ok is true when it found
+// a country, a point, or both (geoip spec §6).
+func (m *MMDB) Lookup(ip net.IP) (Result, bool) {
+ rec, ok := m.decode(ip)
+ if !ok {
+ return Result{}, false
+ }
+ var r Result
+ if isISO2(rec.Country.ISOCode) {
+ r.Country = rec.Country.ISOCode
+ if len(rec.Subdivisions) > 0 && isSubdivisionCode(rec.Subdivisions[0].ISOCode) {
+ r.Subdivision = r.Country + "-" + rec.Subdivisions[0].ISOCode
+ }
+ } else if isISO2(rec.RegisteredCountry.ISOCode) {
+ // The registered country says where the block is registered, not
+ // which subdivision of it the visitor is in: no subdivision.
+ r.Country = rec.RegisteredCountry.ISOCode
+ }
+ if rec.Location.Latitude != nil && rec.Location.Longitude != nil {
+ r.Point, r.HasPoint = Point{*rec.Location.Latitude, *rec.Location.Longitude}, true
+ }
+ return r, r.Country != "" || r.HasPoint
+}
+
+func isISO2(s string) bool {
+ return len(s) == 2 && s[0] >= 'A' && s[0] <= 'Z' && s[1] >= 'A' && s[1] <= 'Z'
+}
+
+func isSubdivisionCode(s string) bool {
+ if len(s) < 1 || len(s) > 3 {
+ return false
+ }
+ for i := 0; i < len(s); i++ {
+ if !(s[i] >= 'A' && s[i] <= 'Z' || s[i] >= '0' && s[i] <= '9') {
+ return false
+ }
+ }
+ return true
+}
+
+// Close releases the mapping.
+func (m *MMDB) Close() error {
+ if m == nil || m.r == nil {
+ return nil
+ }
+ return m.r.Close()
+}
+
// onTheGlobe is false for nonfinite coordinates, a latitude outside
// [-90, 90] or a longitude outside [-180, 180]. Both bounds are inclusive:
// the poles and the antimeridian are places.
diff --git a/internal/geo/mmdb_test.go b/internal/geo/mmdb_test.go
index a6e8dcf15..2a95670d5 100644
--- a/internal/geo/mmdb_test.go
+++ b/internal/geo/mmdb_test.go
@@ -8,7 +8,9 @@ import (
"net"
"os"
"path/filepath"
+ "strings"
"testing"
+ "time"
)
func TestOpenMMDBRefusesNonDatabases(t *testing.T) {
@@ -152,3 +154,147 @@ func TestLocateNeverLooksUpNonPublicAddresses(t *testing.T) {
t.Error("a nil *MMDB located an address")
}
}
+
+func TestLookupCityGivesCountrySubdivisionAndPoint(t *testing.T) {
+ m, err := OpenMMDB("testdata/GeoIP2-City-Test.mmdb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ cases := map[string]Result{
+ "81.2.69.142": {Country: "GB", Subdivision: "GB-ENG", HasPoint: true},
+ "216.160.83.56": {Country: "US", Subdivision: "US-WA", HasPoint: true},
+ "89.160.20.112": {Country: "SE", Subdivision: "SE-E", HasPoint: true},
+ "2001:218::1": {Country: "JP", HasPoint: true}, // no subdivision in the fixture
+ }
+ for ip, want := range cases {
+ got, ok := m.Lookup(net.ParseIP(ip))
+ if !ok || got.Country != want.Country || got.Subdivision != want.Subdivision || got.HasPoint != want.HasPoint {
+ t.Errorf("Lookup(%s) = (%+v, %v), want %+v", ip, got, ok, want)
+ }
+ }
+ if got, _ := m.Lookup(net.ParseIP("81.2.69.142")); got.Point.Lat < 51 || got.Point.Lat > 52 {
+ t.Errorf("London point = %v", got.Point)
+ }
+ if m.Source() != "GeoIP2-City" {
+ t.Errorf("Source = %q", m.Source())
+ }
+ if m.NodeCount() != 1547 {
+ t.Errorf("NodeCount = %d, want 1547 (FACTS.md)", m.NodeCount())
+ }
+ if m.BuildEpoch().Year() < 2020 {
+ t.Errorf("BuildEpoch = %v", m.BuildEpoch())
+ }
+}
+
+func TestLookupCountryDatabaseHasNoPoint(t *testing.T) {
+ m, err := OpenMMDB("testdata/GeoIP2-Country-Test.mmdb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ got, ok := m.Lookup(net.ParseIP("81.2.69.142"))
+ if !ok || got.Country != "GB" || got.HasPoint || got.Subdivision != "" {
+ t.Fatalf("country-only = (%+v, %v), want GB with no point (never a centroid)", got, ok)
+ }
+}
+
+func TestLookupNoCountryDatabaseMisses(t *testing.T) {
+ m, err := OpenMMDB("testdata/MaxMind-DB-test-ipv4-24.mmdb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ if got, ok := m.Lookup(net.ParseIP("81.2.69.142")); ok {
+ t.Fatalf("a database with no country or location answered %+v", got)
+ }
+}
+
+func TestLookupFallsBackToRegisteredCountryButNotItsSubdivision(t *testing.T) {
+ m := &MMDB{lookup: func(_ net.IP, result any) error {
+ rec := result.(*mmdbRecord)
+ rec.RegisteredCountry.ISOCode = "IE"
+ rec.Subdivisions = append(rec.Subdivisions, struct {
+ ISOCode string `maxminddb:"iso_code"`
+ }{"L"})
+ return nil
+ }}
+ got, ok := m.Lookup(net.ParseIP("81.2.69.142"))
+ if !ok || got.Country != "IE" || got.Subdivision != "" {
+ t.Fatalf("registered fallback = (%+v, %v), want IE with no subdivision", got, ok)
+ }
+}
+
+func TestLookupPointWithoutCountrySucceeds(t *testing.T) {
+ lat, lon := 10.0, 20.0
+ m := &MMDB{lookup: func(_ net.IP, result any) error {
+ rec := result.(*mmdbRecord)
+ rec.Location.Latitude, rec.Location.Longitude = &lat, &lon
+ return nil
+ }}
+ got, ok := m.Lookup(net.ParseIP("81.2.69.142"))
+ if !ok || got.Country != "" || !got.HasPoint || got.Point != (Point{10, 20}) {
+ t.Fatalf("point-only = (%+v, %v)", got, ok)
+ }
+ // Locate's existing rule: a valid point succeeds without a country.
+ if p, ok := m.Locate(net.ParseIP("81.2.69.142")); !ok || p != (Point{10, 20}) {
+ t.Fatalf("Locate point-only = (%v, %v)", p, ok)
+ }
+}
+
+func TestLocateRegisteredCountryFallbackUnchanged(t *testing.T) {
+ // Codex R1-9: a country with no representative point falls back to the
+ // registered country's point. "XK" has none in the table; "GB" does.
+ if _, ok := CountryPoint("XK"); ok {
+ t.Skip("XK gained a representative point; pick another code absent from the table")
+ }
+ m := &MMDB{lookup: func(_ net.IP, result any) error {
+ rec := result.(*mmdbRecord)
+ rec.Country.ISOCode, rec.RegisteredCountry.ISOCode = "XK", "GB"
+ return nil
+ }}
+ gb, _ := CountryPoint("GB")
+ if p, ok := m.Locate(net.ParseIP("81.2.69.142")); !ok || p != gb {
+ t.Fatalf("Locate = (%v, %v), want GB's point", p, ok)
+ }
+}
+
+func TestLookupRejectsMalformedCodes(t *testing.T) {
+ m := &MMDB{lookup: func(_ net.IP, result any) error {
+ rec := result.(*mmdbRecord)
+ rec.Country.ISOCode = "gb" // lower case is not ISO
+ rec.Subdivisions = append(rec.Subdivisions, struct {
+ ISOCode string `maxminddb:"iso_code"`
+ }{"TOOLONG"})
+ return nil
+ }}
+ if got, ok := m.Lookup(net.ParseIP("81.2.69.142")); ok {
+ t.Fatalf("malformed codes answered %+v", got)
+ }
+}
+
+func TestOpenMMDBTrustedOpensWithoutVerify(t *testing.T) {
+ m, err := OpenMMDBTrusted("testdata/GeoIP2-City-Test.mmdb")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ if r, ok := m.Lookup(net.ParseIP("81.2.69.142")); !ok || r.Country != "GB" {
+ t.Fatalf("trusted open lookup = (%+v, %v)", r, ok)
+ }
+ if _, err := OpenMMDBTrusted(filepath.Join(t.TempDir(), "missing.mmdb")); err == nil {
+ t.Fatal("a missing file opened")
+ }
+}
+
+func TestSourceIsSanitised(t *testing.T) {
+ m := &MMDB{source: sanitizeSource("GeoLite2-City\n<script>" + strings.Repeat("x", 100))}
+ if s := m.Source(); strings.ContainsAny(s, "\n<>") || len(s) > 64 {
+ t.Fatalf("Source = %q", s)
+ }
+ var nilM *MMDB
+ if nilM.Source() != "" || nilM.NodeCount() != 0 || !nilM.BuildEpoch().IsZero() {
+ t.Fatal("a nil MMDB must answer zero values")
+ }
+ _ = time.Now
+}