From 990794d546fcc54e75bba9e15cffb81f553453f1 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 30 Apr 2026 16:48:50 -0600 Subject: [PATCH] feat(net/geoip): MaxMind GeoLite2 reader with auto-update New package wrapping oschwald/geoip2-golang to open MaxMind .mmdb databases directly out of cached _LATEST.tar.gz files (no Transform, no intermediate extracted mmdb). ParseConf reads MaxMind's GeoIP.conf format for AccountID + LicenseKey. Includes geoip-update CLI, integration test, and a copy of the MMDB binary format spec. --- net/geoip/MMDB_FORMAT.md | 154 ++++++++++++++++++++++++++++ net/geoip/cmd/geoip-update/main.go | 114 ++++++++++++++++++++ net/geoip/conf.go | 59 +++++++++++ net/geoip/databases.go | 141 +++++++++++++++++++++++++ net/geoip/geoip.go | 46 +++++++++ net/geoip/geoip_integration_test.go | 114 ++++++++++++++++++++ net/geoip/go.mod | 8 ++ 7 files changed, 636 insertions(+) create mode 100644 net/geoip/MMDB_FORMAT.md create mode 100644 net/geoip/cmd/geoip-update/main.go create mode 100644 net/geoip/conf.go create mode 100644 net/geoip/databases.go create mode 100644 net/geoip/geoip.go create mode 100644 net/geoip/geoip_integration_test.go create mode 100644 net/geoip/go.mod diff --git a/net/geoip/MMDB_FORMAT.md b/net/geoip/MMDB_FORMAT.md new file mode 100644 index 0000000..03babba --- /dev/null +++ b/net/geoip/MMDB_FORMAT.md @@ -0,0 +1,154 @@ +# MaxMind DB File Format Specification + +Source: https://maxmind.github.io/MaxMind-DB/ + +## Description + +The MaxMind DB format is a database system that maps IPv4 and IPv6 addresses to +data records using an efficient binary search tree architecture. + +## Version + +This specification documents **version 2.0** of the MaxMind DB binary format. +Version numbers comprise separate major and minor components and should not be +interpreted as decimal values (version 2.10 follows 2.9). Code reading a major +version should remain compatible with minor version updates. + +## Overview + +The binary database consists of three primary sections: + +1. **Binary search tree** — Each tree level corresponds to one bit in the network prefix +2. **Data section** — Contains values for networks, ranging from simple strings to complex maps/arrays +3. **Database metadata** — Information describing the database itself + +## Database Metadata + +Located at the file's end, metadata begins after the sequence `\xab\xcd\xefMaxMind.com`. +The _last_ occurrence marks the data section's end. Maximum metadata size is 128KiB +including the marker. + +Metadata is stored as a map structure. Required keys: + +| Key | Type | Description | +|-----|------|-------------| +| `node_count` | uint32 | number of search tree nodes | +| `record_size` | uint16 | bits per record (multiple of 4, minimum 24) | +| `ip_version` | uint16 | 4 or 6 | +| `database_type` | string | describes data record structure | +| `binary_format_major_version` | uint16 | format major version | +| `binary_format_minor_version` | uint16 | format minor version | +| `build_epoch` | uint64 | Unix timestamp of database build | + +Optional keys: `languages` (array of locale codes), `description` (map of lang→UTF-8 string). + +### Search Tree Size + +``` +search_tree_size = ((record_size * 2) / 8) * node_count +``` + +## Binary Search Tree Section + +The file begins with the binary search tree. Node 0 is at the section's start. +Each node contains two records (left/right pointers) that reference: + +- Another tree node: `value < node_count` +- No data: `value == node_count` +- Data section address: `value > node_count` + +### Node Layouts + +**24-bit records** (6 bytes per node): +``` +| 23 .. 0 | 23 .. 0 | +``` + +**28-bit records** (7 bytes per node): +``` +| 23 .. 0 | 27..24 | 27..24 | 23 .. 0 | +``` + +**32-bit records** (8 bytes per node): +``` +| 31 .. 0 | 31 .. 0 | +``` + +### Lookup Algorithm + +1. Convert IP to big-endian binary (32 bits for IPv4, 128 for IPv6) +2. Each bit selects left (0) or right (1) record in the current node +3. `value < node_count` → traverse to that node +4. `value == node_count` → no data for this address +5. `value > node_count` → pointer into data section + +Data section offset: + +``` +data_section_offset = (record_value - node_count) - 16 +file_offset = (record_value - node_count) + search_tree_size +``` + +### IPv4 in IPv6 Trees + +IPv4 addresses occupy the lowest 32 bits of the 128-bit space. MaxMind aliases: + +- `::ffff:0:0/96` (IPv4-mapped) +- `2002::/16` (6to4) + +Teredo (`2001::/32`) requires special handling (XOR-encoded, no tree alias). + +## Data Section Separator + +16 zero bytes separate the search tree from the data section. +Data section starts at `search_tree_size + 16` bytes into the file. + +## Output Data Section + +Each field begins with a control byte encoding type and payload size. +All binary data is big-endian. + +### Data Types + +| Type ID | Name | Notes | +|---------|------|-------| +| 1 | pointer | reference to another data section location | +| 2 | UTF-8 string | variable-length | +| 3 | double | IEEE-754 binary64, 8 bytes | +| 4 | bytes | variable-length binary | +| 5 | uint16 | 0–2 bytes | +| 6 | uint32 | 0–4 bytes | +| 7 | map | key/value pairs; keys always UTF-8; size = pair count | +| 8 | int32 | 2's complement, 0–4 bytes | +| 9 | uint64 | 0–8 bytes | +| 10 | uint128 | 0–16 bytes | +| 11 | array | ordered values; size = element count | +| 14 | boolean | true/false | +| 15 | float | IEEE-754 binary32, 4 bytes | + +Type 0 in the control byte means extended type — the actual type is in the +following byte (offset by 7: stored value 1 = type 8, etc.). + +### Control Byte + +- Bits 7–5: type (0 = extended) +- Bits 4–0: payload size indicator + - 0–28: direct byte count + - 29: size = 29 + next byte (max 284) + - 30: size = 285 + next two bytes (max 65,820) + - 31: size = 65,821 + next three bytes (max 16,843,036) + +### Pointer Encoding + +Pointer size field uses `SS VVV` format in the low 5 bits: + +| SS | Value | Offset | +|----|-------|--------| +| 0 | 11-bit | 0 | +| 1 | 19-bit | +2,048 | +| 2 | 27-bit | +526,336 | +| 3 | 32-bit (4 additional bytes) | 0 | + +## License + +Creative Commons Attribution-ShareAlike 3.0 Unported License diff --git a/net/geoip/cmd/geoip-update/main.go b/net/geoip/cmd/geoip-update/main.go new file mode 100644 index 0000000..fe557b4 --- /dev/null +++ b/net/geoip/cmd/geoip-update/main.go @@ -0,0 +1,114 @@ +// geoip-update downloads GeoLite2 edition tarballs listed in GeoIP.conf +// via conditional HTTP GETs, writing them to the configured directory. +package main + +import ( + "errors" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/therootcompany/golib/net/geoip" + "github.com/therootcompany/golib/net/httpcache" +) + +const version = "dev" + +type Config struct { + ConfPath string + Dir string + FreshDays int +} + +func main() { + cfg := Config{} + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.StringVar(&cfg.ConfPath, "config", "GeoIP.conf", "path to GeoIP.conf") + fs.StringVar(&cfg.Dir, "dir", "", "directory to store .tar.gz files (overrides DatabaseDirectory in config)") + fs.IntVar(&cfg.FreshDays, "fresh-days", 3, "skip download if file is younger than N days") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", os.Args[0]) + fs.PrintDefaults() + } + + if len(os.Args) > 1 { + switch os.Args[1] { + case "-V", "-version", "--version", "version": + fmt.Fprintf(os.Stdout, "geoip-update %s\n", version) + os.Exit(0) + case "help", "-help", "--help": + fmt.Fprintf(os.Stdout, "geoip-update %s\n\n", version) + fs.SetOutput(os.Stdout) + fs.Usage() + os.Exit(0) + } + } + if err := fs.Parse(os.Args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + os.Exit(1) + } + + data, err := os.ReadFile(cfg.ConfPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + conf, err := geoip.ParseConf(string(data)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + outDir := cfg.Dir + if outDir == "" { + outDir = conf.DatabaseDirectory + } + if outDir == "" { + outDir = "." + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + fmt.Fprintf(os.Stderr, "error: mkdir %s: %v\n", outDir, err) + os.Exit(1) + } + + if len(conf.EditionIDs) == 0 { + fmt.Fprintf(os.Stderr, "error: no EditionIDs found in %s\n", cfg.ConfPath) + os.Exit(1) + } + + authHeader := http.Header{"Authorization": []string{httpcache.BasicAuth(conf.AccountID, conf.LicenseKey)}} + maxAge := time.Duration(cfg.FreshDays) * 24 * time.Hour + + exitCode := 0 + for _, edition := range conf.EditionIDs { + path := filepath.Join(outDir, geoip.TarGzName(edition)) + cacher := &httpcache.Cacher{ + URL: geoip.DownloadBase + "/" + edition + "/download?suffix=tar.gz", + Path: path, + MaxAge: maxAge, + Header: authHeader, + } + fmt.Fprintf(os.Stderr, "Fetching %s... ", edition) + t := time.Now() + updated, err := cacher.Fetch() + if err != nil { + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "error: %s: %v\n", edition, err) + exitCode = 1 + continue + } + state := "fresh" + if updated { + state = "updated" + } + fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state) + info, _ := os.Stat(path) + fmt.Printf("%-10s %s -> %s (%s)\n", state+":", edition, path, info.ModTime().Format("2006-01-02")) + } + os.Exit(exitCode) +} diff --git a/net/geoip/conf.go b/net/geoip/conf.go new file mode 100644 index 0000000..3f984db --- /dev/null +++ b/net/geoip/conf.go @@ -0,0 +1,59 @@ +package geoip + +import ( + "bufio" + "errors" + "strings" +) + +// Conf holds the fields parsed from a geoipupdate-style config file. +// +// GeoLite2 is free: sign up at https://www.maxmind.com/en/geolite2/signup +// to get an AccountID and generate a LicenseKey, then write them to +// GeoIP.conf alongside the desired EditionIDs: +// +// AccountID 123456 +// LicenseKey xxxxxxxxxxxxxxxx +// EditionIDs GeoLite2-City GeoLite2-ASN +type Conf struct { + AccountID string + LicenseKey string + EditionIDs []string + DatabaseDirectory string +} + +// ErrMissingCredentials is returned by ParseConf when AccountID or LicenseKey +// is absent from the input. +var ErrMissingCredentials = errors.New("AccountID and LicenseKey are required") + +// ParseConf parses a geoipupdate-style config (whitespace-separated key/value +// pairs, # comments). Compatible with GeoIP.conf files used by the official +// geoipupdate tool. +func ParseConf(s string) (*Conf, error) { + kv := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, _ := strings.Cut(line, " ") + kv[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + if err := scanner.Err(); err != nil { + return nil, err + } + + c := &Conf{ + AccountID: kv["AccountID"], + LicenseKey: kv["LicenseKey"], + DatabaseDirectory: kv["DatabaseDirectory"], + } + if c.AccountID == "" || c.LicenseKey == "" { + return nil, ErrMissingCredentials + } + if ids := kv["EditionIDs"]; ids != "" { + c.EditionIDs = strings.Fields(ids) + } + return c, nil +} diff --git a/net/geoip/databases.go b/net/geoip/databases.go new file mode 100644 index 0000000..caa22f0 --- /dev/null +++ b/net/geoip/databases.go @@ -0,0 +1,141 @@ +package geoip + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "net/netip" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/oschwald/geoip2-golang" +) + +// Databases holds open GeoLite2 City + ASN readers. +type Databases struct { + City *geoip2.Reader + ASN *geoip2.Reader +} + +// Open loads the City and ASN editions from dir. For each edition it +// prefers _LATEST.tar.gz and falls back to the +// lexicographically greatest _*.tar.gz match (MaxMind's +// Content-Disposition names sort chronologically by release date). +// Archives are extracted in memory — no .mmdb files are written to disk. +func Open(dir string) (*Databases, error) { + cityPath, err := FindTarGz(dir, CityEdition) + if err != nil { + return nil, fmt.Errorf("city: %w", err) + } + city, err := openMMDBTarGz(cityPath) + if err != nil { + return nil, fmt.Errorf("city: %w", err) + } + asnPath, err := FindTarGz(dir, ASNEdition) + if err != nil { + _ = city.Close() + return nil, fmt.Errorf("asn: %w", err) + } + asn, err := openMMDBTarGz(asnPath) + if err != nil { + _ = city.Close() + return nil, fmt.Errorf("asn: %w", err) + } + return &Databases{City: city, ASN: asn}, nil +} + +// FindTarGz resolves the cached tarball path for edition inside dir, +// preferring _LATEST.tar.gz and falling back to the +// lexicographically greatest _*.tar.gz match. +func FindTarGz(dir, edition string) (string, error) { + preferred := filepath.Join(dir, TarGzName(edition)) + if _, err := os.Stat(preferred); err == nil { + return preferred, nil + } + matches, err := filepath.Glob(filepath.Join(dir, edition+"_*.tar.gz")) + if err != nil { + return "", err + } + if len(matches) == 0 { + return "", fmt.Errorf("no %s_*.tar.gz in %s", edition, dir) + } + slices.Sort(matches) + return matches[len(matches)-1], nil +} + +func openMMDBTarGz(path string) (*geoip2.Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + gr, err := gzip.NewReader(f) + if err != nil { + return nil, fmt.Errorf("gzip %s: %w", path, err) + } + defer gr.Close() + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil, fmt.Errorf("no .mmdb entry in %s", path) + } + if err != nil { + return nil, err + } + if !strings.HasSuffix(hdr.Name, ".mmdb") { + continue + } + data, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + return geoip2.FromBytes(data) + } +} + +// Close closes the city and ASN readers. +func (d *Databases) Close() error { + return errors.Join(d.City.Close(), d.ASN.Close()) +} + +// Info is the structured result of a GeoIP lookup. +type Info struct { + City string `json:"city,omitempty"` + Region string `json:"region,omitempty"` + Country string `json:"country,omitempty"` + CountryISO string `json:"country_iso,omitempty"` + ASN uint `json:"asn,omitzero"` + ASNOrg string `json:"asn_org,omitempty"` +} + +// Lookup returns city + ASN info for ip. Returns a zero Info on unparseable +// IP or database miss. +func (d *Databases) Lookup(ip string) Info { + var info Info + addr, err := netip.ParseAddr(ip) + if err != nil { + return info + } + stdIP := addr.AsSlice() + + if rec, err := d.City.City(stdIP); err == nil { + info.City = rec.City.Names["en"] + info.Country = rec.Country.Names["en"] + info.CountryISO = rec.Country.IsoCode + if len(rec.Subdivisions) > 0 { + if sub := rec.Subdivisions[0].Names["en"]; sub != "" && sub != info.City { + info.Region = sub + } + } + } + if rec, err := d.ASN.ASN(stdIP); err == nil { + info.ASN = rec.AutonomousSystemNumber + info.ASNOrg = rec.AutonomousSystemOrganization + } + return info +} diff --git a/net/geoip/geoip.go b/net/geoip/geoip.go new file mode 100644 index 0000000..c4490d4 --- /dev/null +++ b/net/geoip/geoip.go @@ -0,0 +1,46 @@ +package geoip + +import ( + "os" + "path/filepath" +) + +const ( + CityEdition = "GeoLite2-City" + ASNEdition = "GeoLite2-ASN" + CountryEdition = "GeoLite2-Country" + + // DownloadBase is the MaxMind databases download endpoint. Full URL: + // //download?suffix=tar.gz + DownloadBase = "https://download.maxmind.com/geoip/databases" +) + +// TarGzName returns the cache filename for edition's tar.gz archive. +// MaxMind's Content-Disposition names include a release date +// (e.g. GeoLite2-ASN_20260101.tar.gz); we use _LATEST so httpcache's +// ETag sidecar stays tied to a stable path across releases. +func TarGzName(edition string) string { + return edition + "_LATEST.tar.gz" +} + +// DefaultConfPaths returns the standard locations where GeoIP.conf is looked +// up: ./GeoIP.conf, then ~/.config/maxmind/GeoIP.conf. +func DefaultConfPaths() []string { + paths := []string{"GeoIP.conf"} + if home, err := os.UserHomeDir(); err == nil { + paths = append(paths, filepath.Join(home, ".config", "maxmind", "GeoIP.conf")) + } + return paths +} + +// DefaultCacheDir returns ~/.cache/maxmind. CLI tools use the XDG +// convention on all platforms — os.UserCacheDir's macOS default +// (~/Library/Caches) is meant for bundled desktop apps and hides the +// files from anyone looking under ~/.cache. +func DefaultCacheDir() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".cache", "maxmind"), nil +} diff --git a/net/geoip/geoip_integration_test.go b/net/geoip/geoip_integration_test.go new file mode 100644 index 0000000..55a9893 --- /dev/null +++ b/net/geoip/geoip_integration_test.go @@ -0,0 +1,114 @@ +//go:build integration + +package geoip_test + +import ( + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/geoip" + "github.com/therootcompany/golib/net/httpcache" +) + +func testdataDir(t *testing.T) string { + t.Helper() + dir, _ := filepath.Abs(".") + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata") + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find module root (go.mod)") + } + dir = parent + } +} + +func geoipConf(t *testing.T) *geoip.Conf { + t.Helper() + dir, _ := filepath.Abs(".") + for { + p := filepath.Join(dir, "GeoIP.conf") + if data, err := os.ReadFile(p); err == nil { + cfg, err := geoip.ParseConf(string(data)) + if err != nil { + t.Fatalf("GeoIP.conf: %v", err) + } + return cfg + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + t.Skip("GeoIP.conf not found; skipping MaxMind integration test") + return nil +} + +func newCacher(cfg *geoip.Conf, edition, path string) *httpcache.Cacher { + return &httpcache.Cacher{ + URL: geoip.DownloadBase + "/" + edition + "/download?suffix=tar.gz", + Path: path, + Header: http.Header{ + "Authorization": []string{httpcache.BasicAuth(cfg.AccountID, cfg.LicenseKey)}, + }, + } +} + +func TestDownload_CityAndASN(t *testing.T) { + cfg := geoipConf(t) + td := testdataDir(t) + + for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} { + path := filepath.Join(td, geoip.TarGzName(edition)) + os.Remove(path) + os.Remove(path + ".meta") + + updated, err := newCacher(cfg, edition, path).Fetch() + if err != nil { + t.Fatalf("%s Fetch: %v", edition, err) + } + if !updated { + t.Errorf("%s: expected updated=true on first fetch", edition) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("%s: file not created: %v", edition, err) + } + if info.Size() == 0 { + t.Errorf("%s: downloaded file is empty", edition) + } + t.Logf("%s: %d bytes", edition, info.Size()) + + if _, err := os.Stat(path + ".meta"); err != nil { + t.Errorf("%s: sidecar not written: %v", edition, err) + } + } +} + +func TestDownload_ConditionalGet_FreshCacher(t *testing.T) { + cfg := geoipConf(t) + td := testdataDir(t) + + for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} { + path := filepath.Join(td, geoip.TarGzName(edition)) + + if _, err := newCacher(cfg, edition, path).Fetch(); err != nil { + t.Fatalf("%s initial Fetch: %v", edition, err) + } + + updated, err := newCacher(cfg, edition, path).Fetch() + if err != nil { + t.Fatalf("%s fresh Fetch: %v", edition, err) + } + if updated { + t.Errorf("%s: fresh cacher expected updated=false (sidecar ETag should have been used)", edition) + } + t.Logf("%s: fresh-cacher conditional GET correctly skipped re-download", edition) + } +} diff --git a/net/geoip/go.mod b/net/geoip/go.mod new file mode 100644 index 0000000..3a91846 --- /dev/null +++ b/net/geoip/go.mod @@ -0,0 +1,8 @@ +module github.com/therootcompany/golib/net/geoip + +go 1.26.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 + github.com/therootcompany/golib/net/httpcache v0.0.0 +)