feat(net/geoip): MaxMind GeoLite2 reader with auto-update

New package wrapping oschwald/geoip2-golang to open MaxMind .mmdb databases
directly out of cached <edition>_LATEST.tar.gz files (no Transform, no
intermediate extracted mmdb). ParseConf reads MaxMind's GeoIP.conf format
for AccountID + LicenseKey. Includes geoip-update CLI, integration test,
and a copy of the MMDB binary format spec.

net/geoip/MMDB_FORMAT.md

@@ -0,0 +1,154 @@
+# MaxMind DB File Format Specification
+
+Source: https://maxmind.github.io/MaxMind-DB/
+
+## Description
+
+The MaxMind DB format is a database system that maps IPv4 and IPv6 addresses to
+data records using an efficient binary search tree architecture.
+
+## Version
+
+This specification documents **version 2.0** of the MaxMind DB binary format.
+Version numbers comprise separate major and minor components and should not be
+interpreted as decimal values (version 2.10 follows 2.9). Code reading a major
+version should remain compatible with minor version updates.
+
+## Overview
+
+The binary database consists of three primary sections:
+
+1. **Binary search tree** — Each tree level corresponds to one bit in the network prefix
+2. **Data section** — Contains values for networks, ranging from simple strings to complex maps/arrays
+3. **Database metadata** — Information describing the database itself
+
+## Database Metadata
+
+Located at the file's end, metadata begins after the sequence `\xab\xcd\xefMaxMind.com`.
+The _last_ occurrence marks the data section's end. Maximum metadata size is 128KiB
+including the marker.
+
+Metadata is stored as a map structure. Required keys:
+
+| Key | Type | Description |
+|-----|------|-------------|
+| `node_count` | uint32 | number of search tree nodes |
+| `record_size` | uint16 | bits per record (multiple of 4, minimum 24) |
+| `ip_version` | uint16 | 4 or 6 |
+| `database_type` | string | describes data record structure |
+| `binary_format_major_version` | uint16 | format major version |
+| `binary_format_minor_version` | uint16 | format minor version |
+| `build_epoch` | uint64 | Unix timestamp of database build |
+
+Optional keys: `languages` (array of locale codes), `description` (map of lang→UTF-8 string).
+
+### Search Tree Size
+
+```
+search_tree_size = ((record_size * 2) / 8) * node_count
+```
+
+## Binary Search Tree Section
+
+The file begins with the binary search tree. Node 0 is at the section's start.
+Each node contains two records (left/right pointers) that reference:
+
+- Another tree node: `value < node_count`
+- No data: `value == node_count`
+- Data section address: `value > node_count`
+
+### Node Layouts
+
+**24-bit records** (6 bytes per node):
+```
+| 23 .. 0          |          23 .. 0 |
+```
+
+**28-bit records** (7 bytes per node):
+```
+| 23 .. 0 | 27..24 | 27..24 | 23 .. 0 |
+```
+
+**32-bit records** (8 bytes per node):
+```
+| 31 .. 0          |          31 .. 0 |
+```
+
+### Lookup Algorithm
+
+1. Convert IP to big-endian binary (32 bits for IPv4, 128 for IPv6)
+2. Each bit selects left (0) or right (1) record in the current node
+3. `value < node_count` → traverse to that node
+4. `value == node_count` → no data for this address
+5. `value > node_count` → pointer into data section
+
+Data section offset:
+
+```
+data_section_offset = (record_value - node_count) - 16
+file_offset         = (record_value - node_count) + search_tree_size
+```
+
+### IPv4 in IPv6 Trees
+
+IPv4 addresses occupy the lowest 32 bits of the 128-bit space. MaxMind aliases:
+
+- `::ffff:0:0/96` (IPv4-mapped)
+- `2002::/16` (6to4)
+
+Teredo (`2001::/32`) requires special handling (XOR-encoded, no tree alias).
+
+## Data Section Separator
+
+16 zero bytes separate the search tree from the data section.
+Data section starts at `search_tree_size + 16` bytes into the file.
+
+## Output Data Section
+
+Each field begins with a control byte encoding type and payload size.
+All binary data is big-endian.
+
+### Data Types
+
+| Type ID | Name | Notes |
+|---------|------|-------|
+| 1 | pointer | reference to another data section location |
+| 2 | UTF-8 string | variable-length |
+| 3 | double | IEEE-754 binary64, 8 bytes |
+| 4 | bytes | variable-length binary |
+| 5 | uint16 | 0–2 bytes |
+| 6 | uint32 | 0–4 bytes |
+| 7 | map | key/value pairs; keys always UTF-8; size = pair count |
+| 8 | int32 | 2's complement, 0–4 bytes |
+| 9 | uint64 | 0–8 bytes |
+| 10 | uint128 | 0–16 bytes |
+| 11 | array | ordered values; size = element count |
+| 14 | boolean | true/false |
+| 15 | float | IEEE-754 binary32, 4 bytes |
+
+Type 0 in the control byte means extended type — the actual type is in the
+following byte (offset by 7: stored value 1 = type 8, etc.).
+
+### Control Byte
+
+- Bits 7–5: type (0 = extended)
+- Bits 4–0: payload size indicator
+  - 0–28: direct byte count
+  - 29: size = 29 + next byte (max 284)
+  - 30: size = 285 + next two bytes (max 65,820)
+  - 31: size = 65,821 + next three bytes (max 16,843,036)
+
+### Pointer Encoding
+
+Pointer size field uses `SS VVV` format in the low 5 bits:
+
+| SS | Value | Offset |
+|----|-------|--------|
+| 0 | 11-bit | 0 |
+| 1 | 19-bit | +2,048 |
+| 2 | 27-bit | +526,336 |
+| 3 | 32-bit (4 additional bytes) | 0 |
+
+## License
+
+Creative Commons Attribution-ShareAlike 3.0 Unported License

net/geoip/cmd/geoip-update/main.go

@@ -0,0 +1,114 @@
+// geoip-update downloads GeoLite2 edition tarballs listed in GeoIP.conf
+// via conditional HTTP GETs, writing them to the configured directory.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/therootcompany/golib/net/geoip"
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+const version = "dev"
+
+type Config struct {
+	ConfPath  string
+	Dir       string
+	FreshDays int
+}
+
+func main() {
+	cfg := Config{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	fs.StringVar(&cfg.ConfPath, "config", "GeoIP.conf", "path to GeoIP.conf")
+	fs.StringVar(&cfg.Dir, "dir", "", "directory to store .tar.gz files (overrides DatabaseDirectory in config)")
+	fs.IntVar(&cfg.FreshDays, "fresh-days", 3, "skip download if file is younger than N days")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", os.Args[0])
+		fs.PrintDefaults()
+	}
+
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "-V", "-version", "--version", "version":
+			fmt.Fprintf(os.Stdout, "geoip-update %s\n", version)
+			os.Exit(0)
+		case "help", "-help", "--help":
+			fmt.Fprintf(os.Stdout, "geoip-update %s\n\n", version)
+			fs.SetOutput(os.Stdout)
+			fs.Usage()
+			os.Exit(0)
+		}
+	}
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			os.Exit(0)
+		}
+		os.Exit(1)
+	}
+
+	data, err := os.ReadFile(cfg.ConfPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+	conf, err := geoip.ParseConf(string(data))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+
+	outDir := cfg.Dir
+	if outDir == "" {
+		outDir = conf.DatabaseDirectory
+	}
+	if outDir == "" {
+		outDir = "."
+	}
+	if err := os.MkdirAll(outDir, 0o755); err != nil {
+		fmt.Fprintf(os.Stderr, "error: mkdir %s: %v\n", outDir, err)
+		os.Exit(1)
+	}
+
+	if len(conf.EditionIDs) == 0 {
+		fmt.Fprintf(os.Stderr, "error: no EditionIDs found in %s\n", cfg.ConfPath)
+		os.Exit(1)
+	}
+
+	authHeader := http.Header{"Authorization": []string{httpcache.BasicAuth(conf.AccountID, conf.LicenseKey)}}
+	maxAge := time.Duration(cfg.FreshDays) * 24 * time.Hour
+
+	exitCode := 0
+	for _, edition := range conf.EditionIDs {
+		path := filepath.Join(outDir, geoip.TarGzName(edition))
+		cacher := &httpcache.Cacher{
+			URL:    geoip.DownloadBase + "/" + edition + "/download?suffix=tar.gz",
+			Path:   path,
+			MaxAge: maxAge,
+			Header: authHeader,
+		}
+		fmt.Fprintf(os.Stderr, "Fetching %s... ", edition)
+		t := time.Now()
+		updated, err := cacher.Fetch()
+		if err != nil {
+			fmt.Fprintln(os.Stderr)
+			fmt.Fprintf(os.Stderr, "error: %s: %v\n", edition, err)
+			exitCode = 1
+			continue
+		}
+		state := "fresh"
+		if updated {
+			state = "updated"
+		}
+		fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state)
+		info, _ := os.Stat(path)
+		fmt.Printf("%-10s %s -> %s (%s)\n", state+":", edition, path, info.ModTime().Format("2006-01-02"))
+	}
+	os.Exit(exitCode)
+}

net/geoip/conf.go

@@ -0,0 +1,59 @@
+package geoip
+
+import (
+	"bufio"
+	"errors"
+	"strings"
+)
+
+// Conf holds the fields parsed from a geoipupdate-style config file.
+//
+// GeoLite2 is free: sign up at https://www.maxmind.com/en/geolite2/signup
+// to get an AccountID and generate a LicenseKey, then write them to
+// GeoIP.conf alongside the desired EditionIDs:
+//
+//	AccountID   123456
+//	LicenseKey  xxxxxxxxxxxxxxxx
+//	EditionIDs  GeoLite2-City GeoLite2-ASN
+type Conf struct {
+	AccountID         string
+	LicenseKey        string
+	EditionIDs        []string
+	DatabaseDirectory string
+}
+
+// ErrMissingCredentials is returned by ParseConf when AccountID or LicenseKey
+// is absent from the input.
+var ErrMissingCredentials = errors.New("AccountID and LicenseKey are required")
+
+// ParseConf parses a geoipupdate-style config (whitespace-separated key/value
+// pairs, # comments). Compatible with GeoIP.conf files used by the official
+// geoipupdate tool.
+func ParseConf(s string) (*Conf, error) {
+	kv := make(map[string]string)
+	scanner := bufio.NewScanner(strings.NewReader(s))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, value, _ := strings.Cut(line, " ")
+		kv[strings.TrimSpace(key)] = strings.TrimSpace(value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	c := &Conf{
+		AccountID:         kv["AccountID"],
+		LicenseKey:        kv["LicenseKey"],
+		DatabaseDirectory: kv["DatabaseDirectory"],
+	}
+	if c.AccountID == "" || c.LicenseKey == "" {
+		return nil, ErrMissingCredentials
+	}
+	if ids := kv["EditionIDs"]; ids != "" {
+		c.EditionIDs = strings.Fields(ids)
+	}
+	return c, nil
+}

net/geoip/databases.go

@@ -0,0 +1,141 @@
+package geoip
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/oschwald/geoip2-golang"
+)
+
+// Databases holds open GeoLite2 City + ASN readers.
+type Databases struct {
+	City *geoip2.Reader
+	ASN  *geoip2.Reader
+}
+
+// Open loads the City and ASN editions from dir. For each edition it
+// prefers <edition>_LATEST.tar.gz and falls back to the
+// lexicographically greatest <edition>_*.tar.gz match (MaxMind's
+// Content-Disposition names sort chronologically by release date).
+// Archives are extracted in memory — no .mmdb files are written to disk.
+func Open(dir string) (*Databases, error) {
+	cityPath, err := FindTarGz(dir, CityEdition)
+	if err != nil {
+		return nil, fmt.Errorf("city: %w", err)
+	}
+	city, err := openMMDBTarGz(cityPath)
+	if err != nil {
+		return nil, fmt.Errorf("city: %w", err)
+	}
+	asnPath, err := FindTarGz(dir, ASNEdition)
+	if err != nil {
+		_ = city.Close()
+		return nil, fmt.Errorf("asn: %w", err)
+	}
+	asn, err := openMMDBTarGz(asnPath)
+	if err != nil {
+		_ = city.Close()
+		return nil, fmt.Errorf("asn: %w", err)
+	}
+	return &Databases{City: city, ASN: asn}, nil
+}
+
+// FindTarGz resolves the cached tarball path for edition inside dir,
+// preferring <edition>_LATEST.tar.gz and falling back to the
+// lexicographically greatest <edition>_*.tar.gz match.
+func FindTarGz(dir, edition string) (string, error) {
+	preferred := filepath.Join(dir, TarGzName(edition))
+	if _, err := os.Stat(preferred); err == nil {
+		return preferred, nil
+	}
+	matches, err := filepath.Glob(filepath.Join(dir, edition+"_*.tar.gz"))
+	if err != nil {
+		return "", err
+	}
+	if len(matches) == 0 {
+		return "", fmt.Errorf("no %s_*.tar.gz in %s", edition, dir)
+	}
+	slices.Sort(matches)
+	return matches[len(matches)-1], nil
+}
+
+func openMMDBTarGz(path string) (*geoip2.Reader, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	gr, err := gzip.NewReader(f)
+	if err != nil {
+		return nil, fmt.Errorf("gzip %s: %w", path, err)
+	}
+	defer gr.Close()
+	tr := tar.NewReader(gr)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("no .mmdb entry in %s", path)
+		}
+		if err != nil {
+			return nil, err
+		}
+		if !strings.HasSuffix(hdr.Name, ".mmdb") {
+			continue
+		}
+		data, err := io.ReadAll(tr)
+		if err != nil {
+			return nil, err
+		}
+		return geoip2.FromBytes(data)
+	}
+}
+
+// Close closes the city and ASN readers.
+func (d *Databases) Close() error {
+	return errors.Join(d.City.Close(), d.ASN.Close())
+}
+
+// Info is the structured result of a GeoIP lookup.
+type Info struct {
+	City       string `json:"city,omitempty"`
+	Region     string `json:"region,omitempty"`
+	Country    string `json:"country,omitempty"`
+	CountryISO string `json:"country_iso,omitempty"`
+	ASN        uint   `json:"asn,omitzero"`
+	ASNOrg     string `json:"asn_org,omitempty"`
+}
+
+// Lookup returns city + ASN info for ip. Returns a zero Info on unparseable
+// IP or database miss.
+func (d *Databases) Lookup(ip string) Info {
+	var info Info
+	addr, err := netip.ParseAddr(ip)
+	if err != nil {
+		return info
+	}
+	stdIP := addr.AsSlice()
+
+	if rec, err := d.City.City(stdIP); err == nil {
+		info.City = rec.City.Names["en"]
+		info.Country = rec.Country.Names["en"]
+		info.CountryISO = rec.Country.IsoCode
+		if len(rec.Subdivisions) > 0 {
+			if sub := rec.Subdivisions[0].Names["en"]; sub != "" && sub != info.City {
+				info.Region = sub
+			}
+		}
+	}
+	if rec, err := d.ASN.ASN(stdIP); err == nil {
+		info.ASN = rec.AutonomousSystemNumber
+		info.ASNOrg = rec.AutonomousSystemOrganization
+	}
+	return info
+}

net/geoip/geoip.go

@@ -0,0 +1,46 @@
+package geoip
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const (
+	CityEdition    = "GeoLite2-City"
+	ASNEdition     = "GeoLite2-ASN"
+	CountryEdition = "GeoLite2-Country"
+
+	// DownloadBase is the MaxMind databases download endpoint. Full URL:
+	// <DownloadBase>/<edition>/download?suffix=tar.gz
+	DownloadBase = "https://download.maxmind.com/geoip/databases"
+)
+
+// TarGzName returns the cache filename for edition's tar.gz archive.
+// MaxMind's Content-Disposition names include a release date
+// (e.g. GeoLite2-ASN_20260101.tar.gz); we use _LATEST so httpcache's
+// ETag sidecar stays tied to a stable path across releases.
+func TarGzName(edition string) string {
+	return edition + "_LATEST.tar.gz"
+}
+
+// DefaultConfPaths returns the standard locations where GeoIP.conf is looked
+// up: ./GeoIP.conf, then ~/.config/maxmind/GeoIP.conf.
+func DefaultConfPaths() []string {
+	paths := []string{"GeoIP.conf"}
+	if home, err := os.UserHomeDir(); err == nil {
+		paths = append(paths, filepath.Join(home, ".config", "maxmind", "GeoIP.conf"))
+	}
+	return paths
+}
+
+// DefaultCacheDir returns ~/.cache/maxmind. CLI tools use the XDG
+// convention on all platforms — os.UserCacheDir's macOS default
+// (~/Library/Caches) is meant for bundled desktop apps and hides the
+// files from anyone looking under ~/.cache.
+func DefaultCacheDir() (string, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, ".cache", "maxmind"), nil
+}

net/geoip/geoip_integration_test.go

@@ -0,0 +1,114 @@
+//go:build integration
+
+package geoip_test
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/geoip"
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+func testdataDir(t *testing.T) string {
+	t.Helper()
+	dir, _ := filepath.Abs(".")
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return filepath.Join(dir, "testdata")
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			t.Fatal("could not find module root (go.mod)")
+		}
+		dir = parent
+	}
+}
+
+func geoipConf(t *testing.T) *geoip.Conf {
+	t.Helper()
+	dir, _ := filepath.Abs(".")
+	for {
+		p := filepath.Join(dir, "GeoIP.conf")
+		if data, err := os.ReadFile(p); err == nil {
+			cfg, err := geoip.ParseConf(string(data))
+			if err != nil {
+				t.Fatalf("GeoIP.conf: %v", err)
+			}
+			return cfg
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			break
+		}
+		dir = parent
+	}
+	t.Skip("GeoIP.conf not found; skipping MaxMind integration test")
+	return nil
+}
+
+func newCacher(cfg *geoip.Conf, edition, path string) *httpcache.Cacher {
+	return &httpcache.Cacher{
+		URL:  geoip.DownloadBase + "/" + edition + "/download?suffix=tar.gz",
+		Path: path,
+		Header: http.Header{
+			"Authorization": []string{httpcache.BasicAuth(cfg.AccountID, cfg.LicenseKey)},
+		},
+	}
+}
+
+func TestDownload_CityAndASN(t *testing.T) {
+	cfg := geoipConf(t)
+	td := testdataDir(t)
+
+	for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} {
+		path := filepath.Join(td, geoip.TarGzName(edition))
+		os.Remove(path)
+		os.Remove(path + ".meta")
+
+		updated, err := newCacher(cfg, edition, path).Fetch()
+		if err != nil {
+			t.Fatalf("%s Fetch: %v", edition, err)
+		}
+		if !updated {
+			t.Errorf("%s: expected updated=true on first fetch", edition)
+		}
+
+		info, err := os.Stat(path)
+		if err != nil {
+			t.Fatalf("%s: file not created: %v", edition, err)
+		}
+		if info.Size() == 0 {
+			t.Errorf("%s: downloaded file is empty", edition)
+		}
+		t.Logf("%s: %d bytes", edition, info.Size())
+
+		if _, err := os.Stat(path + ".meta"); err != nil {
+			t.Errorf("%s: sidecar not written: %v", edition, err)
+		}
+	}
+}
+
+func TestDownload_ConditionalGet_FreshCacher(t *testing.T) {
+	cfg := geoipConf(t)
+	td := testdataDir(t)
+
+	for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} {
+		path := filepath.Join(td, geoip.TarGzName(edition))
+
+		if _, err := newCacher(cfg, edition, path).Fetch(); err != nil {
+			t.Fatalf("%s initial Fetch: %v", edition, err)
+		}
+
+		updated, err := newCacher(cfg, edition, path).Fetch()
+		if err != nil {
+			t.Fatalf("%s fresh Fetch: %v", edition, err)
+		}
+		if updated {
+			t.Errorf("%s: fresh cacher expected updated=false (sidecar ETag should have been used)", edition)
+		}
+		t.Logf("%s: fresh-cacher conditional GET correctly skipped re-download", edition)
+	}
+}

net/geoip/go.mod

@@ -0,0 +1,8 @@
+module github.com/therootcompany/golib/net/geoip
+
+go 1.26.0
+
+require (
+	github.com/oschwald/geoip2-golang v1.13.0
+	github.com/therootcompany/golib/net/httpcache v0.0.0
+)