diff --git a/.gitignore b/.gitignore index aa1e288..90254d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ credentials.tsv +GeoIP.conf +*.mmdb testdata/ .env diff --git a/net/geoip/conf.go b/net/geoip/conf.go new file mode 100644 index 0000000..3f984db --- /dev/null +++ b/net/geoip/conf.go @@ -0,0 +1,59 @@ +package geoip + +import ( + "bufio" + "errors" + "strings" +) + +// Conf holds the fields parsed from a geoipupdate-style config file. +// +// GeoLite2 is free: sign up at https://www.maxmind.com/en/geolite2/signup +// to get an AccountID and generate a LicenseKey, then write them to +// GeoIP.conf alongside the desired EditionIDs: +// +// AccountID 123456 +// LicenseKey xxxxxxxxxxxxxxxx +// EditionIDs GeoLite2-City GeoLite2-ASN +type Conf struct { + AccountID string + LicenseKey string + EditionIDs []string + DatabaseDirectory string +} + +// ErrMissingCredentials is returned by ParseConf when AccountID or LicenseKey +// is absent from the input. +var ErrMissingCredentials = errors.New("AccountID and LicenseKey are required") + +// ParseConf parses a geoipupdate-style config (whitespace-separated key/value +// pairs, # comments). Compatible with GeoIP.conf files used by the official +// geoipupdate tool. +func ParseConf(s string) (*Conf, error) { + kv := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, _ := strings.Cut(line, " ") + kv[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + if err := scanner.Err(); err != nil { + return nil, err + } + + c := &Conf{ + AccountID: kv["AccountID"], + LicenseKey: kv["LicenseKey"], + DatabaseDirectory: kv["DatabaseDirectory"], + } + if c.AccountID == "" || c.LicenseKey == "" { + return nil, ErrMissingCredentials + } + if ids := kv["EditionIDs"]; ids != "" { + c.EditionIDs = strings.Fields(ids) + } + return c, nil +} diff --git a/net/geoip/databases.go b/net/geoip/databases.go new file mode 100644 index 0000000..caa22f0 --- /dev/null +++ b/net/geoip/databases.go @@ -0,0 +1,141 @@ +package geoip + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "net/netip" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/oschwald/geoip2-golang" +) + +// Databases holds open GeoLite2 City + ASN readers. +type Databases struct { + City *geoip2.Reader + ASN *geoip2.Reader +} + +// Open loads the City and ASN editions from dir. For each edition it +// prefers _LATEST.tar.gz and falls back to the +// lexicographically greatest _*.tar.gz match (MaxMind's +// Content-Disposition names sort chronologically by release date). +// Archives are extracted in memory — no .mmdb files are written to disk. +func Open(dir string) (*Databases, error) { + cityPath, err := FindTarGz(dir, CityEdition) + if err != nil { + return nil, fmt.Errorf("city: %w", err) + } + city, err := openMMDBTarGz(cityPath) + if err != nil { + return nil, fmt.Errorf("city: %w", err) + } + asnPath, err := FindTarGz(dir, ASNEdition) + if err != nil { + _ = city.Close() + return nil, fmt.Errorf("asn: %w", err) + } + asn, err := openMMDBTarGz(asnPath) + if err != nil { + _ = city.Close() + return nil, fmt.Errorf("asn: %w", err) + } + return &Databases{City: city, ASN: asn}, nil +} + +// FindTarGz resolves the cached tarball path for edition inside dir, +// preferring _LATEST.tar.gz and falling back to the +// lexicographically greatest _*.tar.gz match. +func FindTarGz(dir, edition string) (string, error) { + preferred := filepath.Join(dir, TarGzName(edition)) + if _, err := os.Stat(preferred); err == nil { + return preferred, nil + } + matches, err := filepath.Glob(filepath.Join(dir, edition+"_*.tar.gz")) + if err != nil { + return "", err + } + if len(matches) == 0 { + return "", fmt.Errorf("no %s_*.tar.gz in %s", edition, dir) + } + slices.Sort(matches) + return matches[len(matches)-1], nil +} + +func openMMDBTarGz(path string) (*geoip2.Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + gr, err := gzip.NewReader(f) + if err != nil { + return nil, fmt.Errorf("gzip %s: %w", path, err) + } + defer gr.Close() + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil, fmt.Errorf("no .mmdb entry in %s", path) + } + if err != nil { + return nil, err + } + if !strings.HasSuffix(hdr.Name, ".mmdb") { + continue + } + data, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + return geoip2.FromBytes(data) + } +} + +// Close closes the city and ASN readers. +func (d *Databases) Close() error { + return errors.Join(d.City.Close(), d.ASN.Close()) +} + +// Info is the structured result of a GeoIP lookup. +type Info struct { + City string `json:"city,omitempty"` + Region string `json:"region,omitempty"` + Country string `json:"country,omitempty"` + CountryISO string `json:"country_iso,omitempty"` + ASN uint `json:"asn,omitzero"` + ASNOrg string `json:"asn_org,omitempty"` +} + +// Lookup returns city + ASN info for ip. Returns a zero Info on unparseable +// IP or database miss. +func (d *Databases) Lookup(ip string) Info { + var info Info + addr, err := netip.ParseAddr(ip) + if err != nil { + return info + } + stdIP := addr.AsSlice() + + if rec, err := d.City.City(stdIP); err == nil { + info.City = rec.City.Names["en"] + info.Country = rec.Country.Names["en"] + info.CountryISO = rec.Country.IsoCode + if len(rec.Subdivisions) > 0 { + if sub := rec.Subdivisions[0].Names["en"]; sub != "" && sub != info.City { + info.Region = sub + } + } + } + if rec, err := d.ASN.ASN(stdIP); err == nil { + info.ASN = rec.AutonomousSystemNumber + info.ASNOrg = rec.AutonomousSystemOrganization + } + return info +} diff --git a/net/geoip/geoip.go b/net/geoip/geoip.go new file mode 100644 index 0000000..c4490d4 --- /dev/null +++ b/net/geoip/geoip.go @@ -0,0 +1,46 @@ +package geoip + +import ( + "os" + "path/filepath" +) + +const ( + CityEdition = "GeoLite2-City" + ASNEdition = "GeoLite2-ASN" + CountryEdition = "GeoLite2-Country" + + // DownloadBase is the MaxMind databases download endpoint. Full URL: + // //download?suffix=tar.gz + DownloadBase = "https://download.maxmind.com/geoip/databases" +) + +// TarGzName returns the cache filename for edition's tar.gz archive. +// MaxMind's Content-Disposition names include a release date +// (e.g. GeoLite2-ASN_20260101.tar.gz); we use _LATEST so httpcache's +// ETag sidecar stays tied to a stable path across releases. +func TarGzName(edition string) string { + return edition + "_LATEST.tar.gz" +} + +// DefaultConfPaths returns the standard locations where GeoIP.conf is looked +// up: ./GeoIP.conf, then ~/.config/maxmind/GeoIP.conf. +func DefaultConfPaths() []string { + paths := []string{"GeoIP.conf"} + if home, err := os.UserHomeDir(); err == nil { + paths = append(paths, filepath.Join(home, ".config", "maxmind", "GeoIP.conf")) + } + return paths +} + +// DefaultCacheDir returns ~/.cache/maxmind. CLI tools use the XDG +// convention on all platforms — os.UserCacheDir's macOS default +// (~/Library/Caches) is meant for bundled desktop apps and hides the +// files from anyone looking under ~/.cache. +func DefaultCacheDir() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".cache", "maxmind"), nil +} diff --git a/net/geoip/geoip_integration_test.go b/net/geoip/geoip_integration_test.go new file mode 100644 index 0000000..2887b4d --- /dev/null +++ b/net/geoip/geoip_integration_test.go @@ -0,0 +1,115 @@ +//go:build integration + +package geoip_test + +import ( + "encoding/base64" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/geoip" + "github.com/therootcompany/golib/net/httpcache" +) + +func testdataDir(t *testing.T) string { + t.Helper() + dir, _ := filepath.Abs(".") + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata") + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find module root (go.mod)") + } + dir = parent + } +} + +func geoipConf(t *testing.T) *geoip.Conf { + t.Helper() + dir, _ := filepath.Abs(".") + for { + p := filepath.Join(dir, "GeoIP.conf") + if data, err := os.ReadFile(p); err == nil { + cfg, err := geoip.ParseConf(string(data)) + if err != nil { + t.Fatalf("GeoIP.conf: %v", err) + } + return cfg + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + t.Skip("GeoIP.conf not found; skipping MaxMind integration test") + return nil +} + +func newCacher(cfg *geoip.Conf, edition, path string) *httpcache.Cacher { + return &httpcache.Cacher{ + URL: geoip.DownloadBase + "/" + edition + "/download?suffix=tar.gz", + Path: path, + Header: http.Header{ + "Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(cfg.AccountID+":"+cfg.LicenseKey))}, + }, + } +} + +func TestDownload_CityAndASN(t *testing.T) { + cfg := geoipConf(t) + td := testdataDir(t) + + for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} { + path := filepath.Join(td, geoip.TarGzName(edition)) + os.Remove(path) + os.Remove(path + ".meta") + + updated, err := newCacher(cfg, edition, path).Fetch(t.Context()) + if err != nil { + t.Fatalf("%s Fetch: %v", edition, err) + } + if !updated { + t.Errorf("%s: expected updated=true on first fetch", edition) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("%s: file not created: %v", edition, err) + } + if info.Size() == 0 { + t.Errorf("%s: downloaded file is empty", edition) + } + t.Logf("%s: %d bytes", edition, info.Size()) + + if _, err := os.Stat(path + ".meta"); err != nil { + t.Errorf("%s: sidecar not written: %v", edition, err) + } + } +} + +func TestDownload_ConditionalGet_FreshCacher(t *testing.T) { + cfg := geoipConf(t) + td := testdataDir(t) + + for _, edition := range []string{geoip.CityEdition, geoip.ASNEdition} { + path := filepath.Join(td, geoip.TarGzName(edition)) + + if _, err := newCacher(cfg, edition, path).Fetch(t.Context()); err != nil { + t.Fatalf("%s initial Fetch: %v", edition, err) + } + + updated, err := newCacher(cfg, edition, path).Fetch(t.Context()) + if err != nil { + t.Fatalf("%s fresh Fetch: %v", edition, err) + } + if updated { + t.Errorf("%s: fresh cacher expected updated=false (sidecar ETag should have been used)", edition) + } + t.Logf("%s: fresh-cacher conditional GET correctly skipped re-download", edition) + } +} diff --git a/net/geoip/go.mod b/net/geoip/go.mod new file mode 100644 index 0000000..d0e30c5 --- /dev/null +++ b/net/geoip/go.mod @@ -0,0 +1,14 @@ +module github.com/therootcompany/golib/net/geoip + +go 1.26.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 + github.com/therootcompany/golib/net/httpcache v0.5.0 +) + +require ( + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.45.0 // indirect +) diff --git a/net/geoip/go.sum b/net/geoip/go.sum new file mode 100644 index 0000000..7f23340 --- /dev/null +++ b/net/geoip/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= +github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/therootcompany/golib/net/httpcache v0.5.0 h1:elq7S8gKJRv/0bH/7xYElI6H3Sf7RNyTZ1Qd9a5e2Zs= +github.com/therootcompany/golib/net/httpcache v0.5.0/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=