diff --git a/go.mod b/go.mod index 0f9cd88..fc9efa0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module github.com/therootcompany/golib go 1.26.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + golang.org/x/sys v0.20.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fcb2781 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= +github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/net/geoip/geoip.go b/net/geoip/geoip.go index 1c79c46..0694d5e 100644 --- a/net/geoip/geoip.go +++ b/net/geoip/geoip.go @@ -5,10 +5,11 @@ import ( "compress/gzip" "fmt" "io" - "net/http" "os" "strings" "time" + + "github.com/therootcompany/golib/net/httpcache" ) const ( @@ -22,11 +23,8 @@ const ( ) // Downloader fetches MaxMind GeoLite2 .mmdb files from the download API. -// It checks file mtime before downloading to stay within the 30/day rate limit. -// -// MaxMind preserves the database release date as the mtime of the .mmdb entry -// inside the tar archive. After extraction, mtime reflects data age — not -// download time — so it is reliable for freshness checks across restarts. +// For one-shot use call Fetch; for polling loops call NewCacher and reuse +// the Cacher so ETag state is preserved across calls. type Downloader struct { AccountID string LicenseKey string @@ -39,62 +37,39 @@ func New(accountID, licenseKey string) *Downloader { return &Downloader{AccountID: accountID, LicenseKey: licenseKey} } -// Fetch downloads the named edition to path if the file is stale (mtime older -// than FreshDays). Returns whether the file was updated. -func (d *Downloader) Fetch(edition, path string) (bool, error) { +// NewCacher returns an httpcache.Cacher pre-configured for this edition and +// path. Hold the Cacher and call Fetch() on it periodically — ETag state is +// preserved across calls, enabling conditional GETs that skip the download +// count on unchanged releases. +func (d *Downloader) NewCacher(edition, path string) *httpcache.Cacher { freshDays := d.FreshDays if freshDays == 0 { freshDays = defaultFreshDays } - - if info, err := os.Stat(path); err == nil { - if time.Since(info.ModTime()) < time.Duration(freshDays)*24*time.Hour { - return false, nil - } - } - timeout := d.Timeout if timeout == 0 { timeout = defaultTimeout } - - url := fmt.Sprintf("%s/%s/download?suffix=tar.gz", downloadBase, edition) - - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return false, err + return &httpcache.Cacher{ + URL: fmt.Sprintf("%s/%s/download?suffix=tar.gz", downloadBase, edition), + Path: path, + MaxAge: time.Duration(freshDays) * 24 * time.Hour, + Timeout: timeout, + Username: d.AccountID, + Password: d.LicenseKey, + Transform: ExtractMMDB, } - req.SetBasicAuth(d.AccountID, d.LicenseKey) - - // Strip auth on redirects: MaxMind issues a 302 to a Cloudflare R2 presigned - // URL that must not receive our credentials. - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - req.Header.Del("Authorization") - return nil - }, - } - - resp, err := client.Do(req) - if err != nil { - return false, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf("unexpected status %d fetching %s", resp.StatusCode, url) - } - - if err := extractMMDB(resp.Body, path); err != nil { - return false, fmt.Errorf("%s: %w", edition, err) - } - return true, nil } -// extractMMDB reads a MaxMind tar.gz archive, writes the .mmdb entry to path +// Fetch downloads edition to path if the file is stale. Convenience wrapper +// around NewCacher for one-shot use; ETag state is not retained. +func (d *Downloader) Fetch(edition, path string) (bool, error) { + return d.NewCacher(edition, path).Fetch() +} + +// ExtractMMDB reads a MaxMind tar.gz archive, writes the .mmdb entry to path // atomically (via tmp+rename), and sets its mtime to MaxMind's release date. -func extractMMDB(r io.Reader, path string) error { +func ExtractMMDB(r io.Reader, path string) error { gr, err := gzip.NewReader(r) if err != nil { return err diff --git a/net/httpcache/httpcache.go b/net/httpcache/httpcache.go index 7f39b9d..cee6387 100644 --- a/net/httpcache/httpcache.go +++ b/net/httpcache/httpcache.go @@ -13,14 +13,35 @@ const defaultTimeout = 30 * time.Second // Cacher fetches a URL to a local file, using ETag/Last-Modified to skip // unchanged responses. +// +// Rate limiting — two independent gates, both checked before any HTTP: +// - MaxAge: skips if the local file's mtime is within this duration. +// Useful when the remote preserves meaningful timestamps (e.g. MaxMind +// encodes the database release date as the tar entry mtime). +// - MinInterval: skips if Fetch was called within this duration (in-memory). +// Guards against tight poll loops hammering a rate-limited API. +// +// Auth — Username/Password sets HTTP Basic Auth on the initial request only. +// The Authorization header is stripped before following any redirect, so +// presigned redirect targets (e.g. Cloudflare R2) never receive credentials. +// +// Transform — if set, called with the response body instead of the default +// atomic file copy. The func is responsible for writing to path atomically. +// Use this for archives (e.g. extracting a .mmdb from a MaxMind tar.gz). type Cacher struct { - URL string - Path string - Timeout time.Duration // 0 uses 30s + URL string + Path string + Timeout time.Duration // 0 uses 30s + MaxAge time.Duration // 0 disables; skip HTTP if file mtime is within this + MinInterval time.Duration // 0 disables; skip HTTP if last Fetch attempt was within this + Username string // Basic Auth — not forwarded on redirects + Password string + Transform func(r io.Reader, path string) error // nil = direct atomic copy - mu sync.Mutex - etag string - lastMod string + mu sync.Mutex + etag string + lastMod string + lastChecked time.Time } // New creates a Cacher that fetches URL and writes it to path. @@ -30,10 +51,29 @@ func New(url, path string) *Cacher { // Fetch sends a conditional GET and writes new content to Path if the server // responds with 200. Returns whether the file was updated. +// +// Both MaxAge and MinInterval are checked before making any HTTP request. func (c *Cacher) Fetch() (updated bool, err error) { + // MaxAge: file-mtime gate (no lock needed — just a stat). + if c.MaxAge > 0 { + if info, err := os.Stat(c.Path); err == nil { + if time.Since(info.ModTime()) < c.MaxAge { + return false, nil + } + } + } + c.mu.Lock() defer c.mu.Unlock() + // MinInterval: in-memory last-checked gate. + if c.MinInterval > 0 && !c.lastChecked.IsZero() { + if time.Since(c.lastChecked) < c.MinInterval { + return false, nil + } + } + c.lastChecked = time.Now() + timeout := c.Timeout if timeout == 0 { timeout = defaultTimeout @@ -50,7 +90,22 @@ func (c *Cacher) Fetch() (updated bool, err error) { req.Header.Set("If-Modified-Since", c.lastMod) } - client := &http.Client{Timeout: timeout} + var client *http.Client + if c.Username != "" { + req.SetBasicAuth(c.Username, c.Password) + // Strip auth before following any redirect — presigned URLs (e.g. R2) + // must not receive our credentials. + client = &http.Client{ + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + req.Header.Del("Authorization") + return nil + }, + } + } else { + client = &http.Client{Timeout: timeout} + } + resp, err := client.Do(req) if err != nil { return false, err @@ -64,22 +119,26 @@ func (c *Cacher) Fetch() (updated bool, err error) { return false, fmt.Errorf("unexpected status %d fetching %s", resp.StatusCode, c.URL) } - // Write to a temp file then rename for an atomic swap. - tmp := c.Path + ".tmp" - f, err := os.Create(tmp) - if err != nil { - return false, err - } - if _, err := io.Copy(f, resp.Body); err != nil { + if c.Transform != nil { + if err := c.Transform(resp.Body, c.Path); err != nil { + return false, err + } + } else { + tmp := c.Path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return false, err + } + if _, err := io.Copy(f, resp.Body); err != nil { + f.Close() + os.Remove(tmp) + return false, err + } f.Close() - os.Remove(tmp) - return false, err - } - f.Close() - - if err := os.Rename(tmp, c.Path); err != nil { - os.Remove(tmp) - return false, err + if err := os.Rename(tmp, c.Path); err != nil { + os.Remove(tmp) + return false, err + } } if etag := resp.Header.Get("ETag"); etag != "" { diff --git a/net/ipcohort/cmd/check-ip-blacklist/blacklist.go b/net/ipcohort/cmd/check-ip/blacklist.go similarity index 100% rename from net/ipcohort/cmd/check-ip-blacklist/blacklist.go rename to net/ipcohort/cmd/check-ip/blacklist.go diff --git a/net/ipcohort/cmd/check-ip-blacklist/main.go b/net/ipcohort/cmd/check-ip/main.go similarity index 63% rename from net/ipcohort/cmd/check-ip-blacklist/main.go rename to net/ipcohort/cmd/check-ip/main.go index 53de620..825a115 100644 --- a/net/ipcohort/cmd/check-ip-blacklist/main.go +++ b/net/ipcohort/cmd/check-ip/main.go @@ -2,16 +2,19 @@ package main import ( "context" + "flag" "fmt" + "net/netip" "os" "strings" "sync/atomic" "time" + "github.com/oschwald/geoip2-golang" "github.com/therootcompany/golib/net/ipcohort" ) -// inbound blocklist - pre-separated by type for independent ETag caching +// inbound blocklist const ( inboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt" inboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt" @@ -24,25 +27,29 @@ const ( ) func main() { - if len(os.Args) < 3 { - fmt.Fprintf(os.Stderr, "Usage: %s [git-url]\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " No remote: load from (inbound only)\n") - fmt.Fprintf(os.Stderr, " git URL: clone/pull into \n") - fmt.Fprintf(os.Stderr, " (default): fetch via HTTP into \n") + cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb") + asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb") + gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) + fmt.Fprintf(os.Stderr, " data-dir: fetch blocklists via HTTP (or git with -git)\n") + fmt.Fprintf(os.Stderr, " blacklist.txt: load single local file as inbound list\n") + flag.PrintDefaults() + } + flag.Parse() + + if flag.NArg() < 2 { + flag.Usage() os.Exit(1) } - dataPath := os.Args[1] - ipStr := os.Args[2] - gitURL := "" - if len(os.Args) >= 4 { - gitURL = os.Args[3] - } + dataPath := flag.Arg(0) + ipStr := flag.Arg(1) var src *Sources switch { - case gitURL != "": - src = newGitSources(gitURL, dataPath, + case *gitURL != "": + src = newGitSources(*gitURL, dataPath, nil, []string{"tables/inbound/single_ips.txt", "tables/inbound/networks.txt"}, []string{"tables/outbound/single_ips.txt", "tables/outbound/networks.txt"}, @@ -77,6 +84,25 @@ func main() { fmt.Fprintf(os.Stderr, "Loaded inbound=%d outbound=%d\n", size(&inbound), size(&outbound)) + // GeoIP readers. + var cityDB, asnDB atomic.Pointer[geoip2.Reader] + if *cityDBPath != "" { + if r, err := geoip2.Open(*cityDBPath); err != nil { + fmt.Fprintf(os.Stderr, "warn: city-db: %v\n", err) + } else { + cityDB.Store(r) + defer r.Close() + } + } + if *asnDBPath != "" { + if r, err := geoip2.Open(*asnDBPath); err != nil { + fmt.Fprintf(os.Stderr, "warn: asn-db: %v\n", err) + } else { + asnDB.Store(r) + defer r.Close() + } + } + // Keep data fresh in the background if running as a daemon. ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -88,16 +114,56 @@ func main() { switch { case blockedInbound && blockedOutbound: fmt.Printf("%s is BLOCKED (inbound + outbound)\n", ipStr) - os.Exit(1) case blockedInbound: fmt.Printf("%s is BLOCKED (inbound)\n", ipStr) - os.Exit(1) case blockedOutbound: fmt.Printf("%s is BLOCKED (outbound)\n", ipStr) - os.Exit(1) default: fmt.Printf("%s is allowed\n", ipStr) } + + printGeoInfo(ipStr, &cityDB, &asnDB) + + if blockedInbound || blockedOutbound { + os.Exit(1) + } +} + +func printGeoInfo(ipStr string, cityDB, asnDB *atomic.Pointer[geoip2.Reader]) { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return + } + stdIP := ip.AsSlice() + + if r := cityDB.Load(); r != nil { + if rec, err := r.City(stdIP); err == nil { + city := rec.City.Names["en"] + country := rec.Country.Names["en"] + iso := rec.Country.IsoCode + var parts []string + if city != "" { + parts = append(parts, city) + } + if len(rec.Subdivisions) > 0 { + if sub := rec.Subdivisions[0].Names["en"]; sub != "" && sub != city { + parts = append(parts, sub) + } + } + if country != "" { + parts = append(parts, fmt.Sprintf("%s (%s)", country, iso)) + } + if len(parts) > 0 { + fmt.Printf(" Location: %s\n", strings.Join(parts, ", ")) + } + } + } + + if r := asnDB.Load(); r != nil { + if rec, err := r.ASN(stdIP); err == nil && rec.AutonomousSystemNumber != 0 { + fmt.Printf(" ASN: AS%d %s\n", rec.AutonomousSystemNumber, rec.AutonomousSystemOrganization) + } + } } func reload(src *Sources,