fix: explicit path flags for blocklist; auto-discover GeoIP.conf

Blocklist:
- Add -inbound, -outbound, -whitelist flags for explicit file paths
- buildSources() replaces the old constructor trio; explicit flags always win
- -data-dir and -git still work as defaults for the bitwire-it layout

GeoIP:
- Auto-discover GeoIP.conf from ./GeoIP.conf then ~/.config/maxmind/GeoIP.conf
- If no conf found and no -city-db/-asn-db given: geoip disabled silently
- If no conf but paths given: use those files (Init fails if absent)
This commit is contained in:
AJ ONeal 2026-04-20 12:18:33 -06:00
parent ddd0986e20
commit e1108f3de7
No known key found for this signature in database
3 changed files with 113 additions and 112 deletions

View File

@ -2,6 +2,7 @@ package main
import ( import (
"path/filepath" "path/filepath"
"strings"
"github.com/therootcompany/golib/net/dataset" "github.com/therootcompany/golib/net/dataset"
"github.com/therootcompany/golib/net/gitshallow" "github.com/therootcompany/golib/net/gitshallow"
@ -9,62 +10,76 @@ import (
"github.com/therootcompany/golib/net/ipcohort" "github.com/therootcompany/golib/net/ipcohort"
) )
// HTTPSource pairs a remote URL with a local cache path. // Sources holds fetch configuration for the blocklist cohorts.
type HTTPSource struct {
URL string
Path string
}
// Sources holds fetch configuration for the three blocklist cohorts.
// It knows how to pull data from git or HTTP, but owns no atomic state.
type Sources struct { type Sources struct {
whitelistPaths []string whitelistPaths []string
inboundPaths []string inboundPaths []string
outboundPaths []string outboundPaths []string
syncs []dataset.Syncer
syncs []dataset.Syncer // all syncable sources
} }
func newFileSources(whitelist, inbound, outbound []string) *Sources { // buildSources constructs the right Sources from CLI flags.
//
// - gitURL set → clone/pull the bitwire-it repo; inbound/outbound from known relative paths
// - inbound/outbound set → use those explicit file paths, no network sync
// - neither set → HTTP-fetch the bitwire-it files into dataDir (or default cache dir)
func buildSources(gitURL, dataDir, whitelistFlag, inboundFlag, outboundFlag string) *Sources {
// Explicit file paths always win.
if inboundFlag != "" || outboundFlag != "" {
return &Sources{ return &Sources{
whitelistPaths: whitelist, whitelistPaths: splitPaths(whitelistFlag),
inboundPaths: inbound, inboundPaths: splitPaths(inboundFlag),
outboundPaths: outbound, outboundPaths: splitPaths(outboundFlag),
}
} }
}
func newGitSources(gitURL, repoDir string, whitelist, inboundRel, outboundRel []string) *Sources { cacheDir := dataDir
abs := func(rel []string) []string { if cacheDir == "" {
out := make([]string, len(rel)) cacheDir = defaultCacheDir("bitwire-it")
for i, p := range rel {
out[i] = filepath.Join(repoDir, p)
} }
return out
} if gitURL != "" {
repo := gitshallow.New(gitURL, repoDir, 1, "") repo := gitshallow.New(gitURL, cacheDir, 1, "")
return &Sources{ return &Sources{
whitelistPaths: whitelist, whitelistPaths: splitPaths(whitelistFlag),
inboundPaths: abs(inboundRel), inboundPaths: []string{
outboundPaths: abs(outboundRel), filepath.Join(cacheDir, "tables/inbound/single_ips.txt"),
filepath.Join(cacheDir, "tables/inbound/networks.txt"),
},
outboundPaths: []string{
filepath.Join(cacheDir, "tables/outbound/single_ips.txt"),
filepath.Join(cacheDir, "tables/outbound/networks.txt"),
},
syncs: []dataset.Syncer{repo}, syncs: []dataset.Syncer{repo},
} }
}
// Default: HTTP fetch from bitwire-it into cacheDir.
inboundSingle := filepath.Join(cacheDir, "inbound_single_ips.txt")
inboundNetwork := filepath.Join(cacheDir, "inbound_networks.txt")
outboundSingle := filepath.Join(cacheDir, "outbound_single_ips.txt")
outboundNetwork := filepath.Join(cacheDir, "outbound_networks.txt")
return &Sources{
whitelistPaths: splitPaths(whitelistFlag),
inboundPaths: []string{inboundSingle, inboundNetwork},
outboundPaths: []string{outboundSingle, outboundNetwork},
syncs: []dataset.Syncer{
httpcache.New(inboundSingleURL, inboundSingle),
httpcache.New(inboundNetworkURL, inboundNetwork),
httpcache.New(outboundSingleURL, outboundSingle),
httpcache.New(outboundNetworkURL, outboundNetwork),
},
}
} }
func newHTTPSources(whitelist []string, inbound, outbound []HTTPSource) *Sources { func splitPaths(s string) []string {
s := &Sources{whitelistPaths: whitelist} if s == "" {
for _, src := range inbound { return nil
s.inboundPaths = append(s.inboundPaths, src.Path)
s.syncs = append(s.syncs, httpcache.New(src.URL, src.Path))
} }
for _, src := range outbound { return strings.Split(s, ",")
s.outboundPaths = append(s.outboundPaths, src.Path)
s.syncs = append(s.syncs, httpcache.New(src.URL, src.Path))
}
return s
} }
// Fetch pulls updates from all sources. Returns whether any new data arrived. // Fetch pulls updates from all sources. Satisfies dataset.Syncer.
// Satisfies dataset.Syncer.
func (s *Sources) Fetch() (bool, error) { func (s *Sources) Fetch() (bool, error) {
var anyUpdated bool var anyUpdated bool
for _, syn := range s.syncs { for _, syn := range s.syncs {
@ -77,9 +92,7 @@ func (s *Sources) Fetch() (bool, error) {
return anyUpdated, nil return anyUpdated, nil
} }
// Datasets builds a dataset.Group backed by this Sources and returns typed // Datasets builds a dataset.Group and returns typed views for each cohort.
// datasets for whitelist, inbound, and outbound cohorts. Either whitelist or
// outbound may be nil if no paths were configured.
func (s *Sources) Datasets() ( func (s *Sources) Datasets() (
g *dataset.Group, g *dataset.Group,
whitelist *dataset.View[ipcohort.Cohort], whitelist *dataset.View[ipcohort.Cohort],

View File

@ -8,12 +8,32 @@ import (
"github.com/therootcompany/golib/net/geoip" "github.com/therootcompany/golib/net/geoip"
) )
// setupGeo parses geoip-conf (if given) and returns a Databases ready to Init. // discoverConf looks for GeoIP.conf in the current directory and then
// Returns nil if no geoip flags were provided. // at ~/.config/maxmind/GeoIP.conf. Returns the path or "".
func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) { func discoverConf() string {
if confPath == "" && cityPath == "" && asnPath == "" { if _, err := os.Stat("GeoIP.conf"); err == nil {
return nil, nil return "GeoIP.conf"
} }
if home, err := os.UserHomeDir(); err == nil {
p := filepath.Join(home, ".config", "maxmind", "GeoIP.conf")
if _, err := os.Stat(p); err == nil {
return p
}
}
return ""
}
// setupGeo returns a Databases ready to Init, or nil if geoip is not configured.
//
// - confPath="" → auto-discover GeoIP.conf from cwd and ~/.config/maxmind/
// - conf found → auto-download; cityPath/asnPath override the default locations
// - conf absent → cityPath and asnPath must point to existing .mmdb files
// - no conf and no paths → geoip disabled (returns nil)
func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) {
if confPath == "" {
confPath = discoverConf()
}
if confPath != "" { if confPath != "" {
cfg, err := geoip.ParseConf(confPath) cfg, err := geoip.ParseConf(confPath)
if err != nil { if err != nil {
@ -36,5 +56,10 @@ func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) {
} }
return geoip.New(cfg.AccountID, cfg.LicenseKey).NewDatabases(cityPath, asnPath), nil return geoip.New(cfg.AccountID, cfg.LicenseKey).NewDatabases(cityPath, asnPath), nil
} }
if cityPath == "" && asnPath == "" {
return nil, nil
}
// Explicit paths only — no auto-download. Init will fail if files are absent.
return geoip.NewDatabases(cityPath, asnPath), nil return geoip.NewDatabases(cityPath, asnPath), nil
} }

View File

@ -5,102 +5,65 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strings" "path/filepath"
"time" "time"
) )
// inbound blocklist // Default HTTP sources for the bitwire-it blocklist.
const ( const (
inboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt" inboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
inboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt" inboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt"
)
// outbound blocklist
const (
outboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/single_ips.txt" outboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/single_ips.txt"
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt" outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
) )
func defaultBlocklistDir() string { func defaultCacheDir(sub string) string {
base, err := os.UserCacheDir() base, err := os.UserCacheDir()
if err != nil { if err != nil {
return os.Getenv("HOME") + "/.cache/bitwire-it" base = filepath.Join(os.Getenv("HOME"), ".cache")
} }
return base + "/bitwire-it" return filepath.Join(base, sub)
} }
func main() { func main() {
// Blocklist source flags — all optional; defaults pull from bitwire-it via HTTP.
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)") dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)") gitURL := flag.String("git", "", "git URL to clone/pull blocklist from (alternative to HTTP)")
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)") whitelist := flag.String("whitelist", "", "path to whitelist file (overrides block)")
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir") inbound := flag.String("inbound", "", "comma-separated paths to inbound blocklist files")
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir") outbound := flag.String("outbound", "", "comma-separated paths to outbound blocklist files")
// GeoIP flags — auto-discovered from ./GeoIP.conf or ~/.config/maxmind/GeoIP.conf.
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf (auto-discovered if absent)")
cityDB := flag.String("city-db", "", "path to GeoLite2-City.mmdb (skips auto-download)")
asnDB := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (skips auto-download)")
flag.Usage = func() { flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0]) fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
fmt.Fprintf(os.Stderr, " Blocklists are fetched via HTTP by default (use -git for git source).\n")
fmt.Fprintf(os.Stderr, " Pass a .txt/.csv path as the first arg to load a single local file.\n")
flag.PrintDefaults() flag.PrintDefaults()
} }
flag.Parse() flag.Parse()
if flag.NArg() < 1 { if flag.NArg() != 1 {
flag.Usage() flag.Usage()
os.Exit(1) os.Exit(1)
} }
ipStr := flag.Arg(0)
// First arg is either a local file or the IP to check.
var dataPath, ipStr string
if flag.NArg() >= 2 || strings.HasSuffix(flag.Arg(0), ".txt") || strings.HasSuffix(flag.Arg(0), ".csv") {
dataPath = flag.Arg(0)
ipStr = flag.Arg(1)
} else {
ipStr = flag.Arg(0)
}
if *dataDir != "" {
dataPath = *dataDir
}
if dataPath == "" {
dataPath = defaultBlocklistDir()
}
// -- Blocklist ---------------------------------------------------------- // -- Blocklist ----------------------------------------------------------
var src *Sources src := buildSources(*gitURL, *dataDir, *whitelist, *inbound, *outbound)
switch { blGroup, whitelistDS, inboundDS, outboundDS := src.Datasets()
case *gitURL != "":
src = newGitSources(*gitURL, dataPath,
nil,
[]string{"tables/inbound/single_ips.txt", "tables/inbound/networks.txt"},
[]string{"tables/outbound/single_ips.txt", "tables/outbound/networks.txt"},
)
case strings.HasSuffix(dataPath, ".txt") || strings.HasSuffix(dataPath, ".csv"):
src = newFileSources(nil, []string{dataPath}, nil)
default:
src = newHTTPSources(
nil,
[]HTTPSource{
{inboundSingleURL, dataPath + "/inbound_single_ips.txt"},
{inboundNetworkURL, dataPath + "/inbound_networks.txt"},
},
[]HTTPSource{
{outboundSingleURL, dataPath + "/outbound_single_ips.txt"},
{outboundNetworkURL, dataPath + "/outbound_networks.txt"},
},
)
}
blGroup, whitelist, inbound, outbound := src.Datasets()
if err := blGroup.Init(); err != nil { if err := blGroup.Init(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err) fmt.Fprintf(os.Stderr, "error: blocklist: %v\n", err)
os.Exit(1) os.Exit(1)
} }
fmt.Fprintf(os.Stderr, "Loaded inbound=%d outbound=%d\n", fmt.Fprintf(os.Stderr, "Loaded inbound=%d outbound=%d\n",
cohortSize(inbound), cohortSize(outbound)) cohortSize(inboundDS), cohortSize(outboundDS))
// -- GeoIP (optional) -------------------------------------------------- // -- GeoIP (optional) --------------------------------------------------
geo, err := setupGeo(*geoipConf, *cityDBPath, *asnDBPath) geo, err := setupGeo(*geoipConf, *cityDB, *asnDB)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err) fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1) os.Exit(1)
@ -119,8 +82,8 @@ func main() {
// -- Check and report -------------------------------------------------- // -- Check and report --------------------------------------------------
blockedIn := isBlocked(ipStr, whitelist, inbound) blockedIn := isBlocked(ipStr, whitelistDS, inboundDS)
blockedOut := isBlocked(ipStr, whitelist, outbound) blockedOut := isBlocked(ipStr, whitelistDS, outboundDS)
switch { switch {
case blockedIn && blockedOut: case blockedIn && blockedOut: