mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 20:58:00 +00:00
fix: explicit path flags for blocklist; auto-discover GeoIP.conf
Blocklist: - Add -inbound, -outbound, -whitelist flags for explicit file paths - buildSources() replaces the old constructor trio; explicit flags always win - -data-dir and -git still work as defaults for the bitwire-it layout GeoIP: - Auto-discover GeoIP.conf from ./GeoIP.conf then ~/.config/maxmind/GeoIP.conf - If no conf found and no -city-db/-asn-db given: geoip disabled silently - If no conf but paths given: use those files (Init fails if absent)
This commit is contained in:
parent
ddd0986e20
commit
e1108f3de7
@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/therootcompany/golib/net/dataset"
|
"github.com/therootcompany/golib/net/dataset"
|
||||||
"github.com/therootcompany/golib/net/gitshallow"
|
"github.com/therootcompany/golib/net/gitshallow"
|
||||||
@ -9,62 +10,76 @@ import (
|
|||||||
"github.com/therootcompany/golib/net/ipcohort"
|
"github.com/therootcompany/golib/net/ipcohort"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTPSource pairs a remote URL with a local cache path.
|
// Sources holds fetch configuration for the blocklist cohorts.
|
||||||
type HTTPSource struct {
|
|
||||||
URL string
|
|
||||||
Path string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sources holds fetch configuration for the three blocklist cohorts.
|
|
||||||
// It knows how to pull data from git or HTTP, but owns no atomic state.
|
|
||||||
type Sources struct {
|
type Sources struct {
|
||||||
whitelistPaths []string
|
whitelistPaths []string
|
||||||
inboundPaths []string
|
inboundPaths []string
|
||||||
outboundPaths []string
|
outboundPaths []string
|
||||||
|
syncs []dataset.Syncer
|
||||||
syncs []dataset.Syncer // all syncable sources
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFileSources(whitelist, inbound, outbound []string) *Sources {
|
// buildSources constructs the right Sources from CLI flags.
|
||||||
|
//
|
||||||
|
// - gitURL set → clone/pull the bitwire-it repo; inbound/outbound from known relative paths
|
||||||
|
// - inbound/outbound set → use those explicit file paths, no network sync
|
||||||
|
// - neither set → HTTP-fetch the bitwire-it files into dataDir (or default cache dir)
|
||||||
|
func buildSources(gitURL, dataDir, whitelistFlag, inboundFlag, outboundFlag string) *Sources {
|
||||||
|
// Explicit file paths always win.
|
||||||
|
if inboundFlag != "" || outboundFlag != "" {
|
||||||
return &Sources{
|
return &Sources{
|
||||||
whitelistPaths: whitelist,
|
whitelistPaths: splitPaths(whitelistFlag),
|
||||||
inboundPaths: inbound,
|
inboundPaths: splitPaths(inboundFlag),
|
||||||
outboundPaths: outbound,
|
outboundPaths: splitPaths(outboundFlag),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func newGitSources(gitURL, repoDir string, whitelist, inboundRel, outboundRel []string) *Sources {
|
cacheDir := dataDir
|
||||||
abs := func(rel []string) []string {
|
if cacheDir == "" {
|
||||||
out := make([]string, len(rel))
|
cacheDir = defaultCacheDir("bitwire-it")
|
||||||
for i, p := range rel {
|
|
||||||
out[i] = filepath.Join(repoDir, p)
|
|
||||||
}
|
}
|
||||||
return out
|
|
||||||
}
|
if gitURL != "" {
|
||||||
repo := gitshallow.New(gitURL, repoDir, 1, "")
|
repo := gitshallow.New(gitURL, cacheDir, 1, "")
|
||||||
return &Sources{
|
return &Sources{
|
||||||
whitelistPaths: whitelist,
|
whitelistPaths: splitPaths(whitelistFlag),
|
||||||
inboundPaths: abs(inboundRel),
|
inboundPaths: []string{
|
||||||
outboundPaths: abs(outboundRel),
|
filepath.Join(cacheDir, "tables/inbound/single_ips.txt"),
|
||||||
|
filepath.Join(cacheDir, "tables/inbound/networks.txt"),
|
||||||
|
},
|
||||||
|
outboundPaths: []string{
|
||||||
|
filepath.Join(cacheDir, "tables/outbound/single_ips.txt"),
|
||||||
|
filepath.Join(cacheDir, "tables/outbound/networks.txt"),
|
||||||
|
},
|
||||||
syncs: []dataset.Syncer{repo},
|
syncs: []dataset.Syncer{repo},
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: HTTP fetch from bitwire-it into cacheDir.
|
||||||
|
inboundSingle := filepath.Join(cacheDir, "inbound_single_ips.txt")
|
||||||
|
inboundNetwork := filepath.Join(cacheDir, "inbound_networks.txt")
|
||||||
|
outboundSingle := filepath.Join(cacheDir, "outbound_single_ips.txt")
|
||||||
|
outboundNetwork := filepath.Join(cacheDir, "outbound_networks.txt")
|
||||||
|
return &Sources{
|
||||||
|
whitelistPaths: splitPaths(whitelistFlag),
|
||||||
|
inboundPaths: []string{inboundSingle, inboundNetwork},
|
||||||
|
outboundPaths: []string{outboundSingle, outboundNetwork},
|
||||||
|
syncs: []dataset.Syncer{
|
||||||
|
httpcache.New(inboundSingleURL, inboundSingle),
|
||||||
|
httpcache.New(inboundNetworkURL, inboundNetwork),
|
||||||
|
httpcache.New(outboundSingleURL, outboundSingle),
|
||||||
|
httpcache.New(outboundNetworkURL, outboundNetwork),
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHTTPSources(whitelist []string, inbound, outbound []HTTPSource) *Sources {
|
func splitPaths(s string) []string {
|
||||||
s := &Sources{whitelistPaths: whitelist}
|
if s == "" {
|
||||||
for _, src := range inbound {
|
return nil
|
||||||
s.inboundPaths = append(s.inboundPaths, src.Path)
|
|
||||||
s.syncs = append(s.syncs, httpcache.New(src.URL, src.Path))
|
|
||||||
}
|
}
|
||||||
for _, src := range outbound {
|
return strings.Split(s, ",")
|
||||||
s.outboundPaths = append(s.outboundPaths, src.Path)
|
|
||||||
s.syncs = append(s.syncs, httpcache.New(src.URL, src.Path))
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch pulls updates from all sources. Returns whether any new data arrived.
|
// Fetch pulls updates from all sources. Satisfies dataset.Syncer.
|
||||||
// Satisfies dataset.Syncer.
|
|
||||||
func (s *Sources) Fetch() (bool, error) {
|
func (s *Sources) Fetch() (bool, error) {
|
||||||
var anyUpdated bool
|
var anyUpdated bool
|
||||||
for _, syn := range s.syncs {
|
for _, syn := range s.syncs {
|
||||||
@ -77,9 +92,7 @@ func (s *Sources) Fetch() (bool, error) {
|
|||||||
return anyUpdated, nil
|
return anyUpdated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Datasets builds a dataset.Group backed by this Sources and returns typed
|
// Datasets builds a dataset.Group and returns typed views for each cohort.
|
||||||
// datasets for whitelist, inbound, and outbound cohorts. Either whitelist or
|
|
||||||
// outbound may be nil if no paths were configured.
|
|
||||||
func (s *Sources) Datasets() (
|
func (s *Sources) Datasets() (
|
||||||
g *dataset.Group,
|
g *dataset.Group,
|
||||||
whitelist *dataset.View[ipcohort.Cohort],
|
whitelist *dataset.View[ipcohort.Cohort],
|
||||||
|
|||||||
@ -8,12 +8,32 @@ import (
|
|||||||
"github.com/therootcompany/golib/net/geoip"
|
"github.com/therootcompany/golib/net/geoip"
|
||||||
)
|
)
|
||||||
|
|
||||||
// setupGeo parses geoip-conf (if given) and returns a Databases ready to Init.
|
// discoverConf looks for GeoIP.conf in the current directory and then
|
||||||
// Returns nil if no geoip flags were provided.
|
// at ~/.config/maxmind/GeoIP.conf. Returns the path or "".
|
||||||
func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) {
|
func discoverConf() string {
|
||||||
if confPath == "" && cityPath == "" && asnPath == "" {
|
if _, err := os.Stat("GeoIP.conf"); err == nil {
|
||||||
return nil, nil
|
return "GeoIP.conf"
|
||||||
}
|
}
|
||||||
|
if home, err := os.UserHomeDir(); err == nil {
|
||||||
|
p := filepath.Join(home, ".config", "maxmind", "GeoIP.conf")
|
||||||
|
if _, err := os.Stat(p); err == nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupGeo returns a Databases ready to Init, or nil if geoip is not configured.
|
||||||
|
//
|
||||||
|
// - confPath="" → auto-discover GeoIP.conf from cwd and ~/.config/maxmind/
|
||||||
|
// - conf found → auto-download; cityPath/asnPath override the default locations
|
||||||
|
// - conf absent → cityPath and asnPath must point to existing .mmdb files
|
||||||
|
// - no conf and no paths → geoip disabled (returns nil)
|
||||||
|
func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) {
|
||||||
|
if confPath == "" {
|
||||||
|
confPath = discoverConf()
|
||||||
|
}
|
||||||
|
|
||||||
if confPath != "" {
|
if confPath != "" {
|
||||||
cfg, err := geoip.ParseConf(confPath)
|
cfg, err := geoip.ParseConf(confPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -36,5 +56,10 @@ func setupGeo(confPath, cityPath, asnPath string) (*geoip.Databases, error) {
|
|||||||
}
|
}
|
||||||
return geoip.New(cfg.AccountID, cfg.LicenseKey).NewDatabases(cityPath, asnPath), nil
|
return geoip.New(cfg.AccountID, cfg.LicenseKey).NewDatabases(cityPath, asnPath), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cityPath == "" && asnPath == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// Explicit paths only — no auto-download. Init will fail if files are absent.
|
||||||
return geoip.NewDatabases(cityPath, asnPath), nil
|
return geoip.NewDatabases(cityPath, asnPath), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,102 +5,65 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// inbound blocklist
|
// Default HTTP sources for the bitwire-it blocklist.
|
||||||
const (
|
const (
|
||||||
inboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
|
inboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
|
||||||
inboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt"
|
inboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt"
|
||||||
)
|
|
||||||
|
|
||||||
// outbound blocklist
|
|
||||||
const (
|
|
||||||
outboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/single_ips.txt"
|
outboundSingleURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/single_ips.txt"
|
||||||
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
|
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
|
||||||
)
|
)
|
||||||
|
|
||||||
func defaultBlocklistDir() string {
|
func defaultCacheDir(sub string) string {
|
||||||
base, err := os.UserCacheDir()
|
base, err := os.UserCacheDir()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return os.Getenv("HOME") + "/.cache/bitwire-it"
|
base = filepath.Join(os.Getenv("HOME"), ".cache")
|
||||||
}
|
}
|
||||||
return base + "/bitwire-it"
|
return filepath.Join(base, sub)
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
// Blocklist source flags — all optional; defaults pull from bitwire-it via HTTP.
|
||||||
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
|
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
|
||||||
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)")
|
gitURL := flag.String("git", "", "git URL to clone/pull blocklist from (alternative to HTTP)")
|
||||||
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)")
|
whitelist := flag.String("whitelist", "", "path to whitelist file (overrides block)")
|
||||||
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir")
|
inbound := flag.String("inbound", "", "comma-separated paths to inbound blocklist files")
|
||||||
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir")
|
outbound := flag.String("outbound", "", "comma-separated paths to outbound blocklist files")
|
||||||
|
|
||||||
|
// GeoIP flags — auto-discovered from ./GeoIP.conf or ~/.config/maxmind/GeoIP.conf.
|
||||||
|
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf (auto-discovered if absent)")
|
||||||
|
cityDB := flag.String("city-db", "", "path to GeoLite2-City.mmdb (skips auto-download)")
|
||||||
|
asnDB := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (skips auto-download)")
|
||||||
|
|
||||||
flag.Usage = func() {
|
flag.Usage = func() {
|
||||||
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
|
||||||
fmt.Fprintf(os.Stderr, " Blocklists are fetched via HTTP by default (use -git for git source).\n")
|
|
||||||
fmt.Fprintf(os.Stderr, " Pass a .txt/.csv path as the first arg to load a single local file.\n")
|
|
||||||
flag.PrintDefaults()
|
flag.PrintDefaults()
|
||||||
}
|
}
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
if flag.NArg() < 1 {
|
if flag.NArg() != 1 {
|
||||||
flag.Usage()
|
flag.Usage()
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
ipStr := flag.Arg(0)
|
||||||
// First arg is either a local file or the IP to check.
|
|
||||||
var dataPath, ipStr string
|
|
||||||
if flag.NArg() >= 2 || strings.HasSuffix(flag.Arg(0), ".txt") || strings.HasSuffix(flag.Arg(0), ".csv") {
|
|
||||||
dataPath = flag.Arg(0)
|
|
||||||
ipStr = flag.Arg(1)
|
|
||||||
} else {
|
|
||||||
ipStr = flag.Arg(0)
|
|
||||||
}
|
|
||||||
if *dataDir != "" {
|
|
||||||
dataPath = *dataDir
|
|
||||||
}
|
|
||||||
if dataPath == "" {
|
|
||||||
dataPath = defaultBlocklistDir()
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- Blocklist ----------------------------------------------------------
|
// -- Blocklist ----------------------------------------------------------
|
||||||
|
|
||||||
var src *Sources
|
src := buildSources(*gitURL, *dataDir, *whitelist, *inbound, *outbound)
|
||||||
switch {
|
blGroup, whitelistDS, inboundDS, outboundDS := src.Datasets()
|
||||||
case *gitURL != "":
|
|
||||||
src = newGitSources(*gitURL, dataPath,
|
|
||||||
nil,
|
|
||||||
[]string{"tables/inbound/single_ips.txt", "tables/inbound/networks.txt"},
|
|
||||||
[]string{"tables/outbound/single_ips.txt", "tables/outbound/networks.txt"},
|
|
||||||
)
|
|
||||||
case strings.HasSuffix(dataPath, ".txt") || strings.HasSuffix(dataPath, ".csv"):
|
|
||||||
src = newFileSources(nil, []string{dataPath}, nil)
|
|
||||||
default:
|
|
||||||
src = newHTTPSources(
|
|
||||||
nil,
|
|
||||||
[]HTTPSource{
|
|
||||||
{inboundSingleURL, dataPath + "/inbound_single_ips.txt"},
|
|
||||||
{inboundNetworkURL, dataPath + "/inbound_networks.txt"},
|
|
||||||
},
|
|
||||||
[]HTTPSource{
|
|
||||||
{outboundSingleURL, dataPath + "/outbound_single_ips.txt"},
|
|
||||||
{outboundNetworkURL, dataPath + "/outbound_networks.txt"},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
blGroup, whitelist, inbound, outbound := src.Datasets()
|
|
||||||
if err := blGroup.Init(); err != nil {
|
if err := blGroup.Init(); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
fmt.Fprintf(os.Stderr, "error: blocklist: %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, "Loaded inbound=%d outbound=%d\n",
|
fmt.Fprintf(os.Stderr, "Loaded inbound=%d outbound=%d\n",
|
||||||
cohortSize(inbound), cohortSize(outbound))
|
cohortSize(inboundDS), cohortSize(outboundDS))
|
||||||
|
|
||||||
// -- GeoIP (optional) --------------------------------------------------
|
// -- GeoIP (optional) --------------------------------------------------
|
||||||
|
|
||||||
geo, err := setupGeo(*geoipConf, *cityDBPath, *asnDBPath)
|
geo, err := setupGeo(*geoipConf, *cityDB, *asnDB)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
@ -119,8 +82,8 @@ func main() {
|
|||||||
|
|
||||||
// -- Check and report --------------------------------------------------
|
// -- Check and report --------------------------------------------------
|
||||||
|
|
||||||
blockedIn := isBlocked(ipStr, whitelist, inbound)
|
blockedIn := isBlocked(ipStr, whitelistDS, inboundDS)
|
||||||
blockedOut := isBlocked(ipStr, whitelist, outbound)
|
blockedOut := isBlocked(ipStr, whitelistDS, outboundDS)
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case blockedIn && blockedOut:
|
case blockedIn && blockedOut:
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user