mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 20:58:00 +00:00
feat: default cache dirs; test both inbound files
- geoip.DefaultCacheDir() → ~/.cache/maxmind (os.UserCacheDir based) - check-ip defaults data dir to ~/.cache/bitwire-it; -data-dir flag overrides; positional data-dir arg removed (IP is now the only required arg) - geoip conf: DatabaseDirectory defaults to geoip.DefaultCacheDir() when blank - httpcache integration tests now cover both inbound files (single_ips + networks)
This commit is contained in:
parent
d24a34e0e5
commit
bd62122ac8
@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -33,6 +34,16 @@ type Downloader struct {
|
|||||||
Timeout time.Duration // 0 uses 5m
|
Timeout time.Duration // 0 uses 5m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultCacheDir returns the OS cache directory for MaxMind databases,
|
||||||
|
// e.g. ~/.cache/maxmind on Linux or ~/Library/Caches/maxmind on macOS.
|
||||||
|
func DefaultCacheDir() (string, error) {
|
||||||
|
base, err := os.UserCacheDir()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Join(base, "maxmind"), nil
|
||||||
|
}
|
||||||
|
|
||||||
// New returns a Downloader configured with the given credentials.
|
// New returns a Downloader configured with the given credentials.
|
||||||
func New(accountID, licenseKey string) *Downloader {
|
func New(accountID, licenseKey string) *Downloader {
|
||||||
return &Downloader{AccountID: accountID, LicenseKey: licenseKey}
|
return &Downloader{AccountID: accountID, LicenseKey: licenseKey}
|
||||||
|
|||||||
@ -11,10 +11,19 @@ import (
|
|||||||
"github.com/therootcompany/golib/net/httpcache"
|
"github.com/therootcompany/golib/net/httpcache"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
var inboundSources = []struct {
|
||||||
testURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
|
name string
|
||||||
testFile = "httpcache_inbound.txt"
|
url string
|
||||||
)
|
}{
|
||||||
|
{
|
||||||
|
"inbound_single_ips",
|
||||||
|
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"inbound_networks",
|
||||||
|
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
func testdataDir(t *testing.T) string {
|
func testdataDir(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@ -32,20 +41,20 @@ func testdataDir(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacher_Download(t *testing.T) {
|
func TestCacher_Download(t *testing.T) {
|
||||||
path := filepath.Join(testdataDir(t), testFile)
|
for _, src := range inboundSources {
|
||||||
|
t.Run(src.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||||
os.Remove(path)
|
os.Remove(path)
|
||||||
os.Remove(path + ".meta")
|
os.Remove(path + ".meta")
|
||||||
|
|
||||||
c := httpcache.New(testURL, path)
|
c := httpcache.New(src.url, path)
|
||||||
|
|
||||||
updated, err := c.Fetch()
|
updated, err := c.Fetch()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first Fetch: %v", err)
|
t.Fatalf("Fetch: %v", err)
|
||||||
}
|
}
|
||||||
if !updated {
|
if !updated {
|
||||||
t.Error("first Fetch: expected updated=true")
|
t.Error("first Fetch: expected updated=true")
|
||||||
}
|
}
|
||||||
|
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("file not created: %v", err)
|
t.Fatalf("file not created: %v", err)
|
||||||
@ -54,14 +63,18 @@ func TestCacher_Download(t *testing.T) {
|
|||||||
t.Error("downloaded file is empty")
|
t.Error("downloaded file is empty")
|
||||||
}
|
}
|
||||||
t.Logf("downloaded %d bytes to %s", info.Size(), path)
|
t.Logf("downloaded %d bytes to %s", info.Size(), path)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacher_SidecarWritten(t *testing.T) {
|
func TestCacher_SidecarWritten(t *testing.T) {
|
||||||
path := filepath.Join(testdataDir(t), testFile)
|
for _, src := range inboundSources {
|
||||||
|
t.Run(src.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||||
os.Remove(path)
|
os.Remove(path)
|
||||||
os.Remove(path + ".meta")
|
os.Remove(path + ".meta")
|
||||||
|
|
||||||
c := httpcache.New(testURL, path)
|
c := httpcache.New(src.url, path)
|
||||||
if _, err := c.Fetch(); err != nil {
|
if _, err := c.Fetch(); err != nil {
|
||||||
t.Fatalf("Fetch: %v", err)
|
t.Fatalf("Fetch: %v", err)
|
||||||
}
|
}
|
||||||
@ -78,32 +91,38 @@ func TestCacher_SidecarWritten(t *testing.T) {
|
|||||||
t.Error("sidecar has neither etag nor last_modified")
|
t.Error("sidecar has neither etag nor last_modified")
|
||||||
}
|
}
|
||||||
t.Logf("sidecar: %s", data)
|
t.Logf("sidecar: %s", data)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
|
func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
|
||||||
path := filepath.Join(testdataDir(t), testFile)
|
for _, src := range inboundSources {
|
||||||
|
t.Run(src.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||||
|
|
||||||
c := httpcache.New(testURL, path)
|
c := httpcache.New(src.url, path)
|
||||||
if _, err := c.Fetch(); err != nil {
|
if _, err := c.Fetch(); err != nil {
|
||||||
t.Fatalf("initial Fetch: %v", err)
|
t.Fatalf("initial Fetch: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Second call on the same instance — ETag already in memory.
|
|
||||||
updated, err := c.Fetch()
|
updated, err := c.Fetch()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("second Fetch: %v", err)
|
t.Fatalf("second Fetch: %v", err)
|
||||||
}
|
}
|
||||||
if updated {
|
if updated {
|
||||||
t.Error("same-cacher second Fetch: expected updated=false")
|
t.Error("second Fetch on same cacher: expected updated=false")
|
||||||
}
|
}
|
||||||
t.Log("same-cacher conditional GET correctly skipped re-download")
|
t.Log("same-cacher conditional GET correctly skipped re-download")
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
|
func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
|
||||||
path := filepath.Join(testdataDir(t), testFile)
|
for _, src := range inboundSources {
|
||||||
|
t.Run(src.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||||
|
|
||||||
// Ensure file + sidecar exist.
|
first := httpcache.New(src.url, path)
|
||||||
first := httpcache.New(testURL, path)
|
|
||||||
if _, err := first.Fetch(); err != nil {
|
if _, err := first.Fetch(); err != nil {
|
||||||
t.Fatalf("initial Fetch: %v", err)
|
t.Fatalf("initial Fetch: %v", err)
|
||||||
}
|
}
|
||||||
@ -111,8 +130,8 @@ func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
|
|||||||
t.Fatalf("sidecar missing after first fetch: %v", err)
|
t.Fatalf("sidecar missing after first fetch: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// New Cacher with no in-memory state — must read sidecar and send conditional GET.
|
// New Cacher with no in-memory state — must read sidecar.
|
||||||
fresh := httpcache.New(testURL, path)
|
fresh := httpcache.New(src.url, path)
|
||||||
updated, err := fresh.Fetch()
|
updated, err := fresh.Fetch()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("fresh-cacher Fetch: %v", err)
|
t.Fatalf("fresh-cacher Fetch: %v", err)
|
||||||
@ -121,4 +140,6 @@ func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
|
|||||||
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
|
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
|
||||||
}
|
}
|
||||||
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
|
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -29,26 +29,48 @@ const (
|
|||||||
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
|
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func defaultBlocklistDir() string {
|
||||||
|
base, err := os.UserCacheDir()
|
||||||
|
if err != nil {
|
||||||
|
return filepath.Join(os.Getenv("HOME"), ".cache", "bitwire-it")
|
||||||
|
}
|
||||||
|
return filepath.Join(base, "bitwire-it")
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
|
||||||
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)")
|
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)")
|
||||||
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)")
|
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)")
|
||||||
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir")
|
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir")
|
||||||
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir")
|
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir")
|
||||||
flag.Usage = func() {
|
flag.Usage = func() {
|
||||||
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <data-dir|blacklist.txt> <ip-address>\n", os.Args[0])
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
|
||||||
fmt.Fprintf(os.Stderr, " data-dir: fetch blocklists via HTTP (or git with -git)\n")
|
fmt.Fprintf(os.Stderr, " Blocklists are fetched via HTTP by default (use -git for git source).\n")
|
||||||
fmt.Fprintf(os.Stderr, " blacklist.txt: load single local file as inbound list\n")
|
fmt.Fprintf(os.Stderr, " Pass a .txt/.csv path as the first arg to load a single local file.\n")
|
||||||
flag.PrintDefaults()
|
flag.PrintDefaults()
|
||||||
}
|
}
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
if flag.NArg() < 2 {
|
if flag.NArg() < 1 {
|
||||||
flag.Usage()
|
flag.Usage()
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
dataPath := flag.Arg(0)
|
// First arg is either a local file or the IP to check.
|
||||||
ipStr := flag.Arg(1)
|
// If it looks like a file, treat it as the inbound list; otherwise use default cache dir.
|
||||||
|
var dataPath, ipStr string
|
||||||
|
if flag.NArg() >= 2 || strings.HasSuffix(flag.Arg(0), ".txt") || strings.HasSuffix(flag.Arg(0), ".csv") {
|
||||||
|
dataPath = flag.Arg(0)
|
||||||
|
ipStr = flag.Arg(1)
|
||||||
|
} else {
|
||||||
|
ipStr = flag.Arg(0)
|
||||||
|
}
|
||||||
|
if *dataDir != "" {
|
||||||
|
dataPath = *dataDir
|
||||||
|
}
|
||||||
|
if dataPath == "" {
|
||||||
|
dataPath = defaultBlocklistDir()
|
||||||
|
}
|
||||||
|
|
||||||
// Blocklist source.
|
// Blocklist source.
|
||||||
var src *Sources
|
var src *Sources
|
||||||
@ -98,7 +120,9 @@ func main() {
|
|||||||
} else {
|
} else {
|
||||||
dbDir := cfg.DatabaseDirectory
|
dbDir := cfg.DatabaseDirectory
|
||||||
if dbDir == "" {
|
if dbDir == "" {
|
||||||
dbDir = dataPath
|
if d, err := geoip.DefaultCacheDir(); err == nil {
|
||||||
|
dbDir = d
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(dbDir, 0o755); err != nil {
|
if err := os.MkdirAll(dbDir, 0o755); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warn: mkdir %s: %v\n", dbDir, err)
|
fmt.Fprintf(os.Stderr, "warn: mkdir %s: %v\n", dbDir, err)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user