mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 12:48:00 +00:00
feat: default cache dirs; test both inbound files
- geoip.DefaultCacheDir() → ~/.cache/maxmind (os.UserCacheDir based) - check-ip defaults data dir to ~/.cache/bitwire-it; -data-dir flag overrides; positional data-dir arg removed (IP is now the only required arg) - geoip conf: DatabaseDirectory defaults to geoip.DefaultCacheDir() when blank - httpcache integration tests now cover both inbound files (single_ips + networks)
This commit is contained in:
parent
d24a34e0e5
commit
bd62122ac8
@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -33,6 +34,16 @@ type Downloader struct {
|
||||
Timeout time.Duration // 0 uses 5m
|
||||
}
|
||||
|
||||
// DefaultCacheDir returns the OS cache directory for MaxMind databases,
|
||||
// e.g. ~/.cache/maxmind on Linux or ~/Library/Caches/maxmind on macOS.
|
||||
func DefaultCacheDir() (string, error) {
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(base, "maxmind"), nil
|
||||
}
|
||||
|
||||
// New returns a Downloader configured with the given credentials.
|
||||
func New(accountID, licenseKey string) *Downloader {
|
||||
return &Downloader{AccountID: accountID, LicenseKey: licenseKey}
|
||||
|
||||
@ -11,10 +11,19 @@ import (
|
||||
"github.com/therootcompany/golib/net/httpcache"
|
||||
)
|
||||
|
||||
const (
|
||||
testURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
|
||||
testFile = "httpcache_inbound.txt"
|
||||
)
|
||||
var inboundSources = []struct {
|
||||
name string
|
||||
url string
|
||||
}{
|
||||
{
|
||||
"inbound_single_ips",
|
||||
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt",
|
||||
},
|
||||
{
|
||||
"inbound_networks",
|
||||
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt",
|
||||
},
|
||||
}
|
||||
|
||||
func testdataDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
@ -32,93 +41,105 @@ func testdataDir(t *testing.T) string {
|
||||
}
|
||||
|
||||
func TestCacher_Download(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), testFile)
|
||||
os.Remove(path)
|
||||
os.Remove(path + ".meta")
|
||||
for _, src := range inboundSources {
|
||||
t.Run(src.name, func(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||
os.Remove(path)
|
||||
os.Remove(path + ".meta")
|
||||
|
||||
c := httpcache.New(testURL, path)
|
||||
|
||||
updated, err := c.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("first Fetch: %v", err)
|
||||
c := httpcache.New(src.url, path)
|
||||
updated, err := c.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if !updated {
|
||||
t.Error("first Fetch: expected updated=true")
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("file not created: %v", err)
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
t.Error("downloaded file is empty")
|
||||
}
|
||||
t.Logf("downloaded %d bytes to %s", info.Size(), path)
|
||||
})
|
||||
}
|
||||
if !updated {
|
||||
t.Error("first Fetch: expected updated=true")
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("file not created: %v", err)
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
t.Error("downloaded file is empty")
|
||||
}
|
||||
t.Logf("downloaded %d bytes to %s", info.Size(), path)
|
||||
}
|
||||
|
||||
func TestCacher_SidecarWritten(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), testFile)
|
||||
os.Remove(path)
|
||||
os.Remove(path + ".meta")
|
||||
for _, src := range inboundSources {
|
||||
t.Run(src.name, func(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||
os.Remove(path)
|
||||
os.Remove(path + ".meta")
|
||||
|
||||
c := httpcache.New(testURL, path)
|
||||
if _, err := c.Fetch(); err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
c := httpcache.New(src.url, path)
|
||||
if _, err := c.Fetch(); err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path + ".meta")
|
||||
if err != nil {
|
||||
t.Fatalf("sidecar not written: %v", err)
|
||||
data, err := os.ReadFile(path + ".meta")
|
||||
if err != nil {
|
||||
t.Fatalf("sidecar not written: %v", err)
|
||||
}
|
||||
var meta map[string]string
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
t.Fatalf("sidecar not valid JSON: %v", err)
|
||||
}
|
||||
if meta["etag"] == "" && meta["last_modified"] == "" {
|
||||
t.Error("sidecar has neither etag nor last_modified")
|
||||
}
|
||||
t.Logf("sidecar: %s", data)
|
||||
})
|
||||
}
|
||||
var meta map[string]string
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
t.Fatalf("sidecar not valid JSON: %v", err)
|
||||
}
|
||||
if meta["etag"] == "" && meta["last_modified"] == "" {
|
||||
t.Error("sidecar has neither etag nor last_modified")
|
||||
}
|
||||
t.Logf("sidecar: %s", data)
|
||||
}
|
||||
|
||||
func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), testFile)
|
||||
for _, src := range inboundSources {
|
||||
t.Run(src.name, func(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||
|
||||
c := httpcache.New(testURL, path)
|
||||
if _, err := c.Fetch(); err != nil {
|
||||
t.Fatalf("initial Fetch: %v", err)
|
||||
}
|
||||
c := httpcache.New(src.url, path)
|
||||
if _, err := c.Fetch(); err != nil {
|
||||
t.Fatalf("initial Fetch: %v", err)
|
||||
}
|
||||
|
||||
// Second call on the same instance — ETag already in memory.
|
||||
updated, err := c.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("second Fetch: %v", err)
|
||||
updated, err := c.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("second Fetch: %v", err)
|
||||
}
|
||||
if updated {
|
||||
t.Error("second Fetch on same cacher: expected updated=false")
|
||||
}
|
||||
t.Log("same-cacher conditional GET correctly skipped re-download")
|
||||
})
|
||||
}
|
||||
if updated {
|
||||
t.Error("same-cacher second Fetch: expected updated=false")
|
||||
}
|
||||
t.Log("same-cacher conditional GET correctly skipped re-download")
|
||||
}
|
||||
|
||||
func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), testFile)
|
||||
for _, src := range inboundSources {
|
||||
t.Run(src.name, func(t *testing.T) {
|
||||
path := filepath.Join(testdataDir(t), src.name+".txt")
|
||||
|
||||
// Ensure file + sidecar exist.
|
||||
first := httpcache.New(testURL, path)
|
||||
if _, err := first.Fetch(); err != nil {
|
||||
t.Fatalf("initial Fetch: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path + ".meta"); err != nil {
|
||||
t.Fatalf("sidecar missing after first fetch: %v", err)
|
||||
}
|
||||
first := httpcache.New(src.url, path)
|
||||
if _, err := first.Fetch(); err != nil {
|
||||
t.Fatalf("initial Fetch: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path + ".meta"); err != nil {
|
||||
t.Fatalf("sidecar missing after first fetch: %v", err)
|
||||
}
|
||||
|
||||
// New Cacher with no in-memory state — must read sidecar and send conditional GET.
|
||||
fresh := httpcache.New(testURL, path)
|
||||
updated, err := fresh.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("fresh-cacher Fetch: %v", err)
|
||||
// New Cacher with no in-memory state — must read sidecar.
|
||||
fresh := httpcache.New(src.url, path)
|
||||
updated, err := fresh.Fetch()
|
||||
if err != nil {
|
||||
t.Fatalf("fresh-cacher Fetch: %v", err)
|
||||
}
|
||||
if updated {
|
||||
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
|
||||
}
|
||||
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
|
||||
})
|
||||
}
|
||||
if updated {
|
||||
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
|
||||
}
|
||||
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
|
||||
}
|
||||
|
||||
@ -29,26 +29,48 @@ const (
|
||||
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
|
||||
)
|
||||
|
||||
func defaultBlocklistDir() string {
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return filepath.Join(os.Getenv("HOME"), ".cache", "bitwire-it")
|
||||
}
|
||||
return filepath.Join(base, "bitwire-it")
|
||||
}
|
||||
|
||||
func main() {
|
||||
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
|
||||
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)")
|
||||
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)")
|
||||
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir")
|
||||
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir")
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <data-dir|blacklist.txt> <ip-address>\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, " data-dir: fetch blocklists via HTTP (or git with -git)\n")
|
||||
fmt.Fprintf(os.Stderr, " blacklist.txt: load single local file as inbound list\n")
|
||||
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, " Blocklists are fetched via HTTP by default (use -git for git source).\n")
|
||||
fmt.Fprintf(os.Stderr, " Pass a .txt/.csv path as the first arg to load a single local file.\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
if flag.NArg() < 2 {
|
||||
if flag.NArg() < 1 {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
dataPath := flag.Arg(0)
|
||||
ipStr := flag.Arg(1)
|
||||
// First arg is either a local file or the IP to check.
|
||||
// If it looks like a file, treat it as the inbound list; otherwise use default cache dir.
|
||||
var dataPath, ipStr string
|
||||
if flag.NArg() >= 2 || strings.HasSuffix(flag.Arg(0), ".txt") || strings.HasSuffix(flag.Arg(0), ".csv") {
|
||||
dataPath = flag.Arg(0)
|
||||
ipStr = flag.Arg(1)
|
||||
} else {
|
||||
ipStr = flag.Arg(0)
|
||||
}
|
||||
if *dataDir != "" {
|
||||
dataPath = *dataDir
|
||||
}
|
||||
if dataPath == "" {
|
||||
dataPath = defaultBlocklistDir()
|
||||
}
|
||||
|
||||
// Blocklist source.
|
||||
var src *Sources
|
||||
@ -98,7 +120,9 @@ func main() {
|
||||
} else {
|
||||
dbDir := cfg.DatabaseDirectory
|
||||
if dbDir == "" {
|
||||
dbDir = dataPath
|
||||
if d, err := geoip.DefaultCacheDir(); err == nil {
|
||||
dbDir = d
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(dbDir, 0o755); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warn: mkdir %s: %v\n", dbDir, err)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user