feat: default cache dirs; test both inbound files

- geoip.DefaultCacheDir() → ~/.cache/maxmind (os.UserCacheDir based)
- check-ip defaults data dir to ~/.cache/bitwire-it; -data-dir flag overrides;
  positional data-dir arg removed (IP is now the only required arg)
- geoip conf: DatabaseDirectory defaults to geoip.DefaultCacheDir() when blank
- httpcache integration tests now cover both inbound files (single_ips + networks)
This commit is contained in:
AJ ONeal 2026-04-20 10:11:49 -06:00
parent d24a34e0e5
commit bd62122ac8
No known key found for this signature in database
3 changed files with 136 additions and 80 deletions

View File

@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@ -33,6 +34,16 @@ type Downloader struct {
Timeout time.Duration // 0 uses 5m Timeout time.Duration // 0 uses 5m
} }
// DefaultCacheDir returns the OS cache directory for MaxMind databases,
// e.g. ~/.cache/maxmind on Linux or ~/Library/Caches/maxmind on macOS.
func DefaultCacheDir() (string, error) {
base, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(base, "maxmind"), nil
}
// New returns a Downloader configured with the given credentials. // New returns a Downloader configured with the given credentials.
func New(accountID, licenseKey string) *Downloader { func New(accountID, licenseKey string) *Downloader {
return &Downloader{AccountID: accountID, LicenseKey: licenseKey} return &Downloader{AccountID: accountID, LicenseKey: licenseKey}

View File

@ -11,10 +11,19 @@ import (
"github.com/therootcompany/golib/net/httpcache" "github.com/therootcompany/golib/net/httpcache"
) )
const ( var inboundSources = []struct {
testURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt" name string
testFile = "httpcache_inbound.txt" url string
) }{
{
"inbound_single_ips",
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt",
},
{
"inbound_networks",
"https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt",
},
}
func testdataDir(t *testing.T) string { func testdataDir(t *testing.T) string {
t.Helper() t.Helper()
@ -32,93 +41,105 @@ func testdataDir(t *testing.T) string {
} }
func TestCacher_Download(t *testing.T) { func TestCacher_Download(t *testing.T) {
path := filepath.Join(testdataDir(t), testFile) for _, src := range inboundSources {
os.Remove(path) t.Run(src.name, func(t *testing.T) {
os.Remove(path + ".meta") path := filepath.Join(testdataDir(t), src.name+".txt")
os.Remove(path)
os.Remove(path + ".meta")
c := httpcache.New(testURL, path) c := httpcache.New(src.url, path)
updated, err := c.Fetch()
updated, err := c.Fetch() if err != nil {
if err != nil { t.Fatalf("Fetch: %v", err)
t.Fatalf("first Fetch: %v", err) }
if !updated {
t.Error("first Fetch: expected updated=true")
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("file not created: %v", err)
}
if info.Size() == 0 {
t.Error("downloaded file is empty")
}
t.Logf("downloaded %d bytes to %s", info.Size(), path)
})
} }
if !updated {
t.Error("first Fetch: expected updated=true")
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("file not created: %v", err)
}
if info.Size() == 0 {
t.Error("downloaded file is empty")
}
t.Logf("downloaded %d bytes to %s", info.Size(), path)
} }
func TestCacher_SidecarWritten(t *testing.T) { func TestCacher_SidecarWritten(t *testing.T) {
path := filepath.Join(testdataDir(t), testFile) for _, src := range inboundSources {
os.Remove(path) t.Run(src.name, func(t *testing.T) {
os.Remove(path + ".meta") path := filepath.Join(testdataDir(t), src.name+".txt")
os.Remove(path)
os.Remove(path + ".meta")
c := httpcache.New(testURL, path) c := httpcache.New(src.url, path)
if _, err := c.Fetch(); err != nil { if _, err := c.Fetch(); err != nil {
t.Fatalf("Fetch: %v", err) t.Fatalf("Fetch: %v", err)
} }
data, err := os.ReadFile(path + ".meta") data, err := os.ReadFile(path + ".meta")
if err != nil { if err != nil {
t.Fatalf("sidecar not written: %v", err) t.Fatalf("sidecar not written: %v", err)
}
var meta map[string]string
if err := json.Unmarshal(data, &meta); err != nil {
t.Fatalf("sidecar not valid JSON: %v", err)
}
if meta["etag"] == "" && meta["last_modified"] == "" {
t.Error("sidecar has neither etag nor last_modified")
}
t.Logf("sidecar: %s", data)
})
} }
var meta map[string]string
if err := json.Unmarshal(data, &meta); err != nil {
t.Fatalf("sidecar not valid JSON: %v", err)
}
if meta["etag"] == "" && meta["last_modified"] == "" {
t.Error("sidecar has neither etag nor last_modified")
}
t.Logf("sidecar: %s", data)
} }
func TestCacher_ConditionalGet_SameCacher(t *testing.T) { func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
path := filepath.Join(testdataDir(t), testFile) for _, src := range inboundSources {
t.Run(src.name, func(t *testing.T) {
path := filepath.Join(testdataDir(t), src.name+".txt")
c := httpcache.New(testURL, path) c := httpcache.New(src.url, path)
if _, err := c.Fetch(); err != nil { if _, err := c.Fetch(); err != nil {
t.Fatalf("initial Fetch: %v", err) t.Fatalf("initial Fetch: %v", err)
} }
// Second call on the same instance — ETag already in memory. updated, err := c.Fetch()
updated, err := c.Fetch() if err != nil {
if err != nil { t.Fatalf("second Fetch: %v", err)
t.Fatalf("second Fetch: %v", err) }
if updated {
t.Error("second Fetch on same cacher: expected updated=false")
}
t.Log("same-cacher conditional GET correctly skipped re-download")
})
} }
if updated {
t.Error("same-cacher second Fetch: expected updated=false")
}
t.Log("same-cacher conditional GET correctly skipped re-download")
} }
func TestCacher_ConditionalGet_FreshCacher(t *testing.T) { func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
path := filepath.Join(testdataDir(t), testFile) for _, src := range inboundSources {
t.Run(src.name, func(t *testing.T) {
path := filepath.Join(testdataDir(t), src.name+".txt")
// Ensure file + sidecar exist. first := httpcache.New(src.url, path)
first := httpcache.New(testURL, path) if _, err := first.Fetch(); err != nil {
if _, err := first.Fetch(); err != nil { t.Fatalf("initial Fetch: %v", err)
t.Fatalf("initial Fetch: %v", err) }
} if _, err := os.Stat(path + ".meta"); err != nil {
if _, err := os.Stat(path + ".meta"); err != nil { t.Fatalf("sidecar missing after first fetch: %v", err)
t.Fatalf("sidecar missing after first fetch: %v", err) }
}
// New Cacher with no in-memory state — must read sidecar and send conditional GET. // New Cacher with no in-memory state — must read sidecar.
fresh := httpcache.New(testURL, path) fresh := httpcache.New(src.url, path)
updated, err := fresh.Fetch() updated, err := fresh.Fetch()
if err != nil { if err != nil {
t.Fatalf("fresh-cacher Fetch: %v", err) t.Fatalf("fresh-cacher Fetch: %v", err)
}
if updated {
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
}
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
})
} }
if updated {
t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)")
}
t.Log("fresh-cacher conditional GET correctly used sidecar ETag")
} }

View File

@ -29,26 +29,48 @@ const (
outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt" outboundNetworkURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/outbound/networks.txt"
) )
func defaultBlocklistDir() string {
base, err := os.UserCacheDir()
if err != nil {
return filepath.Join(os.Getenv("HOME"), ".cache", "bitwire-it")
}
return filepath.Join(base, "bitwire-it")
}
func main() { func main() {
dataDir := flag.String("data-dir", "", "blocklist cache dir (default ~/.cache/bitwire-it)")
cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)") cityDBPath := flag.String("city-db", "", "path to GeoLite2-City.mmdb (overrides -geoip-conf)")
asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)") asnDBPath := flag.String("asn-db", "", "path to GeoLite2-ASN.mmdb (overrides -geoip-conf)")
geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir") geoipConf := flag.String("geoip-conf", "", "path to GeoIP.conf; auto-downloads City+ASN into data-dir")
gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir") gitURL := flag.String("git", "", "clone/pull blocklist from this git URL into data-dir")
flag.Usage = func() { flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <data-dir|blacklist.txt> <ip-address>\n", os.Args[0]) fmt.Fprintf(os.Stderr, "Usage: %s [flags] <ip-address>\n", os.Args[0])
fmt.Fprintf(os.Stderr, " data-dir: fetch blocklists via HTTP (or git with -git)\n") fmt.Fprintf(os.Stderr, " Blocklists are fetched via HTTP by default (use -git for git source).\n")
fmt.Fprintf(os.Stderr, " blacklist.txt: load single local file as inbound list\n") fmt.Fprintf(os.Stderr, " Pass a .txt/.csv path as the first arg to load a single local file.\n")
flag.PrintDefaults() flag.PrintDefaults()
} }
flag.Parse() flag.Parse()
if flag.NArg() < 2 { if flag.NArg() < 1 {
flag.Usage() flag.Usage()
os.Exit(1) os.Exit(1)
} }
dataPath := flag.Arg(0) // First arg is either a local file or the IP to check.
ipStr := flag.Arg(1) // If it looks like a file, treat it as the inbound list; otherwise use default cache dir.
var dataPath, ipStr string
if flag.NArg() >= 2 || strings.HasSuffix(flag.Arg(0), ".txt") || strings.HasSuffix(flag.Arg(0), ".csv") {
dataPath = flag.Arg(0)
ipStr = flag.Arg(1)
} else {
ipStr = flag.Arg(0)
}
if *dataDir != "" {
dataPath = *dataDir
}
if dataPath == "" {
dataPath = defaultBlocklistDir()
}
// Blocklist source. // Blocklist source.
var src *Sources var src *Sources
@ -98,7 +120,9 @@ func main() {
} else { } else {
dbDir := cfg.DatabaseDirectory dbDir := cfg.DatabaseDirectory
if dbDir == "" { if dbDir == "" {
dbDir = dataPath if d, err := geoip.DefaultCacheDir(); err == nil {
dbDir = d
}
} }
if err := os.MkdirAll(dbDir, 0o755); err != nil { if err := os.MkdirAll(dbDir, 0o755); err != nil {
fmt.Fprintf(os.Stderr, "warn: mkdir %s: %v\n", dbDir, err) fmt.Fprintf(os.Stderr, "warn: mkdir %s: %v\n", dbDir, err)