mirror of
https://github.com/therootcompany/golib.git
synced 2026-11-04 06:55:00 +00:00
refactor(net/formmailer): delegate GeoIP downloads
This commit is contained in:
parent
219365ffd8
commit
37c004cec4
@ -235,21 +235,10 @@ type FormMailer struct {
|
|||||||
|
|
||||||
// --- GeoIP configuration (used by Run) ---
|
// --- GeoIP configuration (used by Run) ---
|
||||||
|
|
||||||
// GeoIPConfPath is the path to a GeoIP.conf file. Zero searches
|
// GeoIPDir is the directory holding the GeoLite2 tarballs. An external
|
||||||
// ./GeoIP.conf and ~/.config/maxmind/GeoIP.conf. Its presence selects
|
// geoip-update command owns downloads. Zero uses ~/.cache/maxmind.
|
||||||
// download mode (formmailer owns the geoip-update loop); its absence
|
// Ignored when using programmatic setup.
|
||||||
// falls back to polling pre-existing archives.
|
|
||||||
GeoIPConfPath string
|
|
||||||
// GeoIPDir is the directory holding the GeoLite2 tarballs. Zero uses
|
|
||||||
// ~/.cache/maxmind. Ignored when using programmatic setup.
|
|
||||||
GeoIPDir string
|
GeoIPDir string
|
||||||
// GeoIPBaseURL overrides the MaxMind download base. Zero uses the
|
|
||||||
// geoip.DownloadBase default. Library-level; not exposed on the CLI.
|
|
||||||
GeoIPBaseURL string
|
|
||||||
// GeoIPFreshDays gates downloads: skip HTTP when a local archive is
|
|
||||||
// younger than N days. Zero uses 3. Maps to httpcache.Cacher.MaxAge;
|
|
||||||
// a library-level option, rare to expose all the way up to the CLI.
|
|
||||||
GeoIPFreshDays int
|
|
||||||
// AllowedCountries — if non-nil, only requests from listed ISO codes are
|
// AllowedCountries — if non-nil, only requests from listed ISO codes are
|
||||||
// accepted. Unknown country ("") is always allowed. Requires GeoIP to be
|
// accepted. Unknown country ("") is always allowed. Requires GeoIP to be
|
||||||
// loaded.
|
// loaded.
|
||||||
@ -343,9 +332,8 @@ func (fm *FormMailer) contentType() string {
|
|||||||
// Data sources:
|
// Data sources:
|
||||||
// - Blocklist: gitshallow clones the configured repo, ipcohort loads the
|
// - Blocklist: gitshallow clones the configured repo, ipcohort loads the
|
||||||
// inbound cohort files, dataset manages hot-swap.
|
// inbound cohort files, dataset manages hot-swap.
|
||||||
// - GeoIP: geoip.Fetcher pulls the GeoLite2 City + ASN tarballs via
|
// - GeoIP: an external geoip-update job owns downloads; formmailer polls
|
||||||
// httpcache on each refresh tick; geoip.Open extracts them in memory
|
// local City + ASN tarballs and geoip.Open extracts them in memory.
|
||||||
// and dataset manages hot-swap. formmailer owns its own downloads.
|
|
||||||
//
|
//
|
||||||
// Background refreshes run at RefreshInterval (default 47min). The server
|
// Background refreshes run at RefreshInterval (default 47min). The server
|
||||||
// starts serving immediately; /healthz returns 503 until both blocklist and
|
// starts serving immediately; /healthz returns 503 until both blocklist and
|
||||||
@ -401,16 +389,13 @@ func (fm *FormMailer) Run(ctx context.Context) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// GeoIP directory. Archives live here; formmailer either downloads them
|
// GeoIP archives are downloaded by the external geoip-update command.
|
||||||
// (when a GeoIP.conf is present) or polls pre-existing ones (an external
|
// Formmailer only polls and reloads the local tarballs.
|
||||||
// geoip-update job owns the downloads).
|
|
||||||
geoDir := filepath.Join(fm.CacheDir, "maxmind")
|
geoDir := filepath.Join(fm.CacheDir, "maxmind")
|
||||||
if fm.GeoIPDir != "" {
|
if fm.GeoIPDir != "" {
|
||||||
geoDir = fm.GeoIPDir
|
geoDir = fm.GeoIPDir
|
||||||
}
|
}
|
||||||
|
|
||||||
// dirPopulated reports whether dir holds any entries. Used to pick the
|
|
||||||
// poll mode when no GeoIP.conf is present.
|
|
||||||
dirPopulated := func(dir string) bool {
|
dirPopulated := func(dir string) bool {
|
||||||
entries, err := os.ReadDir(dir)
|
entries, err := os.ReadDir(dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -419,51 +404,10 @@ func (fm *FormMailer) Run(ctx context.Context) error {
|
|||||||
return len(entries) > 0
|
return len(entries) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decide GeoIP mode from what's on disk. A present GeoIP.conf selects
|
|
||||||
// download mode (formmailer owns the geoip-update loop via
|
|
||||||
// geoip.Fetcher); otherwise, if the archive dir is already populated,
|
|
||||||
// poll it (external job owns downloads). Either way the view loader
|
|
||||||
// extracts the archives in memory. The refresh interval is the
|
|
||||||
// dataset layer's job (Set.Tick); freshness gating is httpcache's
|
|
||||||
// (Cacher.MaxAge == fresh-days).
|
|
||||||
confExists := fm.GeoIPConfPath != ""
|
|
||||||
if confExists {
|
|
||||||
if _, err := os.Stat(fm.GeoIPConfPath); err != nil {
|
|
||||||
confExists = false
|
|
||||||
log.Printf("geoip: configured conf %s not found; falling back to poll mode", fm.GeoIPConfPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !confExists {
|
|
||||||
for _, p := range geoip.DefaultConfPaths() {
|
|
||||||
if _, err := os.Stat(p); err == nil {
|
|
||||||
fm.GeoIPConfPath = p
|
|
||||||
confExists = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var geoSet *dataset.Set
|
var geoSet *dataset.Set
|
||||||
var geoView *dataset.View[geoip.Databases]
|
var geoView *dataset.View[geoip.Databases]
|
||||||
geoMode := "disabled"
|
geoMode := "disabled"
|
||||||
switch {
|
if dirPopulated(geoDir) {
|
||||||
case confExists:
|
|
||||||
// Download mode: formmailer owns the geoip-update loop.
|
|
||||||
geoMode = "download"
|
|
||||||
geoFetcher, ferr := geoip.NewFetcher(fm.GeoIPConfPath)
|
|
||||||
if ferr != nil {
|
|
||||||
log.Printf("geoip: %v", ferr)
|
|
||||||
} else {
|
|
||||||
geoFetcher.Dir = geoDir
|
|
||||||
geoFetcher.BaseURL = fm.GeoIPBaseURL
|
|
||||||
geoFetcher.FreshDays = fm.GeoIPFreshDays
|
|
||||||
geoSet = dataset.NewSet(geoFetcher)
|
|
||||||
geoView = dataset.Add(geoSet, func(ctx context.Context) (*geoip.Databases, error) {
|
|
||||||
return geoip.Open(geoDir)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
case dirPopulated(geoDir):
|
|
||||||
// Poll mode: an external job owns the downloads; we just reload.
|
|
||||||
geoMode = "poll"
|
geoMode = "poll"
|
||||||
cityPath := filepath.Join(geoDir, geoip.TarGzName(geoip.CityEdition))
|
cityPath := filepath.Join(geoDir, geoip.TarGzName(geoip.CityEdition))
|
||||||
asnPath := filepath.Join(geoDir, geoip.TarGzName(geoip.ASNEdition))
|
asnPath := filepath.Join(geoDir, geoip.TarGzName(geoip.ASNEdition))
|
||||||
@ -575,14 +519,9 @@ func (fm *FormMailer) Run(ctx context.Context) error {
|
|||||||
if fm.BlocklistRepo != "" {
|
if fm.BlocklistRepo != "" {
|
||||||
fmt.Println("Blocklist: enabled")
|
fmt.Println("Blocklist: enabled")
|
||||||
}
|
}
|
||||||
if geoSet != nil {
|
if geoSet != nil && geoMode == "poll" {
|
||||||
switch geoMode {
|
|
||||||
case "download":
|
|
||||||
fmt.Printf("GeoIP: download mode (conf=%s, dir=%s)\n", fm.GeoIPConfPath, geoDir)
|
|
||||||
case "poll":
|
|
||||||
fmt.Printf("GeoIP: poll mode (dir=%s)\n", geoDir)
|
fmt.Printf("GeoIP: poll mode (dir=%s)\n", geoDir)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if len(fm.AllowedCountries) > 0 {
|
if len(fm.AllowedCountries) > 0 {
|
||||||
fmt.Printf("Country gate: %s\n", strings.Join(fm.AllowedCountries, ", "))
|
fmt.Printf("Country gate: %s\n", strings.Join(fm.AllowedCountries, ", "))
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,6 +10,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
|
|||||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
|
github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
|
||||||
github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
|
github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
|
||||||
|
github.com/therootcompany/golib/net/geoip v0.5.3 h1:wps0TrRoGYw7XA5mWP0uOkGBF5oZ5rQ+1JPIAkRYrOU=
|
||||||
|
github.com/therootcompany/golib/net/geoip v0.5.3/go.mod h1:3CuuG2igsZySLNrRxoRTHPw3fmw9c4b0jRajSOOKJNU=
|
||||||
github.com/therootcompany/golib/net/gitshallow v0.9.3 h1:FKi0IvpqEDnYasjTmkRmXDdi+y3bhQBcjgvggwAwxFs=
|
github.com/therootcompany/golib/net/gitshallow v0.9.3 h1:FKi0IvpqEDnYasjTmkRmXDdi+y3bhQBcjgvggwAwxFs=
|
||||||
github.com/therootcompany/golib/net/gitshallow v0.9.3/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
|
github.com/therootcompany/golib/net/gitshallow v0.9.3/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
|
||||||
github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
|
github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
|
||||||
|
|||||||
@ -3,8 +3,6 @@ package formmailer
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@ -47,41 +45,10 @@ func runOnce(t *testing.T, fm *FormMailer) string {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunGeoModeSelection sanity-checks the hybrid GeoIP mode selection:
|
// TestRunGeoModeSelection checks that formmailer only polls existing
|
||||||
// - conf present -> download mode
|
// archives; downloading is owned by the external geoip-update command.
|
||||||
// - conf absent + populated -> poll mode
|
|
||||||
// - conf absent + empty dir -> disabled
|
|
||||||
func TestRunGeoModeSelection(t *testing.T) {
|
func TestRunGeoModeSelection(t *testing.T) {
|
||||||
t.Run("download", func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
conf := filepath.Join(dir, "GeoIP.conf")
|
|
||||||
if err := os.WriteFile(conf, []byte(
|
|
||||||
"AccountID 123456\nLicenseKey testkey\nEditionIDs GeoLite2-City GeoLite2-ASN\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// 500 server so the owned download loop fails fast; the mode is
|
|
||||||
// selected and logged before serving regardless.
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
http.Error(w, "nope", http.StatusInternalServerError)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
out := runOnce(t, &FormMailer{
|
|
||||||
CacheDir: dir,
|
|
||||||
GeoIPConfPath: conf,
|
|
||||||
GeoIPBaseURL: srv.URL,
|
|
||||||
SMTPHost: "localhost:25",
|
|
||||||
SMTPFrom: "a@example.com",
|
|
||||||
SMTPTo: []string{"b@example.com"},
|
|
||||||
})
|
|
||||||
if !strings.Contains(out, "GeoIP: download mode") {
|
|
||||||
t.Fatalf("missing download mode line; stdout:\n%s", out)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("poll", func(t *testing.T) {
|
t.Run("poll", func(t *testing.T) {
|
||||||
// Point HOME at a temp dir so the default conf discovery can't find a
|
|
||||||
// real ~/.config/maxmind/GeoIP.conf.
|
|
||||||
home := t.TempDir()
|
home := t.TempDir()
|
||||||
t.Setenv("HOME", home)
|
t.Setenv("HOME", home)
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user