From 8648c5accb423eba4caf428a0463b90403456724 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 9 Sep 2026 22:38:50 -0600 Subject: [PATCH] refactor(cmd/check-ip): use released cache modules --- cmd/check-ip/go.mod | 11 +++--- cmd/check-ip/go.sum | 22 ++++++------ cmd/check-ip/main.go | 79 ++++++++++++++++++++---------------------- cmd/check-ip/server.go | 20 +++++------ 4 files changed, 62 insertions(+), 70 deletions(-) diff --git a/cmd/check-ip/go.mod b/cmd/check-ip/go.mod index fb0399a..c95bffd 100644 --- a/cmd/check-ip/go.mod +++ b/cmd/check-ip/go.mod @@ -3,19 +3,18 @@ module github.com/therootcompany/golib/cmd/check-ip go 1.26.5 require ( - github.com/therootcompany/golib/net/geoip v0.5.0 - github.com/therootcompany/golib/net/gitshallow v0.9.1 - github.com/therootcompany/golib/net/httpcache v0.5.1 + github.com/therootcompany/golib/net/geoip v0.6.0 + github.com/therootcompany/golib/net/gitshallow v0.10.0 + github.com/therootcompany/golib/net/httpcache v0.6.0 github.com/therootcompany/golib/net/ipcohort v0.9.0 - github.com/therootcompany/golib/net/iplist v0.8.5 - github.com/therootcompany/golib/sync/dataset v0.5.0 + github.com/therootcompany/golib/sync/dataset v0.6.0 ) require ( github.com/oschwald/geoip2-golang v1.13.0 // indirect github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/therootcompany/golib/https v0.9.0 // indirect - github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect + github.com/therootcompany/golib/sync/cachable v0.1.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect ) diff --git a/cmd/check-ip/go.sum b/cmd/check-ip/go.sum index 22ce811..7f4a8d9 100644 --- a/cmd/check-ip/go.sum +++ b/cmd/check-ip/go.sum @@ -10,20 +10,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= -github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo= -github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o= -github.com/therootcompany/golib/net/geoip v0.5.0 h1:dmduwHLX3RCe2NExhOKcFZ4m1ML2C3gQKnNyaBhNE88= -github.com/therootcompany/golib/net/geoip v0.5.0/go.mod h1:BLf5g25UhvI/+/x/WpM1NTVs4ZMnjySoHv+LXfALT6M= -github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q= -github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc= -github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= -github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= +github.com/therootcompany/golib/net/geoip v0.6.0 h1:+LyOKt47Mbxoi8ssaX9RaN0loW6wyPDA5XvExSiRzB0= +github.com/therootcompany/golib/net/geoip v0.6.0/go.mod h1:yrk7CuIet0BntICfRwRSX44fwu/1YAuHHjTQYoWHSOE= +github.com/therootcompany/golib/net/gitshallow v0.10.0 h1:cLN9ElTmNWk5HSbaRD3sSFuGdpgy4YXc+LVzjzT0nM4= +github.com/therootcompany/golib/net/gitshallow v0.10.0/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc= +github.com/therootcompany/golib/net/httpcache v0.6.0 h1:qFBnSs6rZrtUQ+fweP4bHF2FQd/Syvs7mRN5nO5YtkY= +github.com/therootcompany/golib/net/httpcache v0.6.0/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E= github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys= -github.com/therootcompany/golib/net/iplist v0.8.5 h1:swn/hcCt6fAG3tYApmxh1OlDak2PjQdczldZWaH1Eus= -github.com/therootcompany/golib/net/iplist v0.8.5/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU= -github.com/therootcompany/golib/sync/dataset v0.5.0 h1:4i3+7Izy/vnpK6IW5m9u4UqLA9RM1IjUsedbZo3MW6E= -github.com/therootcompany/golib/sync/dataset v0.5.0/go.mod h1:HMa25n3eqYnjRcPXny0IUGU2wFKD5/x2g9UUkt3iAmI= +github.com/therootcompany/golib/sync/cachable v0.1.0 h1:swCXtXB116jhSmla81HaXi/9cPsqFI+Oq/6OcSfdU5Q= +github.com/therootcompany/golib/sync/cachable v0.1.0/go.mod h1:l/NZdGiSbW5CkLlkkRa0yObkqak2sj/jQTXXqXwvCl0= +github.com/therootcompany/golib/sync/dataset v0.6.0 h1:Gerq315Ld1PyfD+6EmTDxSEAOg6s0kDchL2oo/jAQZ0= +github.com/therootcompany/golib/sync/dataset v0.6.0/go.mod h1:ktn0ppeIpNvgNLFYFuD9W6hGEQfwaIEcE8TV048+Y9M= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= diff --git a/cmd/check-ip/main.go b/cmd/check-ip/main.go index f17bf0e..0470584 100644 --- a/cmd/check-ip/main.go +++ b/cmd/check-ip/main.go @@ -19,7 +19,6 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" "syscall" "time" @@ -27,7 +26,6 @@ import ( "github.com/therootcompany/golib/net/gitshallow" "github.com/therootcompany/golib/net/httpcache" "github.com/therootcompany/golib/net/ipcohort" - "github.com/therootcompany/golib/net/iplist" "github.com/therootcompany/golib/sync/dataset" ) @@ -117,7 +115,7 @@ type IPCheck struct { inbound *dataset.View[ipcohort.Cohort] outbound *dataset.View[ipcohort.Cohort] - whitelist atomic.Pointer[ipcohort.Cohort] + whitelist *dataset.View[ipcohort.Cohort] geo *dataset.View[geoip.Databases] } @@ -233,7 +231,7 @@ func main() { loadBlocklists := func() { fmt.Fprint(os.Stderr, "Loading blocklists... ") t := time.Now() - if err := blocklists.Load(context.Background()); err != nil { + if err := blocklists.Load(context.Background(), true); err != nil { fmt.Fprintln(os.Stderr) log.Printf("blocklists: %v", err) if !asyncServe { @@ -243,8 +241,8 @@ func main() { } fmt.Fprintf(os.Stderr, "%s (inbound=%s, outbound=%s)\n", time.Since(t).Round(time.Millisecond), - commafy(cfg.inbound.Value().Size()), - commafy(cfg.outbound.Value().Size()), + commafy(cfg.inbound.Current().Size()), + commafy(cfg.outbound.Current().Size()), ) } if !asyncServe { @@ -286,46 +284,41 @@ func main() { }) fmt.Fprint(os.Stderr, "Loading geoip... ") tGeo := time.Now() - if err := geoSet.Load(context.Background()); err != nil { + if err := geoSet.Load(context.Background(), true); err != nil { fmt.Fprintln(os.Stderr) log.Fatalf("geoip: %v", err) } fmt.Fprintf(os.Stderr, "%s\n", time.Since(tGeo).Round(time.Millisecond)) - defer func() { _ = cfg.geo.Value().Close() }() + defer func() { _ = cfg.geo.Current().Close() }() - // Whitelist: combined IPs + CIDRs in one file, refreshed on an interval. + // Whitelist: combined IPs + CIDRs in one file, polled for mtime changes. // A match here overrides any block decision from the blocklists. - // Uses net/iplist.Source — the refreshed-IP-list primitive net/ippolicy - // builds on — instead of a dataset wrapper; iplist owns the refresh. + var whitelistSet *dataset.Set + var loadWhitelist func() if cfg.WhitelistPath != "" { - t := time.Now() - whitelistSrc, err := iplist.NewSource(context.Background(), iplist.SourceConfig{ - Source: cfg.WhitelistPath, - RefreshInterval: refreshInterval, - Optional: true, + whitelistSet = dataset.NewSet(dataset.PollFiles(cfg.WhitelistPath)) + cfg.whitelist = addCohort(whitelistSet, func(_ context.Context) (*ipcohort.Cohort, error) { + return ipcohort.LoadFile(cfg.WhitelistPath) }) - if err != nil { - log.Fatalf("whitelist: %v", err) - } - store := func() { - coh, perr := ipcohort.Parse(whitelistSrc.Entries()) - if perr != nil { - log.Printf("whitelist: %v", perr) - } - cfg.whitelist.Store(coh) - } - store() - whitelistSrc.Subscribe(func(e iplist.SourceEvent) { - if e.Err != nil { - log.Printf("whitelist refresh: %v", e.Err) + loadWhitelist = func() { + fmt.Fprint(os.Stderr, "Loading whitelist... ") + t := time.Now() + if err := whitelistSet.Load(context.Background(), true); err != nil { + fmt.Fprintln(os.Stderr) + log.Printf("whitelist: %v", err) + if !asyncServe { + os.Exit(1) + } return } - store() - }) - fmt.Fprintf(os.Stderr, "Loading whitelist... %s (entries=%s)\n", - time.Since(t).Round(time.Millisecond), - commafy(cfg.whitelist.Load().Size()), - ) + fmt.Fprintf(os.Stderr, "%s (entries=%s)\n", + time.Since(t).Round(time.Millisecond), + commafy(cfg.whitelist.Current().Size()), + ) + } + if !asyncServe { + loadWhitelist() + } } // Blank line separates the stderr "Loading ..." block from the real @@ -380,13 +373,15 @@ func main() { defer stop() if asyncServe { go loadBlocklists() + if loadWhitelist != nil { + go loadWhitelist() + } + } + blocklists.Start(ctx, refreshInterval) + geoSet.Start(ctx, refreshInterval) + if whitelistSet != nil { + whitelistSet.Start(ctx, refreshInterval) } - go blocklists.Tick(ctx, refreshInterval, func(err error) { - log.Printf("blocklists refresh: %v", err) - }) - go geoSet.Tick(ctx, refreshInterval, func(err error) { - log.Printf("geoip refresh: %v", err) - }) if err := cfg.serve(ctx); err != nil { log.Fatalf("serve: %v", err) } diff --git a/cmd/check-ip/server.go b/cmd/check-ip/server.go index d49803c..7635bcf 100644 --- a/cmd/check-ip/server.go +++ b/cmd/check-ip/server.go @@ -31,7 +31,7 @@ type Result struct { // lookup builds a Result for ip against the currently loaded blocklists // and GeoIP databases. func (c *IPCheck) lookup(ip string) Result { - res := Result{IP: ip, Geo: c.geo.Value().Lookup(ip)} + res := Result{IP: ip, Geo: c.geo.Current().Lookup(ip)} addr, err := netip.ParseAddr(ip) if err != nil { res.Blocked = true @@ -39,12 +39,12 @@ func (c *IPCheck) lookup(ip string) Result { res.BlockedOutbound = true return res } - if cur := c.whitelist.Load(); cur != nil && cur.ContainsAddr(addr) { + if c.whitelist != nil && c.whitelist.Current().ContainsAddr(addr) { res.Allowlisted = true return res } - res.BlockedInbound = c.inbound.Value().ContainsAddr(addr) - res.BlockedOutbound = c.outbound.Value().ContainsAddr(addr) + res.BlockedInbound = c.inbound.Current().ContainsAddr(addr) + res.BlockedOutbound = c.outbound.Current().ContainsAddr(addr) res.Blocked = res.BlockedInbound || res.BlockedOutbound return res } @@ -119,8 +119,8 @@ type dsStatus struct { // loaded, 503 while any are still empty. func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) { cohortStatus := func(v *dataset.View[ipcohort.Cohort]) dsStatus { - s := dsStatus{LoadedAt: v.LoadedAt()} - if cur := v.Value(); cur != nil { + s := dsStatus{LoadedAt: v.Status().LoadedAt} + if cur := v.Current(); cur != nil { s.Loaded, s.Size = true, cur.Size() } return s @@ -130,12 +130,12 @@ func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) { "inbound": cohortStatus(c.inbound), "outbound": cohortStatus(c.outbound), "geoip": { - Loaded: c.geo.Value() != nil, - LoadedAt: c.geo.LoadedAt(), + Loaded: c.geo.Current() != nil, + LoadedAt: c.geo.Status().LoadedAt, }, } - if cur := c.whitelist.Load(); cur != nil { - datasets["whitelist"] = dsStatus{Loaded: true, Size: cur.Size()} + if c.whitelist != nil { + datasets["whitelist"] = cohortStatus(c.whitelist) } ready := datasets["inbound"].Loaded && datasets["outbound"].Loaded && datasets["geoip"].Loaded