root/golib
1
0
Fork 0
mirror of https://github.com/therootcompany/golib.git synced 2026-11-04 06:55:00 +00:00
Code Issues Projects Releases Wiki Activity

refactor(cmd/check-ip): use released cache modules

Browse Source
This commit is contained in:
AJ ONeal 2026-09-09 22:38:50 -06:00
parent 4e85e00d5a
commit 8648c5accb
No known key found for this signature in database
4 changed files with 62 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/check-ip/go.mod
View File

@ -3,19 +3,18 @@ module github.com/therootcompany/golib/cmd/check-ip
go 1.26.5
require (
github.com/therootcompany/golib/net/geoip v0.5.0
github.com/therootcompany/golib/net/gitshallow v0.9.1
github.com/therootcompany/golib/net/httpcache v0.5.1
github.com/therootcompany/golib/net/geoip v0.6.0
github.com/therootcompany/golib/net/gitshallow v0.10.0
github.com/therootcompany/golib/net/httpcache v0.6.0
github.com/therootcompany/golib/net/ipcohort v0.9.0
github.com/therootcompany/golib/net/iplist v0.8.5
github.com/therootcompany/golib/sync/dataset v0.5.0
github.com/therootcompany/golib/sync/dataset v0.6.0
)
require (
github.com/oschwald/geoip2-golang v1.13.0 // indirect
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
github.com/therootcompany/golib/https v0.9.0 // indirect
github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect
github.com/therootcompany/golib/sync/cachable v0.1.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.45.0 // indirect
)

22
cmd/check-ip/go.sum
View File

@ -10,20 +10,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo=
github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o=
github.com/therootcompany/golib/net/geoip v0.5.0 h1:dmduwHLX3RCe2NExhOKcFZ4m1ML2C3gQKnNyaBhNE88=
github.com/therootcompany/golib/net/geoip v0.5.0/go.mod h1:BLf5g25UhvI/+/x/WpM1NTVs4ZMnjySoHv+LXfALT6M=
github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q=
github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
github.com/therootcompany/golib/net/geoip v0.6.0 h1:+LyOKt47Mbxoi8ssaX9RaN0loW6wyPDA5XvExSiRzB0=
github.com/therootcompany/golib/net/geoip v0.6.0/go.mod h1:yrk7CuIet0BntICfRwRSX44fwu/1YAuHHjTQYoWHSOE=
github.com/therootcompany/golib/net/gitshallow v0.10.0 h1:cLN9ElTmNWk5HSbaRD3sSFuGdpgy4YXc+LVzjzT0nM4=
github.com/therootcompany/golib/net/gitshallow v0.10.0/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
github.com/therootcompany/golib/net/httpcache v0.6.0 h1:qFBnSs6rZrtUQ+fweP4bHF2FQd/Syvs7mRN5nO5YtkY=
github.com/therootcompany/golib/net/httpcache v0.6.0/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E=
github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys=
github.com/therootcompany/golib/net/iplist v0.8.5 h1:swn/hcCt6fAG3tYApmxh1OlDak2PjQdczldZWaH1Eus=
github.com/therootcompany/golib/net/iplist v0.8.5/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU=
github.com/therootcompany/golib/sync/dataset v0.5.0 h1:4i3+7Izy/vnpK6IW5m9u4UqLA9RM1IjUsedbZo3MW6E=
github.com/therootcompany/golib/sync/dataset v0.5.0/go.mod h1:HMa25n3eqYnjRcPXny0IUGU2wFKD5/x2g9UUkt3iAmI=
github.com/therootcompany/golib/sync/cachable v0.1.0 h1:swCXtXB116jhSmla81HaXi/9cPsqFI+Oq/6OcSfdU5Q=
github.com/therootcompany/golib/sync/cachable v0.1.0/go.mod h1:l/NZdGiSbW5CkLlkkRa0yObkqak2sj/jQTXXqXwvCl0=
github.com/therootcompany/golib/sync/dataset v0.6.0 h1:Gerq315Ld1PyfD+6EmTDxSEAOg6s0kDchL2oo/jAQZ0=
github.com/therootcompany/golib/sync/dataset v0.6.0/go.mod h1:ktn0ppeIpNvgNLFYFuD9W6hGEQfwaIEcE8TV048+Y9M=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=

79
cmd/check-ip/main.go
View File

@ -19,7 +19,6 @@ import (
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
@ -27,7 +26,6 @@ import (
"github.com/therootcompany/golib/net/gitshallow"
"github.com/therootcompany/golib/net/httpcache"
"github.com/therootcompany/golib/net/ipcohort"
"github.com/therootcompany/golib/net/iplist"
"github.com/therootcompany/golib/sync/dataset"
)
@ -117,7 +115,7 @@ type IPCheck struct {
inbound *dataset.View[ipcohort.Cohort]
outbound *dataset.View[ipcohort.Cohort]
whitelist atomic.Pointer[ipcohort.Cohort]
whitelist *dataset.View[ipcohort.Cohort]
geo *dataset.View[geoip.Databases]
}
@ -233,7 +231,7 @@ func main() {
loadBlocklists := func() {
fmt.Fprint(os.Stderr, "Loading blocklists... ")
t := time.Now()
if err := blocklists.Load(context.Background()); err != nil {
if err := blocklists.Load(context.Background(), true); err != nil {
fmt.Fprintln(os.Stderr)
log.Printf("blocklists: %v", err)
if !asyncServe {
@ -243,8 +241,8 @@ func main() {
}
fmt.Fprintf(os.Stderr, "%s (inbound=%s, outbound=%s)\n",
time.Since(t).Round(time.Millisecond),
commafy(cfg.inbound.Value().Size()),
commafy(cfg.outbound.Value().Size()),
commafy(cfg.inbound.Current().Size()),
commafy(cfg.outbound.Current().Size()),
)
}
if !asyncServe {
@ -286,46 +284,41 @@ func main() {
})
fmt.Fprint(os.Stderr, "Loading geoip... ")
tGeo := time.Now()
if err := geoSet.Load(context.Background()); err != nil {
if err := geoSet.Load(context.Background(), true); err != nil {
fmt.Fprintln(os.Stderr)
log.Fatalf("geoip: %v", err)
}
fmt.Fprintf(os.Stderr, "%s\n", time.Since(tGeo).Round(time.Millisecond))
defer func() { _ = cfg.geo.Value().Close() }()
defer func() { _ = cfg.geo.Current().Close() }()
// Whitelist: combined IPs + CIDRs in one file, refreshed on an interval.
// Whitelist: combined IPs + CIDRs in one file, polled for mtime changes.
// A match here overrides any block decision from the blocklists.
// Uses net/iplist.Source — the refreshed-IP-list primitive net/ippolicy
// builds on — instead of a dataset wrapper; iplist owns the refresh.
var whitelistSet *dataset.Set
var loadWhitelist func()
if cfg.WhitelistPath != "" {
t := time.Now()
whitelistSrc, err := iplist.NewSource(context.Background(), iplist.SourceConfig{
Source: cfg.WhitelistPath,
RefreshInterval: refreshInterval,
Optional: true,
whitelistSet = dataset.NewSet(dataset.PollFiles(cfg.WhitelistPath))
cfg.whitelist = addCohort(whitelistSet, func(_ context.Context) (*ipcohort.Cohort, error) {
return ipcohort.LoadFile(cfg.WhitelistPath)
})
if err != nil {
log.Fatalf("whitelist: %v", err)
}
store := func() {
coh, perr := ipcohort.Parse(whitelistSrc.Entries())
if perr != nil {
log.Printf("whitelist: %v", perr)
}
cfg.whitelist.Store(coh)
}
store()
whitelistSrc.Subscribe(func(e iplist.SourceEvent) {
if e.Err != nil {
log.Printf("whitelist refresh: %v", e.Err)
loadWhitelist = func() {
fmt.Fprint(os.Stderr, "Loading whitelist... ")
t := time.Now()
if err := whitelistSet.Load(context.Background(), true); err != nil {
fmt.Fprintln(os.Stderr)
log.Printf("whitelist: %v", err)
if !asyncServe {
os.Exit(1)
}
return
}
store()
})
fmt.Fprintf(os.Stderr, "Loading whitelist... %s (entries=%s)\n",
time.Since(t).Round(time.Millisecond),
commafy(cfg.whitelist.Load().Size()),
)
fmt.Fprintf(os.Stderr, "%s (entries=%s)\n",
time.Since(t).Round(time.Millisecond),
commafy(cfg.whitelist.Current().Size()),
)
}
if !asyncServe {
loadWhitelist()
}
}
// Blank line separates the stderr "Loading ..." block from the real
@ -380,13 +373,15 @@ func main() {
defer stop()
if asyncServe {
go loadBlocklists()
if loadWhitelist != nil {
go loadWhitelist()
}
}
blocklists.Start(ctx, refreshInterval)
geoSet.Start(ctx, refreshInterval)
if whitelistSet != nil {
whitelistSet.Start(ctx, refreshInterval)
}
go blocklists.Tick(ctx, refreshInterval, func(err error) {
log.Printf("blocklists refresh: %v", err)
})
go geoSet.Tick(ctx, refreshInterval, func(err error) {
log.Printf("geoip refresh: %v", err)
})
if err := cfg.serve(ctx); err != nil {
log.Fatalf("serve: %v", err)
}

20
cmd/check-ip/server.go
View File

@ -31,7 +31,7 @@ type Result struct {
// lookup builds a Result for ip against the currently loaded blocklists
// and GeoIP databases.
func (c *IPCheck) lookup(ip string) Result {
res := Result{IP: ip, Geo: c.geo.Value().Lookup(ip)}
res := Result{IP: ip, Geo: c.geo.Current().Lookup(ip)}
addr, err := netip.ParseAddr(ip)
if err != nil {
res.Blocked = true
@ -39,12 +39,12 @@ func (c *IPCheck) lookup(ip string) Result {
res.BlockedOutbound = true
return res
}
if cur := c.whitelist.Load(); cur != nil && cur.ContainsAddr(addr) {
if c.whitelist != nil && c.whitelist.Current().ContainsAddr(addr) {
res.Allowlisted = true
return res
}
res.BlockedInbound = c.inbound.Value().ContainsAddr(addr)
res.BlockedOutbound = c.outbound.Value().ContainsAddr(addr)
res.BlockedInbound = c.inbound.Current().ContainsAddr(addr)
res.BlockedOutbound = c.outbound.Current().ContainsAddr(addr)
res.Blocked = res.BlockedInbound || res.BlockedOutbound
return res
}
@ -119,8 +119,8 @@ type dsStatus struct {
// loaded, 503 while any are still empty.
func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) {
cohortStatus := func(v *dataset.View[ipcohort.Cohort]) dsStatus {
s := dsStatus{LoadedAt: v.LoadedAt()}
if cur := v.Value(); cur != nil {
s := dsStatus{LoadedAt: v.Status().LoadedAt}
if cur := v.Current(); cur != nil {
s.Loaded, s.Size = true, cur.Size()
}
return s
@ -130,12 +130,12 @@ func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) {
"inbound": cohortStatus(c.inbound),
"outbound": cohortStatus(c.outbound),
"geoip": {
Loaded: c.geo.Value() != nil,
LoadedAt: c.geo.LoadedAt(),
Loaded: c.geo.Current() != nil,
LoadedAt: c.geo.Status().LoadedAt,
},
}
if cur := c.whitelist.Load(); cur != nil {
datasets["whitelist"] = dsStatus{Loaded: true, Size: cur.Size()}
if c.whitelist != nil {
datasets["whitelist"] = cohortStatus(c.whitelist)
}
ready := datasets["inbound"].Loaded && datasets["outbound"].Loaded && datasets["geoip"].Loaded