refactor(cmd/check-ip): use released cache modules

cmd/check-ip/go.mod

@@ -3,19 +3,18 @@ module github.com/therootcompany/golib/cmd/check-ip
 go 1.26.5
 
 require (
-	github.com/therootcompany/golib/net/geoip v0.5.0
+	github.com/therootcompany/golib/net/geoip v0.6.0
-	github.com/therootcompany/golib/net/gitshallow v0.9.1
+	github.com/therootcompany/golib/net/gitshallow v0.10.0
-	github.com/therootcompany/golib/net/httpcache v0.5.1
+	github.com/therootcompany/golib/net/httpcache v0.6.0
 	github.com/therootcompany/golib/net/ipcohort v0.9.0
-	github.com/therootcompany/golib/net/iplist v0.8.5
+	github.com/therootcompany/golib/sync/dataset v0.6.0
-	github.com/therootcompany/golib/sync/dataset v0.5.0
 )
 
 require (
 	github.com/oschwald/geoip2-golang v1.13.0 // indirect
 	github.com/oschwald/maxminddb-golang v1.13.0 // indirect
 	github.com/therootcompany/golib/https v0.9.0 // indirect
-	github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect
+	github.com/therootcompany/golib/sync/cachable v0.1.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
 )

cmd/check-ip/go.sum

@@ -10,20 +10,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
 github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
-github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo=
+github.com/therootcompany/golib/net/geoip v0.6.0 h1:+LyOKt47Mbxoi8ssaX9RaN0loW6wyPDA5XvExSiRzB0=
-github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o=
+github.com/therootcompany/golib/net/geoip v0.6.0/go.mod h1:yrk7CuIet0BntICfRwRSX44fwu/1YAuHHjTQYoWHSOE=
-github.com/therootcompany/golib/net/geoip v0.5.0 h1:dmduwHLX3RCe2NExhOKcFZ4m1ML2C3gQKnNyaBhNE88=
+github.com/therootcompany/golib/net/gitshallow v0.10.0 h1:cLN9ElTmNWk5HSbaRD3sSFuGdpgy4YXc+LVzjzT0nM4=
-github.com/therootcompany/golib/net/geoip v0.5.0/go.mod h1:BLf5g25UhvI/+/x/WpM1NTVs4ZMnjySoHv+LXfALT6M=
+github.com/therootcompany/golib/net/gitshallow v0.10.0/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
-github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q=
+github.com/therootcompany/golib/net/httpcache v0.6.0 h1:qFBnSs6rZrtUQ+fweP4bHF2FQd/Syvs7mRN5nO5YtkY=
-github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
+github.com/therootcompany/golib/net/httpcache v0.6.0/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
-github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
-github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
 github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E=
 github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys=
-github.com/therootcompany/golib/net/iplist v0.8.5 h1:swn/hcCt6fAG3tYApmxh1OlDak2PjQdczldZWaH1Eus=
+github.com/therootcompany/golib/sync/cachable v0.1.0 h1:swCXtXB116jhSmla81HaXi/9cPsqFI+Oq/6OcSfdU5Q=
-github.com/therootcompany/golib/net/iplist v0.8.5/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU=
+github.com/therootcompany/golib/sync/cachable v0.1.0/go.mod h1:l/NZdGiSbW5CkLlkkRa0yObkqak2sj/jQTXXqXwvCl0=
-github.com/therootcompany/golib/sync/dataset v0.5.0 h1:4i3+7Izy/vnpK6IW5m9u4UqLA9RM1IjUsedbZo3MW6E=
+github.com/therootcompany/golib/sync/dataset v0.6.0 h1:Gerq315Ld1PyfD+6EmTDxSEAOg6s0kDchL2oo/jAQZ0=
-github.com/therootcompany/golib/sync/dataset v0.5.0/go.mod h1:HMa25n3eqYnjRcPXny0IUGU2wFKD5/x2g9UUkt3iAmI=
+github.com/therootcompany/golib/sync/dataset v0.6.0/go.mod h1:ktn0ppeIpNvgNLFYFuD9W6hGEQfwaIEcE8TV048+Y9M=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=

cmd/check-ip/main.go

@@ -19,7 +19,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -27,7 +26,6 @@ import (
 	"github.com/therootcompany/golib/net/gitshallow"
 	"github.com/therootcompany/golib/net/httpcache"
 	"github.com/therootcompany/golib/net/ipcohort"
-	"github.com/therootcompany/golib/net/iplist"
 	"github.com/therootcompany/golib/sync/dataset"
 )
 
@@ -117,7 +115,7 @@ type IPCheck struct {
 
 	inbound   *dataset.View[ipcohort.Cohort]
 	outbound  *dataset.View[ipcohort.Cohort]
-	whitelist atomic.Pointer[ipcohort.Cohort]
+	whitelist *dataset.View[ipcohort.Cohort]
 	geo       *dataset.View[geoip.Databases]
 }
 
@@ -233,7 +231,7 @@ func main() {
 	loadBlocklists := func() {
 		fmt.Fprint(os.Stderr, "Loading blocklists... ")
 		t := time.Now()
-		if err := blocklists.Load(context.Background()); err != nil {
+		if err := blocklists.Load(context.Background(), true); err != nil {
 			fmt.Fprintln(os.Stderr)
 			log.Printf("blocklists: %v", err)
 			if !asyncServe {
@@ -243,8 +241,8 @@ func main() {
 		}
 		fmt.Fprintf(os.Stderr, "%s (inbound=%s, outbound=%s)\n",
 			time.Since(t).Round(time.Millisecond),
-			commafy(cfg.inbound.Value().Size()),
+			commafy(cfg.inbound.Current().Size()),
-			commafy(cfg.outbound.Value().Size()),
+			commafy(cfg.outbound.Current().Size()),
 		)
 	}
 	if !asyncServe {
@@ -286,46 +284,41 @@ func main() {
 	})
 	fmt.Fprint(os.Stderr, "Loading geoip... ")
 	tGeo := time.Now()
-	if err := geoSet.Load(context.Background()); err != nil {
+	if err := geoSet.Load(context.Background(), true); err != nil {
 		fmt.Fprintln(os.Stderr)
 		log.Fatalf("geoip: %v", err)
 	}
 	fmt.Fprintf(os.Stderr, "%s\n", time.Since(tGeo).Round(time.Millisecond))
-	defer func() { _ = cfg.geo.Value().Close() }()
+	defer func() { _ = cfg.geo.Current().Close() }()
 
-	// Whitelist: combined IPs + CIDRs in one file, refreshed on an interval.
+	// Whitelist: combined IPs + CIDRs in one file, polled for mtime changes.
 	// A match here overrides any block decision from the blocklists.
-	// Uses net/iplist.Source — the refreshed-IP-list primitive net/ippolicy
+	var whitelistSet *dataset.Set
-	// builds on — instead of a dataset wrapper; iplist owns the refresh.
+	var loadWhitelist func()
 	if cfg.WhitelistPath != "" {
-		t := time.Now()
+		whitelistSet = dataset.NewSet(dataset.PollFiles(cfg.WhitelistPath))
-		whitelistSrc, err := iplist.NewSource(context.Background(), iplist.SourceConfig{
+		cfg.whitelist = addCohort(whitelistSet, func(_ context.Context) (*ipcohort.Cohort, error) {
-			Source:          cfg.WhitelistPath,
+			return ipcohort.LoadFile(cfg.WhitelistPath)
-			RefreshInterval: refreshInterval,
-			Optional:        true,
 		})
-		if err != nil {
+		loadWhitelist = func() {
-			log.Fatalf("whitelist: %v", err)
+			fmt.Fprint(os.Stderr, "Loading whitelist... ")
-		}
+			t := time.Now()
-		store := func() {
+			if err := whitelistSet.Load(context.Background(), true); err != nil {
-			coh, perr := ipcohort.Parse(whitelistSrc.Entries())
+				fmt.Fprintln(os.Stderr)
-			if perr != nil {
+				log.Printf("whitelist: %v", err)
-				log.Printf("whitelist: %v", perr)
+				if !asyncServe {
-			}
+					os.Exit(1)
-			cfg.whitelist.Store(coh)
+				}
-		}
-		store()
-		whitelistSrc.Subscribe(func(e iplist.SourceEvent) {
-			if e.Err != nil {
-				log.Printf("whitelist refresh: %v", e.Err)
 				return
 			}
-			store()
+			fmt.Fprintf(os.Stderr, "%s (entries=%s)\n",
-		})
+				time.Since(t).Round(time.Millisecond),
-		fmt.Fprintf(os.Stderr, "Loading whitelist... %s (entries=%s)\n",
+				commafy(cfg.whitelist.Current().Size()),
-			time.Since(t).Round(time.Millisecond),
+			)
-			commafy(cfg.whitelist.Load().Size()),
+		}
-		)
+		if !asyncServe {
+			loadWhitelist()
+		}
 	}
 
 	// Blank line separates the stderr "Loading ..." block from the real
@@ -380,13 +373,15 @@ func main() {
 	defer stop()
 	if asyncServe {
 		go loadBlocklists()
+		if loadWhitelist != nil {
+			go loadWhitelist()
+		}
+	}
+	blocklists.Start(ctx, refreshInterval)
+	geoSet.Start(ctx, refreshInterval)
+	if whitelistSet != nil {
+		whitelistSet.Start(ctx, refreshInterval)
 	}
-	go blocklists.Tick(ctx, refreshInterval, func(err error) {
-		log.Printf("blocklists refresh: %v", err)
-	})
-	go geoSet.Tick(ctx, refreshInterval, func(err error) {
-		log.Printf("geoip refresh: %v", err)
-	})
 	if err := cfg.serve(ctx); err != nil {
 		log.Fatalf("serve: %v", err)
 	}

cmd/check-ip/server.go

@@ -31,7 +31,7 @@ type Result struct {
 // lookup builds a Result for ip against the currently loaded blocklists
 // and GeoIP databases.
 func (c *IPCheck) lookup(ip string) Result {
-	res := Result{IP: ip, Geo: c.geo.Value().Lookup(ip)}
+	res := Result{IP: ip, Geo: c.geo.Current().Lookup(ip)}
 	addr, err := netip.ParseAddr(ip)
 	if err != nil {
 		res.Blocked = true
@@ -39,12 +39,12 @@ func (c *IPCheck) lookup(ip string) Result {
 		res.BlockedOutbound = true
 		return res
 	}
-	if cur := c.whitelist.Load(); cur != nil && cur.ContainsAddr(addr) {
+	if c.whitelist != nil && c.whitelist.Current().ContainsAddr(addr) {
 		res.Allowlisted = true
 		return res
 	}
-	res.BlockedInbound = c.inbound.Value().ContainsAddr(addr)
+	res.BlockedInbound = c.inbound.Current().ContainsAddr(addr)
-	res.BlockedOutbound = c.outbound.Value().ContainsAddr(addr)
+	res.BlockedOutbound = c.outbound.Current().ContainsAddr(addr)
 	res.Blocked = res.BlockedInbound || res.BlockedOutbound
 	return res
 }
@@ -119,8 +119,8 @@ type dsStatus struct {
 // loaded, 503 while any are still empty.
 func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) {
 	cohortStatus := func(v *dataset.View[ipcohort.Cohort]) dsStatus {
-		s := dsStatus{LoadedAt: v.LoadedAt()}
+		s := dsStatus{LoadedAt: v.Status().LoadedAt}
-		if cur := v.Value(); cur != nil {
+		if cur := v.Current(); cur != nil {
 			s.Loaded, s.Size = true, cur.Size()
 		}
 		return s
@@ -130,12 +130,12 @@ func (c *IPCheck) healthz(w http.ResponseWriter, _ *http.Request) {
 		"inbound":  cohortStatus(c.inbound),
 		"outbound": cohortStatus(c.outbound),
 		"geoip": {
-			Loaded:   c.geo.Value() != nil,
+			Loaded:   c.geo.Current() != nil,
-			LoadedAt: c.geo.LoadedAt(),
+			LoadedAt: c.geo.Status().LoadedAt,
 		},
 	}
-	if cur := c.whitelist.Load(); cur != nil {
+	if c.whitelist != nil {
-		datasets["whitelist"] = dsStatus{Loaded: true, Size: cur.Size()}
+		datasets["whitelist"] = cohortStatus(c.whitelist)
 	}
 
 	ready := datasets["inbound"].Loaded && datasets["outbound"].Loaded && datasets["geoip"].Loaded