feat!(ippolicy): share refreshed IP policy behavior

Rename net/ipgate to net/ippolicy and compose independently refreshed
whitelist and blacklist sources into one IP decision policy.

- rename slog group and error prefixes from ipgate to ippolicy
- delete unused ParseDomainSet and RecordReader from domainsetcsv.go
- rewrite README to match the actual Config API
- document nil-whitelist behavior on New()
- bump iplist dependency to v0.8.5
- go mod tidy: remove stale ipgate and iplist entries

net/ipgate/domainsetcsv.go

@@ -1,61 +0,0 @@
-package ipgate
-
-import (
-	"fmt"
-	"io"
-	"net/netip"
-	"strings"
-)
-
-// RecordReader is satisfied by *csv.Reader.
-type RecordReader interface {
-	Read() (record []string, err error)
-}
-
-// ParseDomainSet reads rows from r and returns static CIDR prefixes and
-// hostnames to resolve. The first row is skipped when it contains no IP,
-// prefix, or dotted hostname (header detection).
-func ParseDomainSet(r RecordReader) (staticPrefixes []string, domains []string, err error) {
-	firstRow := true
-	for {
-		record, readErr := r.Read()
-		if readErr == io.EOF {
-			break
-		}
-		if readErr != nil {
-			return nil, nil, fmt.Errorf("csv read: %w", readErr)
-		}
-		if len(record) == 0 {
-			continue
-		}
-
-		raw := strings.TrimSpace(record[0])
-
-		if firstRow {
-			firstRow = false
-			if _, err := netip.ParseAddr(raw); err != nil {
-				if _, err := netip.ParsePrefix(raw); err != nil {
-					if !strings.Contains(raw, ".") {
-						continue
-					}
-				}
-			}
-		}
-		if raw == "" {
-			continue
-		}
-
-		if _, err := netip.ParseAddr(raw); err == nil {
-			staticPrefixes = append(staticPrefixes, raw+"/32")
-			continue
-		}
-		if _, err := netip.ParsePrefix(raw); err == nil {
-			staticPrefixes = append(staticPrefixes, raw)
-			continue
-		}
-
-		domains = append(domains, raw)
-	}
-
-	return staticPrefixes, domains, nil
-}

net/ippolicy/README.md

@@ -0,0 +1,73 @@
+# net/ippolicy
+
+`ippolicy` composes independently refreshed IP sources into a fast policy
+check. It uses `net/iplist` for local/HTTP source refresh and provides IP
+matching, DNS refresh, and the legacy git blacklist.
+
+## Quick start
+
+Construct each source with `net/iplist.NewSource`, then pass them to
+`ippolicy.New`:
+
+```go
+whitelist, err := iplist.NewSource(ctx, iplist.SourceConfig{
+    Source:     "allowed.csv",
+    CacheDir:   "/var/cache/app/iplist",
+    // HTTPClient: nil, // uses golib's internal client
+})
+if err != nil {
+    return err
+}
+defer whitelist.Close()
+
+blacklistExtra, err := iplist.NewSource(ctx, iplist.SourceConfig{
+    Source:     "blocked-ips-extra.tsv",
+    CacheDir:   "/var/cache/app/iplist",
+    Optional:   true,
+})
+if err != nil {
+    return err
+}
+defer blacklistExtra.Close()
+
+gitBlacklist, err := ippolicy.NewPrefixSet(
+    ctx,
+    "https://github.com/bitwire-it/ipblocklist.git",
+    "/var/cache/app/ipblocklist",
+    []string{"ipblocklist.txt"},
+    ippolicy.DefaultPrefixSetRefreshInterval,
+)
+if err != nil {
+    return err
+}
+defer gitBlacklist.Close()
+
+policy := ippolicy.New(ctx, ippolicy.Config{
+    Whitelist:      whitelist,
+    Blacklist:      gitBlacklist,
+    BlacklistExtra: blacklistExtra,
+    OnRefresh: func(e ippolicy.RefreshEvent) {
+        slog.Info("ippolicy refresh", "kind", e.Kind, "err", e.Err)
+    },
+})
+defer policy.Close()
+
+switch policy.Evaluate(addr) {
+case ippolicy.Blacklisted:
+    // reject
+case ippolicy.Whitelisted, ippolicy.Unlisted:
+    // continue
+}
+```
+
+## Behavior
+
+Each source refreshes independently and keeps its last valid data on failure.
+Whitelist matches take precedence over blacklist matches. The policy does not
+log; use `OnRefresh` for application logging and metrics.
+
+A nil `Whitelist` produces a policy where every address evaluates to
+`Unlisted`. This intentionally drops the blacklists as well: without a
+whitelist the policy cannot distinguish "allowed" from "unknown", so blocking
+would be meaningless. The caller receives a `RefreshFallback` event so it can
+log or alert.

net/ippolicy/domains.go

@@ -1,7 +1,8 @@
-package ipgate
+package ippolicy
 
 import (
 	"context"
+	"net"
 	"net/netip"
 	"sync"
 	"sync/atomic"
@@ -34,6 +35,29 @@ func EmptyDomainSet() *DomainSet {
 	return ds
 }
 
+func NewDomainSetFromEntries(ctx context.Context, entries []string, interval time.Duration) *DomainSet {
+	static, domains := splitEntries(entries)
+	return NewDomainSet(ctx, static, domains, interval)
+}
+
+func (ds *DomainSet) ReplaceEntries(entries []string) {
+	static, domains := splitEntries(entries)
+	ds.Replace(static, domains)
+}
+
+func splitEntries(entries []string) (static, domains []string) {
+	for _, entry := range entries {
+		if net.ParseIP(entry) != nil {
+			static = append(static, entry)
+		} else if _, _, err := net.ParseCIDR(entry); err == nil {
+			static = append(static, entry)
+		} else {
+			domains = append(domains, entry)
+		}
+	}
+	return static, domains
+}
+
 // NewDomainSet creates a DomainSet from pre-parsed inputs.
 // staticPrefixes is a list of CIDRs or bare IPs.
 // domains is a list of hostnames to resolve periodically.

net/ippolicy/format.go

@@ -1,4 +1,4 @@
-package ipgate
+package ippolicy
 
 import (
 	"log/slog"
@@ -6,7 +6,7 @@ import (
 	"strings"
 )
 
-func log() *slog.Logger { return slog.Default().WithGroup("ipgate") }
+func log() *slog.Logger { return slog.Default().WithGroup("ippolicy") }
 
 func commaify(n int) string {
 	s := strconv.Itoa(n)

net/ippolicy/go.mod

@@ -1,15 +1,19 @@
-module github.com/therootcompany/golib/net/ipgate
+module github.com/therootcompany/golib/net/ippolicy
 
-go 1.26.0
+go 1.26.5
 
 require (
 	github.com/therootcompany/golib/net/dnsresolver v0.5.0
 	github.com/therootcompany/golib/net/gitshallow v0.9.1
 	github.com/therootcompany/golib/net/ipcohort v0.9.0
+	github.com/therootcompany/golib/net/iplist v0.8.5
 )
 
 require (
 	github.com/miekg/dns v1.1.69 // indirect
+	github.com/therootcompany/golib/https v0.9.0 // indirect
+	github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect
+	github.com/therootcompany/golib/net/httpcache v0.5.1 // indirect
 	golang.org/x/mod v0.30.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect

net/ippolicy/go.sum

@@ -2,12 +2,20 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc=
 github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g=
+github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
+github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
+github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo=
+github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o=
 github.com/therootcompany/golib/net/dnsresolver v0.5.0 h1:JTHvRVjc/cCL6SpWFo6btvv0lllW2wTYjRXLE8cOdMw=
 github.com/therootcompany/golib/net/dnsresolver v0.5.0/go.mod h1:tOV99lX6ghpWPksxSFAt4CDe5XLefFzIGf2qXsZJap4=
 github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q=
 github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc=
+github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
+github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
 github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E=
 github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys=
+github.com/therootcompany/golib/net/iplist v0.8.5 h1:swn/hcCt6fAG3tYApmxh1OlDak2PjQdczldZWaH1Eus=
+github.com/therootcompany/golib/net/iplist v0.8.5/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU=
 golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
 golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=

net/ippolicy/ipprefix.go

@@ -1,4 +1,4 @@
-package ipgate
+package ippolicy
 
 import (
 	"context"
@@ -30,7 +30,7 @@ func EmptyPrefixSet() *PrefixSet {
 
 func NewPrefixSet(ctx context.Context, repoURL, dataPath string, files []string, interval time.Duration) (*PrefixSet, error) {
 	if err := os.MkdirAll(dataPath, 0o755); err != nil {
-		return nil, fmt.Errorf("ipgate: create data dir: %w", err)
+		return nil, fmt.Errorf("ippolicy: create data dir: %w", err)
 	}
 
 	if interval <= 0 {

net/ippolicy/ipprefix_test.go

@@ -1,4 +1,4 @@
-package ipgate
+package ippolicy
 
 import (
 	"net/netip"

net/ippolicy/policy.go

@@ -0,0 +1,156 @@
+// Package ippolicy composes independently refreshed whitelist and blacklist
+// sources into one IP decision policy.
+package ippolicy
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/netip"
+
+	"github.com/therootcompany/golib/net/iplist"
+)
+
+type Config struct {
+	Whitelist      *iplist.Source
+	Blacklist      *PrefixSet
+	BlacklistExtra *iplist.Source
+	OnRefresh      func(RefreshEvent)
+}
+
+// Decision is the mutually exclusive result of evaluating an address.
+type Decision uint8
+
+const (
+	Blacklisted Decision = iota
+	Unlisted
+	Whitelisted
+)
+
+func (d Decision) String() string {
+	switch d {
+	case Blacklisted:
+		return "blacklisted"
+	case Unlisted:
+		return "unlisted"
+	case Whitelisted:
+		return "whitelisted"
+	default:
+		return fmt.Sprintf("decision(%d)", d)
+	}
+}
+
+type RefreshKind uint8
+
+const (
+	RefreshSuccess RefreshKind = iota
+	RefreshFailure
+	RefreshFallback
+)
+
+func (k RefreshKind) String() string {
+	switch k {
+	case RefreshSuccess:
+		return "success"
+	case RefreshFailure:
+		return "failure"
+	case RefreshFallback:
+		return "fallback"
+	default:
+		return fmt.Sprintf("refresh(%d)", k)
+	}
+}
+
+type RefreshEvent struct {
+	Kind RefreshKind
+	Err  error
+}
+
+type Policy struct {
+	onRefresh func(RefreshEvent)
+	whitelist *DomainSet
+	extra     *DomainSet
+	git       *PrefixSet
+}
+
+// New creates a Policy from the given sources. The caller is responsible for
+// constructing and later closing the *iplist.Source values; New does not take
+// ownership of them.
+//
+// A nil Whitelist is treated as an empty whitelist: every address evaluates to
+// Unlisted. This also intentionally drops the blacklists, because a policy
+// without a whitelist provides no meaningful allow-list semantics — blocking
+// would reject everything not on the blacklist rather than only what is on it.
+// The caller receives a RefreshFallback event so it can log or alert.
+func New(ctx context.Context, config Config) *Policy {
+	p := &Policy{onRefresh: config.OnRefresh}
+	if config.Whitelist == nil {
+		p.whitelist = EmptyDomainSet()
+		p.emit(RefreshFallback, errors.New("whitelist source is required"))
+		return p
+	}
+
+	p.whitelist = NewDomainSetFromEntries(ctx, config.Whitelist.Entries(), DefaultDomainSetRefreshInterval)
+	config.Whitelist.Subscribe(func(event iplist.SourceEvent) {
+		if event.Err != nil {
+			p.emit(RefreshFailure, event.Err)
+			return
+		}
+		p.whitelist.ReplaceEntries(config.Whitelist.Entries())
+		p.emit(RefreshSuccess, nil)
+	})
+
+	if config.BlacklistExtra != nil {
+		p.extra = NewDomainSetFromEntries(ctx, config.BlacklistExtra.Entries(), DefaultDomainSetRefreshInterval)
+		config.BlacklistExtra.Subscribe(func(event iplist.SourceEvent) {
+			if event.Err != nil {
+				p.emit(RefreshFailure, event.Err)
+				return
+			}
+			p.extra.ReplaceEntries(config.BlacklistExtra.Entries())
+			p.emit(RefreshSuccess, nil)
+		})
+	} else {
+		p.extra = EmptyDomainSet()
+	}
+	p.git = config.Blacklist
+	p.emit(RefreshSuccess, nil)
+	return p
+}
+
+func (p *Policy) Evaluate(addr netip.Addr) Decision {
+	if p.whitelist == nil {
+		return Unlisted
+	}
+	if p.whitelist.Contains(addr) {
+		return Whitelisted
+	}
+	if (p.extra != nil && p.extra.Contains(addr)) ||
+		(p.git != nil && p.git.Contains(addr)) {
+		return Blacklisted
+	}
+	return Unlisted
+}
+
+func (p *Policy) Close() error {
+	if p == nil {
+		return nil
+	}
+	if p.whitelist != nil {
+		_ = p.whitelist.Close()
+	}
+	if p.extra != nil {
+		_ = p.extra.Close()
+	}
+	if p.git != nil {
+		_ = p.git.Close()
+	}
+	return nil
+}
+
+func (p *Policy) emit(kind RefreshKind, err error) {
+	if p.onRefresh == nil {
+		return
+	}
+	p.onRefresh(RefreshEvent{Kind: kind, Err: err})
+}

net/ippolicy/policy_test.go

@@ -0,0 +1,94 @@
+package ippolicy
+
+import (
+	"context"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/iplist"
+)
+
+func TestEnumStrings(t *testing.T) {
+	for _, test := range []struct {
+		value Decision
+		want  string
+	}{
+		{Blacklisted, "blacklisted"},
+		{Unlisted, "unlisted"},
+		{Whitelisted, "whitelisted"},
+		{Decision(99), "decision(99)"},
+	} {
+		if got := test.value.String(); got != test.want {
+			t.Fatalf("Decision(%d).String() = %q, want %q", test.value, got, test.want)
+		}
+	}
+	for _, test := range []struct {
+		value RefreshKind
+		want  string
+	}{
+		{RefreshSuccess, "success"},
+		{RefreshFailure, "failure"},
+		{RefreshFallback, "fallback"},
+		{RefreshKind(99), "refresh(99)"},
+	} {
+		if got := test.value.String(); got != test.want {
+			t.Fatalf("RefreshKind(%d).String() = %q, want %q", test.value, got, test.want)
+		}
+	}
+}
+
+func TestPolicyKeepsSnapshotAfterFailedRefresh(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "allowed.tsv")
+	if err := os.WriteFile(path, []byte("192.0.2.1\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	whitelist, err := iplist.NewSource(t.Context(), iplist.SourceConfig{Source: path})
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := New(t.Context(), Config{Whitelist: whitelist})
+	defer func() { _ = p.Close() }()
+
+	if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted {
+		t.Fatal("initial whitelist did not match")
+	}
+	if err := os.WriteFile(path, []byte("invalid entry\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := whitelist.Refresh(); err == nil {
+		t.Fatal("invalid refresh unexpectedly succeeded")
+	}
+	if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted {
+		t.Fatal("failed refresh replaced valid snapshot")
+	}
+}
+
+func TestPolicyWhitelistPrecedesExtraBlacklist(t *testing.T) {
+	dir := t.TempDir()
+	white := filepath.Join(dir, "white.tsv")
+	black := filepath.Join(dir, "black.tsv")
+	if err := os.WriteFile(white, []byte("192.0.2.1\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(black, []byte("192.0.2.0/24\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	whitelist, err := iplist.NewSource(context.Background(), iplist.SourceConfig{Source: white})
+	if err != nil {
+		t.Fatal(err)
+	}
+	extra, err := iplist.NewSource(context.Background(), iplist.SourceConfig{Source: black})
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := New(context.Background(), Config{Whitelist: whitelist, BlacklistExtra: extra})
+	defer func() { _ = p.Close() }()
+	if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted {
+		t.Fatal("whitelist did not take precedence")
+	}
+	if p.Evaluate(netip.MustParseAddr("192.0.2.2")) != Blacklisted {
+		t.Fatal("extra blacklist did not block")
+	}
+}