From 07d4a2652cfea0fda6f1d65f8342dc6e2c995510 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 9 Sep 2026 02:07:16 -0600 Subject: [PATCH] feat!(ippolicy): share refreshed IP policy behavior Rename net/ipgate to net/ippolicy and compose independently refreshed whitelist and blacklist sources into one IP decision policy. - rename slog group and error prefixes from ipgate to ippolicy - delete unused ParseDomainSet and RecordReader from domainsetcsv.go - rewrite README to match the actual Config API - document nil-whitelist behavior on New() - bump iplist dependency to v0.8.5 - go mod tidy: remove stale ipgate and iplist entries --- net/ipgate/domainsetcsv.go | 61 --------- net/ippolicy/README.md | 73 ++++++++++ net/{ipgate => ippolicy}/domains.go | 26 +++- net/{ipgate => ippolicy}/format.go | 4 +- net/{ipgate => ippolicy}/go.mod | 8 +- net/{ipgate => ippolicy}/go.sum | 8 ++ net/{ipgate => ippolicy}/ipprefix.go | 4 +- net/{ipgate => ippolicy}/ipprefix_test.go | 2 +- net/ippolicy/policy.go | 156 ++++++++++++++++++++++ net/ippolicy/policy_test.go | 94 +++++++++++++ 10 files changed, 367 insertions(+), 69 deletions(-) delete mode 100644 net/ipgate/domainsetcsv.go create mode 100644 net/ippolicy/README.md rename net/{ipgate => ippolicy}/domains.go (84%) rename net/{ipgate => ippolicy}/format.go (80%) rename net/{ipgate => ippolicy}/go.mod (57%) rename net/{ipgate => ippolicy}/go.sum (67%) rename net/{ipgate => ippolicy}/ipprefix.go (97%) rename net/{ipgate => ippolicy}/ipprefix_test.go (98%) create mode 100644 net/ippolicy/policy.go create mode 100644 net/ippolicy/policy_test.go diff --git a/net/ipgate/domainsetcsv.go b/net/ipgate/domainsetcsv.go deleted file mode 100644 index daf2c32..0000000 --- a/net/ipgate/domainsetcsv.go +++ /dev/null @@ -1,61 +0,0 @@ -package ipgate - -import ( - "fmt" - "io" - "net/netip" - "strings" -) - -// RecordReader is satisfied by *csv.Reader. -type RecordReader interface { - Read() (record []string, err error) -} - -// ParseDomainSet reads rows from r and returns static CIDR prefixes and -// hostnames to resolve. The first row is skipped when it contains no IP, -// prefix, or dotted hostname (header detection). -func ParseDomainSet(r RecordReader) (staticPrefixes []string, domains []string, err error) { - firstRow := true - for { - record, readErr := r.Read() - if readErr == io.EOF { - break - } - if readErr != nil { - return nil, nil, fmt.Errorf("csv read: %w", readErr) - } - if len(record) == 0 { - continue - } - - raw := strings.TrimSpace(record[0]) - - if firstRow { - firstRow = false - if _, err := netip.ParseAddr(raw); err != nil { - if _, err := netip.ParsePrefix(raw); err != nil { - if !strings.Contains(raw, ".") { - continue - } - } - } - } - if raw == "" { - continue - } - - if _, err := netip.ParseAddr(raw); err == nil { - staticPrefixes = append(staticPrefixes, raw+"/32") - continue - } - if _, err := netip.ParsePrefix(raw); err == nil { - staticPrefixes = append(staticPrefixes, raw) - continue - } - - domains = append(domains, raw) - } - - return staticPrefixes, domains, nil -} diff --git a/net/ippolicy/README.md b/net/ippolicy/README.md new file mode 100644 index 0000000..82726de --- /dev/null +++ b/net/ippolicy/README.md @@ -0,0 +1,73 @@ +# net/ippolicy + +`ippolicy` composes independently refreshed IP sources into a fast policy +check. It uses `net/iplist` for local/HTTP source refresh and provides IP +matching, DNS refresh, and the legacy git blacklist. + +## Quick start + +Construct each source with `net/iplist.NewSource`, then pass them to +`ippolicy.New`: + +```go +whitelist, err := iplist.NewSource(ctx, iplist.SourceConfig{ + Source: "allowed.csv", + CacheDir: "/var/cache/app/iplist", + // HTTPClient: nil, // uses golib's internal client +}) +if err != nil { + return err +} +defer whitelist.Close() + +blacklistExtra, err := iplist.NewSource(ctx, iplist.SourceConfig{ + Source: "blocked-ips-extra.tsv", + CacheDir: "/var/cache/app/iplist", + Optional: true, +}) +if err != nil { + return err +} +defer blacklistExtra.Close() + +gitBlacklist, err := ippolicy.NewPrefixSet( + ctx, + "https://github.com/bitwire-it/ipblocklist.git", + "/var/cache/app/ipblocklist", + []string{"ipblocklist.txt"}, + ippolicy.DefaultPrefixSetRefreshInterval, +) +if err != nil { + return err +} +defer gitBlacklist.Close() + +policy := ippolicy.New(ctx, ippolicy.Config{ + Whitelist: whitelist, + Blacklist: gitBlacklist, + BlacklistExtra: blacklistExtra, + OnRefresh: func(e ippolicy.RefreshEvent) { + slog.Info("ippolicy refresh", "kind", e.Kind, "err", e.Err) + }, +}) +defer policy.Close() + +switch policy.Evaluate(addr) { +case ippolicy.Blacklisted: + // reject +case ippolicy.Whitelisted, ippolicy.Unlisted: + // continue +} +``` + +## Behavior + +Each source refreshes independently and keeps its last valid data on failure. +Whitelist matches take precedence over blacklist matches. The policy does not +log; use `OnRefresh` for application logging and metrics. + +A nil `Whitelist` produces a policy where every address evaluates to +`Unlisted`. This intentionally drops the blacklists as well: without a +whitelist the policy cannot distinguish "allowed" from "unknown", so blocking +would be meaningless. The caller receives a `RefreshFallback` event so it can +log or alert. diff --git a/net/ipgate/domains.go b/net/ippolicy/domains.go similarity index 84% rename from net/ipgate/domains.go rename to net/ippolicy/domains.go index 865b3e7..0cc981e 100644 --- a/net/ipgate/domains.go +++ b/net/ippolicy/domains.go @@ -1,7 +1,8 @@ -package ipgate +package ippolicy import ( "context" + "net" "net/netip" "sync" "sync/atomic" @@ -34,6 +35,29 @@ func EmptyDomainSet() *DomainSet { return ds } +func NewDomainSetFromEntries(ctx context.Context, entries []string, interval time.Duration) *DomainSet { + static, domains := splitEntries(entries) + return NewDomainSet(ctx, static, domains, interval) +} + +func (ds *DomainSet) ReplaceEntries(entries []string) { + static, domains := splitEntries(entries) + ds.Replace(static, domains) +} + +func splitEntries(entries []string) (static, domains []string) { + for _, entry := range entries { + if net.ParseIP(entry) != nil { + static = append(static, entry) + } else if _, _, err := net.ParseCIDR(entry); err == nil { + static = append(static, entry) + } else { + domains = append(domains, entry) + } + } + return static, domains +} + // NewDomainSet creates a DomainSet from pre-parsed inputs. // staticPrefixes is a list of CIDRs or bare IPs. // domains is a list of hostnames to resolve periodically. diff --git a/net/ipgate/format.go b/net/ippolicy/format.go similarity index 80% rename from net/ipgate/format.go rename to net/ippolicy/format.go index 7fc1e4a..63efafc 100644 --- a/net/ipgate/format.go +++ b/net/ippolicy/format.go @@ -1,4 +1,4 @@ -package ipgate +package ippolicy import ( "log/slog" @@ -6,7 +6,7 @@ import ( "strings" ) -func log() *slog.Logger { return slog.Default().WithGroup("ipgate") } +func log() *slog.Logger { return slog.Default().WithGroup("ippolicy") } func commaify(n int) string { s := strconv.Itoa(n) diff --git a/net/ipgate/go.mod b/net/ippolicy/go.mod similarity index 57% rename from net/ipgate/go.mod rename to net/ippolicy/go.mod index f1b5c54..0faacac 100644 --- a/net/ipgate/go.mod +++ b/net/ippolicy/go.mod @@ -1,15 +1,19 @@ -module github.com/therootcompany/golib/net/ipgate +module github.com/therootcompany/golib/net/ippolicy -go 1.26.0 +go 1.26.5 require ( github.com/therootcompany/golib/net/dnsresolver v0.5.0 github.com/therootcompany/golib/net/gitshallow v0.9.1 github.com/therootcompany/golib/net/ipcohort v0.9.0 + github.com/therootcompany/golib/net/iplist v0.8.5 ) require ( github.com/miekg/dns v1.1.69 // indirect + github.com/therootcompany/golib/https v0.9.0 // indirect + github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect + github.com/therootcompany/golib/net/httpcache v0.5.1 // indirect golang.org/x/mod v0.30.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sync v0.20.0 // indirect diff --git a/net/ipgate/go.sum b/net/ippolicy/go.sum similarity index 67% rename from net/ipgate/go.sum rename to net/ippolicy/go.sum index 2cd8c06..7b365a4 100644 --- a/net/ipgate/go.sum +++ b/net/ippolicy/go.sum @@ -2,12 +2,20 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc= github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g= +github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= +github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o= github.com/therootcompany/golib/net/dnsresolver v0.5.0 h1:JTHvRVjc/cCL6SpWFo6btvv0lllW2wTYjRXLE8cOdMw= github.com/therootcompany/golib/net/dnsresolver v0.5.0/go.mod h1:tOV99lX6ghpWPksxSFAt4CDe5XLefFzIGf2qXsZJap4= github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q= github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc= +github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= +github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E= github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys= +github.com/therootcompany/golib/net/iplist v0.8.5 h1:swn/hcCt6fAG3tYApmxh1OlDak2PjQdczldZWaH1Eus= +github.com/therootcompany/golib/net/iplist v0.8.5/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= diff --git a/net/ipgate/ipprefix.go b/net/ippolicy/ipprefix.go similarity index 97% rename from net/ipgate/ipprefix.go rename to net/ippolicy/ipprefix.go index 53923e3..cbd2cb4 100644 --- a/net/ipgate/ipprefix.go +++ b/net/ippolicy/ipprefix.go @@ -1,4 +1,4 @@ -package ipgate +package ippolicy import ( "context" @@ -30,7 +30,7 @@ func EmptyPrefixSet() *PrefixSet { func NewPrefixSet(ctx context.Context, repoURL, dataPath string, files []string, interval time.Duration) (*PrefixSet, error) { if err := os.MkdirAll(dataPath, 0o755); err != nil { - return nil, fmt.Errorf("ipgate: create data dir: %w", err) + return nil, fmt.Errorf("ippolicy: create data dir: %w", err) } if interval <= 0 { diff --git a/net/ipgate/ipprefix_test.go b/net/ippolicy/ipprefix_test.go similarity index 98% rename from net/ipgate/ipprefix_test.go rename to net/ippolicy/ipprefix_test.go index e22cf55..b62f980 100644 --- a/net/ipgate/ipprefix_test.go +++ b/net/ippolicy/ipprefix_test.go @@ -1,4 +1,4 @@ -package ipgate +package ippolicy import ( "net/netip" diff --git a/net/ippolicy/policy.go b/net/ippolicy/policy.go new file mode 100644 index 0000000..6fcc5f3 --- /dev/null +++ b/net/ippolicy/policy.go @@ -0,0 +1,156 @@ +// Package ippolicy composes independently refreshed whitelist and blacklist +// sources into one IP decision policy. +package ippolicy + +import ( + "context" + "errors" + "fmt" + "net/netip" + + "github.com/therootcompany/golib/net/iplist" +) + +type Config struct { + Whitelist *iplist.Source + Blacklist *PrefixSet + BlacklistExtra *iplist.Source + OnRefresh func(RefreshEvent) +} + +// Decision is the mutually exclusive result of evaluating an address. +type Decision uint8 + +const ( + Blacklisted Decision = iota + Unlisted + Whitelisted +) + +func (d Decision) String() string { + switch d { + case Blacklisted: + return "blacklisted" + case Unlisted: + return "unlisted" + case Whitelisted: + return "whitelisted" + default: + return fmt.Sprintf("decision(%d)", d) + } +} + +type RefreshKind uint8 + +const ( + RefreshSuccess RefreshKind = iota + RefreshFailure + RefreshFallback +) + +func (k RefreshKind) String() string { + switch k { + case RefreshSuccess: + return "success" + case RefreshFailure: + return "failure" + case RefreshFallback: + return "fallback" + default: + return fmt.Sprintf("refresh(%d)", k) + } +} + +type RefreshEvent struct { + Kind RefreshKind + Err error +} + +type Policy struct { + onRefresh func(RefreshEvent) + whitelist *DomainSet + extra *DomainSet + git *PrefixSet +} + +// New creates a Policy from the given sources. The caller is responsible for +// constructing and later closing the *iplist.Source values; New does not take +// ownership of them. +// +// A nil Whitelist is treated as an empty whitelist: every address evaluates to +// Unlisted. This also intentionally drops the blacklists, because a policy +// without a whitelist provides no meaningful allow-list semantics — blocking +// would reject everything not on the blacklist rather than only what is on it. +// The caller receives a RefreshFallback event so it can log or alert. +func New(ctx context.Context, config Config) *Policy { + p := &Policy{onRefresh: config.OnRefresh} + if config.Whitelist == nil { + p.whitelist = EmptyDomainSet() + p.emit(RefreshFallback, errors.New("whitelist source is required")) + return p + } + + p.whitelist = NewDomainSetFromEntries(ctx, config.Whitelist.Entries(), DefaultDomainSetRefreshInterval) + config.Whitelist.Subscribe(func(event iplist.SourceEvent) { + if event.Err != nil { + p.emit(RefreshFailure, event.Err) + return + } + p.whitelist.ReplaceEntries(config.Whitelist.Entries()) + p.emit(RefreshSuccess, nil) + }) + + if config.BlacklistExtra != nil { + p.extra = NewDomainSetFromEntries(ctx, config.BlacklistExtra.Entries(), DefaultDomainSetRefreshInterval) + config.BlacklistExtra.Subscribe(func(event iplist.SourceEvent) { + if event.Err != nil { + p.emit(RefreshFailure, event.Err) + return + } + p.extra.ReplaceEntries(config.BlacklistExtra.Entries()) + p.emit(RefreshSuccess, nil) + }) + } else { + p.extra = EmptyDomainSet() + } + p.git = config.Blacklist + p.emit(RefreshSuccess, nil) + return p +} + +func (p *Policy) Evaluate(addr netip.Addr) Decision { + if p.whitelist == nil { + return Unlisted + } + if p.whitelist.Contains(addr) { + return Whitelisted + } + if (p.extra != nil && p.extra.Contains(addr)) || + (p.git != nil && p.git.Contains(addr)) { + return Blacklisted + } + return Unlisted +} + +func (p *Policy) Close() error { + if p == nil { + return nil + } + if p.whitelist != nil { + _ = p.whitelist.Close() + } + if p.extra != nil { + _ = p.extra.Close() + } + if p.git != nil { + _ = p.git.Close() + } + return nil +} + +func (p *Policy) emit(kind RefreshKind, err error) { + if p.onRefresh == nil { + return + } + p.onRefresh(RefreshEvent{Kind: kind, Err: err}) +} diff --git a/net/ippolicy/policy_test.go b/net/ippolicy/policy_test.go new file mode 100644 index 0000000..3598df9 --- /dev/null +++ b/net/ippolicy/policy_test.go @@ -0,0 +1,94 @@ +package ippolicy + +import ( + "context" + "net/netip" + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/iplist" +) + +func TestEnumStrings(t *testing.T) { + for _, test := range []struct { + value Decision + want string + }{ + {Blacklisted, "blacklisted"}, + {Unlisted, "unlisted"}, + {Whitelisted, "whitelisted"}, + {Decision(99), "decision(99)"}, + } { + if got := test.value.String(); got != test.want { + t.Fatalf("Decision(%d).String() = %q, want %q", test.value, got, test.want) + } + } + for _, test := range []struct { + value RefreshKind + want string + }{ + {RefreshSuccess, "success"}, + {RefreshFailure, "failure"}, + {RefreshFallback, "fallback"}, + {RefreshKind(99), "refresh(99)"}, + } { + if got := test.value.String(); got != test.want { + t.Fatalf("RefreshKind(%d).String() = %q, want %q", test.value, got, test.want) + } + } +} + +func TestPolicyKeepsSnapshotAfterFailedRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "allowed.tsv") + if err := os.WriteFile(path, []byte("192.0.2.1\n"), 0o600); err != nil { + t.Fatal(err) + } + whitelist, err := iplist.NewSource(t.Context(), iplist.SourceConfig{Source: path}) + if err != nil { + t.Fatal(err) + } + p := New(t.Context(), Config{Whitelist: whitelist}) + defer func() { _ = p.Close() }() + + if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted { + t.Fatal("initial whitelist did not match") + } + if err := os.WriteFile(path, []byte("invalid entry\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := whitelist.Refresh(); err == nil { + t.Fatal("invalid refresh unexpectedly succeeded") + } + if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted { + t.Fatal("failed refresh replaced valid snapshot") + } +} + +func TestPolicyWhitelistPrecedesExtraBlacklist(t *testing.T) { + dir := t.TempDir() + white := filepath.Join(dir, "white.tsv") + black := filepath.Join(dir, "black.tsv") + if err := os.WriteFile(white, []byte("192.0.2.1\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(black, []byte("192.0.2.0/24\n"), 0o600); err != nil { + t.Fatal(err) + } + whitelist, err := iplist.NewSource(context.Background(), iplist.SourceConfig{Source: white}) + if err != nil { + t.Fatal(err) + } + extra, err := iplist.NewSource(context.Background(), iplist.SourceConfig{Source: black}) + if err != nil { + t.Fatal(err) + } + p := New(context.Background(), Config{Whitelist: whitelist, BlacklistExtra: extra}) + defer func() { _ = p.Close() }() + if p.Evaluate(netip.MustParseAddr("192.0.2.1")) != Whitelisted { + t.Fatal("whitelist did not take precedence") + } + if p.Evaluate(netip.MustParseAddr("192.0.2.2")) != Blacklisted { + t.Fatal("extra blacklist did not block") + } +}