diff --git a/net/ippolicy/go.mod b/net/ippolicy/go.mod new file mode 100644 index 0000000..cad3f28 --- /dev/null +++ b/net/ippolicy/go.mod @@ -0,0 +1,23 @@ +module github.com/therootcompany/golib/net/ippolicy + +go 1.26.5 + +require ( + github.com/therootcompany/golib/net/ipgate v0.5.3 + github.com/therootcompany/golib/net/iplist v0.7.0 +) + +require ( + github.com/miekg/dns v1.1.69 // indirect + github.com/therootcompany/golib/https v0.9.0 // indirect + github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 // indirect + github.com/therootcompany/golib/net/dnsresolver v0.5.0 // indirect + github.com/therootcompany/golib/net/gitshallow v0.9.1 // indirect + github.com/therootcompany/golib/net/httpcache v0.5.1 // indirect + github.com/therootcompany/golib/net/ipcohort v0.9.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/tools v0.39.0 // indirect +) diff --git a/net/ippolicy/go.sum b/net/ippolicy/go.sum new file mode 100644 index 0000000..13023d5 --- /dev/null +++ b/net/ippolicy/go.sum @@ -0,0 +1,30 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc= +github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g= +github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= +github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o= +github.com/therootcompany/golib/net/dnsresolver v0.5.0 h1:JTHvRVjc/cCL6SpWFo6btvv0lllW2wTYjRXLE8cOdMw= +github.com/therootcompany/golib/net/dnsresolver v0.5.0/go.mod h1:tOV99lX6ghpWPksxSFAt4CDe5XLefFzIGf2qXsZJap4= +github.com/therootcompany/golib/net/gitshallow v0.9.1 h1:VI/Q4jViKvVuJ+lzZrD7suhSXLfNNhUcd4ZJrkpTP/Q= +github.com/therootcompany/golib/net/gitshallow v0.9.1/go.mod h1:S/3OK7wMOqJJjP79U9Qw/l3jjEyrb/zsjJ8HAPAX/Fc= +github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= +github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= +github.com/therootcompany/golib/net/ipcohort v0.9.0 h1:j3AaoW2u4XQJz1ya6fixv3pb3KgxIpZV8t0082/kS3E= +github.com/therootcompany/golib/net/ipcohort v0.9.0/go.mod h1:Ljcnt2xOryVDGIVV5Kzt5hj2LhW4lBAWo5OeeVYU9ys= +github.com/therootcompany/golib/net/ipgate v0.5.3 h1:4alg4iIm719ckz1xt9MZ5MJSUG63DcodJBl8AEJZchI= +github.com/therootcompany/golib/net/ipgate v0.5.3/go.mod h1:8JGSNE0c36HG7iUMTkJjkXCE0GKu8LL0VhIDDTOjdu0= +github.com/therootcompany/golib/net/iplist v0.7.0 h1:elF67cjxkxTMVlWHujLRpS+R6vxcg73dtIkDdhxlJXI= +github.com/therootcompany/golib/net/iplist v0.7.0/go.mod h1:vK5Na53RcLdtp/IPM/NX9BudyKhXp+8T+elmWBVWzeU= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= diff --git a/net/ippolicy/policy.go b/net/ippolicy/policy.go new file mode 100644 index 0000000..2546338 --- /dev/null +++ b/net/ippolicy/policy.go @@ -0,0 +1,265 @@ +// Package ippolicy composes whitelist and blacklist IP sources into one +// atomically refreshed policy. +package ippolicy + +import ( + "context" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "sync/atomic" + "time" + + "github.com/therootcompany/golib/net/ipgate" + "github.com/therootcompany/golib/net/iplist" +) + +const DefaultRefreshInterval = time.Hour + 57*time.Minute + 13*time.Second + +type Config struct { + HTTPClient *http.Client + Whitelist string + BlacklistExtra string + BlacklistRepo string + BlacklistRepoDir string + BlacklistRepoFiles []string + CacheDir string + RefreshInterval time.Duration + Resolver *net.Resolver + OnRefresh func(RefreshEvent) +} + +type Decision struct { + Whitelisted bool + Blacklisted bool + Blocked bool + GitMatched bool + ExtraMatched bool +} + +type RefreshKind string + +const ( + RefreshSuccess RefreshKind = "success" + RefreshFailure RefreshKind = "failure" + RefreshFallback RefreshKind = "fallback" +) + +type RefreshEvent struct { + Kind RefreshKind + Generation uint64 + Err error +} + +type snapshot struct { + whitelist *ipgate.DomainSet + extra *ipgate.DomainSet + git *ipgate.PrefixSet + generation uint64 +} + +type Policy struct { + ctx context.Context + cancel context.CancelFunc + config Config + current atomic.Pointer[snapshot] + sequence atomic.Uint64 +} + +func New(ctx context.Context, config Config) (*Policy, error) { + if config.RefreshInterval <= 0 { + config.RefreshInterval = DefaultRefreshInterval + } + if config.Resolver == nil { + config.Resolver = net.DefaultResolver + } + policyCtx, cancel := context.WithCancel(ctx) + p := &Policy{ctx: policyCtx, cancel: cancel, config: config} + if err := p.refresh(true); err != nil { + cancel() + return nil, err + } + go p.refreshLoop() + return p, nil +} + +func (p *Policy) Evaluate(addr netip.Addr) Decision { + current := p.current.Load() + if current == nil { + return Decision{} + } + whitelisted := current.whitelist.Contains(addr) + extraMatched := current.extra != nil && current.extra.Contains(addr) + gitMatched := current.git != nil && current.git.Contains(addr) + blacklisted := extraMatched || gitMatched + return Decision{ + Whitelisted: whitelisted, + Blacklisted: blacklisted, + Blocked: blacklisted && !whitelisted, + GitMatched: gitMatched, + ExtraMatched: extraMatched, + } +} + +func (p *Policy) Close() error { + if p != nil && p.cancel != nil { + p.cancel() + } + return nil +} + +func (p *Policy) refresh(initial bool) error { + whitelist, err := p.load(p.config.Whitelist, false) + if err != nil { + if initial { + p.publish(&snapshot{whitelist: ipgate.EmptyDomainSet()}) + p.emit(RefreshFallback, err) + return nil + } + p.emit(RefreshFailure, err) + return err + } + whitelistSet, err := p.build(whitelist) + if err != nil { + return p.failed(initial, err) + } + + extra := []string(nil) + var fallbackErr error + if p.config.BlacklistExtra != "" { + extra, err = p.load(p.config.BlacklistExtra, true) + if err != nil { + if !initial { + return p.failed(false, err) + } + fallbackErr = err + } + } + extraSet := ipgate.EmptyDomainSet() + if fallbackErr == nil { + extraSet, err = p.build(extra) + if err != nil { + if !initial { + return p.failed(false, err) + } + fallbackErr = err + } + } + + var git *ipgate.PrefixSet + if p.config.BlacklistRepo != "" && p.config.BlacklistRepo != "none" { + files := p.config.BlacklistRepoFiles + if len(files) == 0 { + files = []string{"tables/inbound/single_ips.txt", "tables/inbound/networks.txt"} + } + git, err = ipgate.NewPrefixSet(p.ctx, p.config.BlacklistRepo, p.config.BlacklistRepoDir, files) + if err != nil { + if !initial { + return p.failed(false, err) + } + fallbackErr = err + } + } + p.publish(&snapshot{whitelist: whitelistSet, extra: extraSet, git: git}) + if fallbackErr != nil { + p.emit(RefreshFallback, fallbackErr) + } else { + p.emit(RefreshSuccess, nil) + } + return nil +} + +func (p *Policy) failed(initial bool, err error) error { + if initial { + p.publish(&snapshot{whitelist: ipgate.EmptyDomainSet()}) + p.emit(RefreshFallback, err) + return nil + } + p.emit(RefreshFailure, err) + return err +} + +func (p *Policy) load(source string, optional bool) ([]string, error) { + if source == "" { + if optional { + return nil, nil + } + return nil, fmt.Errorf("whitelist source is required") + } + if optional { + parsed, parseErr := url.Parse(source) + isURL := parseErr == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") + if !isURL { + if _, err := os.Stat(source); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + } + } + entries, err := iplist.Load(p.ctx, source, p.config.CacheDir, p.config.HTTPClient) + if err != nil { + return nil, err + } + return entries, nil +} + +func (p *Policy) build(entries []string) (*ipgate.DomainSet, error) { + prefixes := make([]string, 0, len(entries)) + domains := make([]string, 0, len(entries)) + for _, entry := range entries { + if net.ParseIP(entry) != nil { + prefixes = append(prefixes, entry) + } else if _, _, err := net.ParseCIDR(entry); err == nil { + prefixes = append(prefixes, entry) + } else { + domains = append(domains, entry) + } + } + for _, domain := range domains { + ips, err := p.config.Resolver.LookupNetIP(p.ctx, "ip", domain) + if err != nil { + return nil, fmt.Errorf("resolve domain %q: %w", domain, err) + } + if len(ips) == 0 { + return nil, fmt.Errorf("resolve domain %q: no addresses", domain) + } + for _, ip := range ips { + prefixes = append(prefixes, ip.String()) + } + } + return ipgate.NewDomainSet(p.ctx, prefixes, nil), nil +} + +func (p *Policy) publish(next *snapshot) { + next.generation = p.sequence.Add(1) + p.current.Store(next) +} + +func (p *Policy) emit(kind RefreshKind, err error) { + if p.config.OnRefresh == nil { + return + } + current := p.current.Load() + var generation uint64 + if current != nil { + generation = current.generation + } + p.config.OnRefresh(RefreshEvent{Kind: kind, Generation: generation, Err: err}) +} + +func (p *Policy) refreshLoop() { + ticker := time.NewTicker(p.config.RefreshInterval) + defer ticker.Stop() + for { + select { + case <-p.ctx.Done(): + return + case <-ticker.C: + _ = p.refresh(false) + } + } +} diff --git a/net/ippolicy/policy_test.go b/net/ippolicy/policy_test.go new file mode 100644 index 0000000..51d933e --- /dev/null +++ b/net/ippolicy/policy_test.go @@ -0,0 +1,57 @@ +package ippolicy + +import ( + "context" + "net/netip" + "os" + "path/filepath" + "testing" +) + +func TestPolicyKeepsSnapshotAfterFailedRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "allowed.tsv") + if err := os.WriteFile(path, []byte("192.0.2.1\n"), 0o600); err != nil { + t.Fatal(err) + } + p, err := New(t.Context(), Config{Whitelist: path}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = p.Close() }() + + if !p.Evaluate(netip.MustParseAddr("192.0.2.1")).Whitelisted { + t.Fatal("initial whitelist did not match") + } + if err := os.WriteFile(path, []byte("invalid entry\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := p.refresh(false); err == nil { + t.Fatal("invalid refresh unexpectedly succeeded") + } + if !p.Evaluate(netip.MustParseAddr("192.0.2.1")).Whitelisted { + t.Fatal("failed refresh replaced valid snapshot") + } +} + +func TestPolicyWhitelistPrecedesExtraBlacklist(t *testing.T) { + dir := t.TempDir() + white := filepath.Join(dir, "white.tsv") + black := filepath.Join(dir, "black.tsv") + if err := os.WriteFile(white, []byte("192.0.2.1\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(black, []byte("192.0.2.0/24\n"), 0o600); err != nil { + t.Fatal(err) + } + p, err := New(context.Background(), Config{Whitelist: white, BlacklistExtra: black}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = p.Close() }() + if p.Evaluate(netip.MustParseAddr("192.0.2.1")).Blocked { + t.Fatal("whitelist did not take precedence") + } + if !p.Evaluate(netip.MustParseAddr("192.0.2.2")).Blocked { + t.Fatal("extra blacklist did not block") + } +}