From e007da3e44d5dfbb9d8d3ff3710bc6496127880f Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Fri, 22 May 2026 11:16:33 -0400 Subject: [PATCH] feat(net/ipgate): IP allowlist/blocklist via PrefixSet and DomainSet PrefixSet: git-backed CIDR files via gitshallow, refreshed every 47 min. DomainSet: CSV of static IPs/prefixes + hostnames resolved every 5 min; retains stale IPs on resolve failure. Both use atomic-swapped ipcohort.Cohort for lock-free reads. replace directives in go.mod point to local siblings; remove after tagging. --- net/ipgate/domains.go | 200 +++++++++++++++++++++++++++++++++++++++++ net/ipgate/format.go | 31 +++++++ net/ipgate/go.mod | 24 +++++ net/ipgate/go.sum | 14 +++ net/ipgate/ipprefix.go | 94 +++++++++++++++++++ 5 files changed, 363 insertions(+) create mode 100644 net/ipgate/domains.go create mode 100644 net/ipgate/format.go create mode 100644 net/ipgate/go.mod create mode 100644 net/ipgate/go.sum create mode 100644 net/ipgate/ipprefix.go diff --git a/net/ipgate/domains.go b/net/ipgate/domains.go new file mode 100644 index 0000000..726c91f --- /dev/null +++ b/net/ipgate/domains.go @@ -0,0 +1,200 @@ +package ipgate + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "net/netip" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/therootcompany/golib/net/dnsresolver" + "github.com/therootcompany/golib/net/ipcohort" +) + +const domainSetRefreshInterval = 5 * time.Minute + +type domainEntry struct { + Raw string + Label string +} + +type DomainSet struct { + staticPrefixes []string + domains []domainEntry + resolved atomic.Pointer[map[string][]string] + cohort atomic.Pointer[ipcohort.Cohort] +} + +func EmptyDomainSet() *DomainSet { + ds := &DomainSet{} + ds.cohort.Store(&ipcohort.Cohort{}) + return ds +} + +func NewDomainSet(ctx context.Context, csvPath string) (*DomainSet, error) { + if csvPath == "" { + return nil, nil + } + + staticPrefixes, domains, err := parseDomainSetCSV(csvPath) + if err != nil { + return nil, fmt.Errorf("domain set %q: %w", csvPath, err) + } + + ds := &DomainSet{ + staticPrefixes: staticPrefixes, + domains: domains, + } + + emptyResolved := make(map[string][]string) + ds.resolved.Store(&emptyResolved) + + ds.rebuildCohort() + + log().Info("domain set loaded", "static", commaify(len(staticPrefixes)), "domains", commaify(len(domains)), "path", csvPath) + + go ds.refreshLoop(ctx) + + return ds, nil +} + +func (ds *DomainSet) Contains(addr netip.Addr) bool { + return ds.cohort.Load().ContainsAddr(addr) +} + +func (ds *DomainSet) refreshLoop(ctx context.Context) { + ds.resolveDomains(ctx) + ds.rebuildCohort() + + ticker := time.NewTicker(domainSetRefreshInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ds.resolveDomains(ctx) + ds.rebuildCohort() + } + } +} + +func (ds *DomainSet) resolveDomains(ctx context.Context) { + if len(ds.domains) == 0 { + return + } + + prev := *ds.resolved.Load() + next := make(map[string][]string, len(ds.domains)) + + resolver := dnsresolver.New() + for _, entry := range ds.domains { + ips, _, err := resolver.LookupIP(ctx, entry.Raw) + + if err != nil || len(ips) == 0 { + if old, ok := prev[entry.Raw]; ok { + next[entry.Raw] = old + log().Warn("resolve failed, keeping prior IPs", "domain", entry.Raw, "count", len(old), "err", err) + } else { + log().Warn("resolve failed, no prior data", "domain", entry.Raw, "err", err) + } + continue + } + + addrs := make([]string, 0, len(ips)) + for _, ip := range ips { + addrs = append(addrs, ip.String()) + } + next[entry.Raw] = addrs + } + + ds.resolved.Store(&next) +} + +func (ds *DomainSet) rebuildCohort() { + var all []string + all = append(all, ds.staticPrefixes...) + + resolved := *ds.resolved.Load() + for _, addrs := range resolved { + for _, addr := range addrs { + all = append(all, addr+"/32") + } + } + + cohort, err := ipcohort.Parse(all) + if err != nil { + log().Warn("domain set rebuild failed", "err", err) + } + if cohort != nil { + ds.cohort.Store(cohort) + } +} + +func parseDomainSetCSV(path string) (staticPrefixes []string, domains []domainEntry, err error) { + file, err := os.Open(path) + if err != nil { + return nil, nil, err + } + defer func() { _ = file.Close() }() + + r := csv.NewReader(file) + r.FieldsPerRecord = -1 + r.Comment = '#' + if strings.HasSuffix(path, ".tsv") { + r.Comma = '\t' + } + + firstRow := true + for { + record, readErr := r.Read() + if readErr == io.EOF { + break + } + if readErr != nil { + return nil, nil, fmt.Errorf("csv read: %w", readErr) + } + if len(record) == 0 { + continue + } + + raw := strings.TrimSpace(record[0]) + + if firstRow { + firstRow = false + if _, err := netip.ParseAddr(raw); err != nil { + if _, err := netip.ParsePrefix(raw); err != nil { + if !strings.Contains(raw, ".") { + continue + } + } + } + } + if raw == "" { + continue + } + + var label string + if len(record) > 1 { + label = strings.TrimSpace(record[1]) + } + + if _, err := netip.ParseAddr(raw); err == nil { + staticPrefixes = append(staticPrefixes, raw+"/32") + continue + } + if _, err := netip.ParsePrefix(raw); err == nil { + staticPrefixes = append(staticPrefixes, raw) + continue + } + + domains = append(domains, domainEntry{Raw: raw, Label: label}) + } + + return staticPrefixes, domains, nil +} diff --git a/net/ipgate/format.go b/net/ipgate/format.go new file mode 100644 index 0000000..085194b --- /dev/null +++ b/net/ipgate/format.go @@ -0,0 +1,31 @@ +package ipgate + +import ( + "log/slog" + "strconv" + "strings" +) + +func log() *slog.Logger { return slog.Default().WithGroup("ipgate") } + +func commaify(n int) string { + s := strconv.Itoa(n) + if n < 0 { + return "-" + commaify(-n) + } + if len(s) <= 3 { + return s + } + + rem := len(s) % 3 + if rem == 0 { + rem = 3 + } + + var result strings.Builder + result.WriteString(s[:rem]) + for i := rem; i < len(s); i += 3 { + result.WriteString("," + s[i:i+3]) + } + return result.String() +} diff --git a/net/ipgate/go.mod b/net/ipgate/go.mod new file mode 100644 index 0000000..1e43853 --- /dev/null +++ b/net/ipgate/go.mod @@ -0,0 +1,24 @@ +module github.com/therootcompany/golib/net/ipgate + +go 1.26.0 + +require ( + github.com/therootcompany/golib/net/dnsresolver v0.0.0 + github.com/therootcompany/golib/net/gitshallow v0.0.0 + github.com/therootcompany/golib/net/ipcohort v0.0.0 +) + +require ( + github.com/miekg/dns v1.1.69 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/tools v0.39.0 // indirect +) + +replace ( + github.com/therootcompany/golib/net/dnsresolver => ../dnsresolver + github.com/therootcompany/golib/net/gitshallow => ../gitshallow + github.com/therootcompany/golib/net/ipcohort => ../ipcohort +) diff --git a/net/ipgate/go.sum b/net/ipgate/go.sum new file mode 100644 index 0000000..05c3ae5 --- /dev/null +++ b/net/ipgate/go.sum @@ -0,0 +1,14 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc= +github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= diff --git a/net/ipgate/ipprefix.go b/net/ipgate/ipprefix.go new file mode 100644 index 0000000..a6a6f31 --- /dev/null +++ b/net/ipgate/ipprefix.go @@ -0,0 +1,94 @@ +package ipgate + +import ( + "context" + "fmt" + "net/netip" + "os" + "sync/atomic" + "time" + + "github.com/therootcompany/golib/net/gitshallow" + "github.com/therootcompany/golib/net/ipcohort" +) + +const prefixSetRefreshInterval = 47 * time.Minute + +type PrefixSet struct { + repo *gitshallow.Repo + files []string + cohort atomic.Pointer[ipcohort.Cohort] +} + +func EmptyPrefixSet() *PrefixSet { + ps := &PrefixSet{} + ps.cohort.Store(&ipcohort.Cohort{}) + return ps +} + +func NewPrefixSet(ctx context.Context, repoURL, dataPath string, files []string) (*PrefixSet, error) { + if err := os.MkdirAll(dataPath, 0o755); err != nil { + return nil, fmt.Errorf("ipgate: create data dir: %w", err) + } + + repo := gitshallow.New(repoURL, dataPath, 0, "") + + ps := &PrefixSet{ + repo: repo, + files: files, + } + ps.cohort.Store(&ipcohort.Cohort{}) + + go ps.refreshLoop(ctx) + + return ps, nil +} + +func (ps *PrefixSet) Contains(addr netip.Addr) bool { + return ps.cohort.Load().ContainsAddr(addr) +} + +func (ps *PrefixSet) reload(ctx context.Context) error { + updated, err := ps.repo.Fetch(ctx) + if err != nil { + return err + } + if !updated && ps.cohort.Load().Size() > 0 { + return nil + } + + paths := make([]string, len(ps.files)) + for i, f := range ps.files { + paths[i] = ps.repo.FilePath(f) + } + + cohort, err := ipcohort.LoadFiles(paths...) + if err != nil { + return fmt.Errorf("load files: %w", err) + } + + ps.cohort.Store(cohort) + + log().Info("prefix set loaded", "entries", commaify(cohort.Size())) + return nil +} + +func (ps *PrefixSet) refreshLoop(ctx context.Context) { + if err := ps.reload(ctx); err != nil { + log().Warn("prefix set initial load (will retry)", "err", err) + } + + ticker := time.NewTicker(prefixSetRefreshInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := ps.reload(ctx); err != nil { + log().Warn("prefix set reload failed", "err", err) + } + } + } +}