feat(net/ipcohort): IP block/allow cohort filter

New package for matching IPs against cohort lists (single IPs and
networks, in separate files or combined) with inbound/outbound/whitelist
semantics. Supports the bitwire-it/ipblocklist data shape out of the box
and is meant to be paired with gitshallow or httpcache for periodic
refresh. Includes README, ipcohort-contains CLI, and tests.
This commit is contained in:
AJ ONeal 2026-04-30 16:49:01 -06:00
parent c2f5dbeeca
commit 846cfe0070
No known key found for this signature in database
5 changed files with 675 additions and 0 deletions

122
net/ipcohort/README.md Normal file
View File

@ -0,0 +1,122 @@
# [ipcohort](https://github.com/therootcompany/golib/tree/main/net/ipcohort)
A memory-efficient, fast IP cohort checker for blacklists, whitelists, and ad cohorts.
- 6 bytes per IP address (5 + 1 for alignment)
- binary search for /32 hosts, linear scan for CIDR ranges
- immutable cohorts — callers swap via `atomic.Pointer` for lock-free reads
## Example
Check if an IP address belongs to a cohort (such as a blacklist):
```go
cohort, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
if cohort.Contains("92.255.85.72") {
fmt.Println("BLOCKED")
os.Exit(1)
}
fmt.Println("allowed")
```
## Update the list periodically: git (shallow)
```go
import (
"sync/atomic"
"github.com/therootcompany/golib/net/gitshallow"
"github.com/therootcompany/golib/net/ipcohort"
)
var cohort atomic.Pointer[ipcohort.Cohort]
repo := gitshallow.New("https://github.com/bitwire-it/ipblocklist.git", "/srv/data/ipblocklist", 1, "")
// Init: clone if missing, pull, load.
if _, err := repo.Init(false); err != nil {
log.Fatalf("init: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: pull and reload when HEAD changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := repo.Sync(false)
if err != nil {
log.Printf("sync: %v", err)
continue
}
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()
```
## Update the list periodically: HTTP (cache)
```go
import (
"sync/atomic"
"github.com/therootcompany/golib/net/httpcache"
"github.com/therootcompany/golib/net/ipcohort"
)
const listURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
var cohort atomic.Pointer[ipcohort.Cohort]
cacher := httpcache.New(listURL, "/srv/data/inbound.txt")
// Init: fetch unconditionally, load.
if _, err := cacher.Fetch(); err != nil {
log.Fatalf("fetch: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: conditional GET, reload only when content changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := cacher.Fetch()
if err != nil {
log.Printf("fetch: %v", err)
continue
}
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()
```

View File

@ -0,0 +1,161 @@
// ipcohort-contains checks whether one or more IP addresses appear in a set
// of cohort files (plain text, one IP/CIDR per line).
//
// Usage:
//
// ipcohort-contains [flags] <file>... -- <ip>...
// ipcohort-contains [flags] --ip <ip> <file>...
// echo "<ip>" | ipcohort-contains <file>...
//
// Exit code: 0 if all queried IPs are found, 1 if any are not found, 2 on error.
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/therootcompany/golib/net/ipcohort"
)
const version = "dev"
func commafy(n int) string {
s := strconv.Itoa(n)
neg := ""
if n < 0 {
neg, s = "-", s[1:]
}
if len(s) <= 3 {
return neg + s
}
var b strings.Builder
head := len(s) % 3
if head > 0 {
b.WriteString(s[:head])
b.WriteByte(',')
}
for i := head; i < len(s); i += 3 {
b.WriteString(s[i : i+3])
if i+3 < len(s) {
b.WriteByte(',')
}
}
return neg + b.String()
}
type Config struct {
IP string
}
func main() {
cfg := Config{}
fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
fs.StringVar(&cfg.IP, "ip", "", "IP address to check (alternative to -- separator)")
fs.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <file>... -- <ip>...\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s --ip <ip> <file>...\n", os.Args[0])
fmt.Fprintf(os.Stderr, " echo <ip> | %s <file>...\n", os.Args[0])
fs.PrintDefaults()
fmt.Fprintln(os.Stderr, "Exit: 0=all found, 1=not found, 2=error")
}
if len(os.Args) > 1 {
switch os.Args[1] {
case "-V", "-version", "--version", "version":
fmt.Fprintf(os.Stdout, "ipcohort-contains %s\n", version)
os.Exit(0)
case "help", "-help", "--help":
fmt.Fprintf(os.Stdout, "ipcohort-contains %s\n\n", version)
fs.SetOutput(os.Stdout)
fs.Usage()
os.Exit(0)
}
}
if err := fs.Parse(os.Args[1:]); err != nil {
if errors.Is(err, flag.ErrHelp) {
os.Exit(0)
}
os.Exit(2)
}
args := fs.Args()
var filePaths, ips []string
switch {
case cfg.IP != "":
filePaths = args
ips = []string{cfg.IP}
default:
sep := -1
for i, a := range args {
if a == "--" {
sep = i
break
}
}
if sep >= 0 {
filePaths = args[:sep]
ips = args[sep+1:]
} else {
filePaths = args
}
}
if len(filePaths) == 0 {
fmt.Fprintln(os.Stderr, "error: at least one file path required")
fs.Usage()
os.Exit(2)
}
fmt.Fprint(os.Stderr, "Loading cohort... ")
t := time.Now()
cohort, err := ipcohort.LoadFiles(filePaths...)
if err != nil {
fmt.Fprintln(os.Stderr)
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(2)
}
fmt.Fprintf(os.Stderr, "%s (entries=%s)\n",
time.Since(t).Round(time.Millisecond),
commafy(cohort.Size()),
)
if len(ips) == 0 {
sc := bufio.NewScanner(os.Stdin)
for sc.Scan() {
if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
ips = append(ips, line)
}
}
if err := sc.Err(); err != nil {
fmt.Fprintf(os.Stderr, "error reading stdin: %v\n", err)
os.Exit(2)
}
}
if len(ips) == 0 {
fmt.Fprintln(os.Stderr, "error: no IP addresses to check")
fs.Usage()
os.Exit(2)
}
fmt.Fprintln(os.Stderr)
allFound := true
for _, ip := range ips {
if cohort.Contains(ip) {
fmt.Printf("%s\tFOUND\n", ip)
} else {
fmt.Printf("%s\tNOT FOUND\n", ip)
allFound = false
}
}
if !allFound {
os.Exit(1)
}
}

3
net/ipcohort/go.mod Normal file
View File

@ -0,0 +1,3 @@
module github.com/therootcompany/golib/net/ipcohort
go 1.26.0

223
net/ipcohort/ipcohort.go Normal file
View File

@ -0,0 +1,223 @@
package ipcohort
import (
"cmp"
"encoding/binary"
"encoding/csv"
"fmt"
"io"
"log"
"net/netip"
"os"
"slices"
"strings"
)
// IPv4Net represents a subnet or single address (/32).
// 6 bytes: networkBE uint32 + prefix uint8 + shift uint8.
type IPv4Net struct {
networkBE uint32
prefix uint8
shift uint8
}
func NewIPv4Net(ip4be uint32, prefix uint8) IPv4Net {
return IPv4Net{
networkBE: ip4be,
prefix: prefix,
shift: 32 - prefix,
}
}
func (r IPv4Net) Contains(ip uint32) bool {
mask := uint32(0xFFFFFFFF << r.shift)
return (ip & mask) == r.networkBE
}
// Cohort is an immutable, read-only set of IPv4 addresses and subnets.
// Contains is safe for concurrent use without locks.
//
// hosts holds sorted /32 addresses for O(log n) binary search.
// nets holds CIDR ranges (prefix < 32) for O(k) linear scan — typically small.
type Cohort struct {
hosts []uint32
nets []IPv4Net
}
func New() *Cohort {
return &Cohort{}
}
func sortNets(nets []IPv4Net) {
slices.SortFunc(nets, func(a, b IPv4Net) int {
return cmp.Compare(a.networkBE, b.networkBE)
})
}
// Size returns the total number of entries (hosts + nets).
func (c *Cohort) Size() int {
return len(c.hosts) + len(c.nets)
}
// Contains reports whether ipStr falls within any host or subnet in the cohort.
// Returns true on parse error (fail-closed): unparseable input is treated as
// blocked so that garbage strings never accidentally bypass a blocklist check.
// IPv6 addresses are not stored and always return false.
func (c *Cohort) Contains(ipStr string) bool {
ip, err := netip.ParseAddr(ipStr)
if err != nil {
return true // fail-closed
}
return c.ContainsAddr(ip)
}
// ContainsAddr reports whether ip falls within any host or subnet in the cohort.
// IPv6 addresses always return false (cohort is IPv4-only).
func (c *Cohort) ContainsAddr(ip netip.Addr) bool {
if !ip.Is4() {
return false
}
ip4 := ip.As4()
ipU32 := binary.BigEndian.Uint32(ip4[:])
if _, found := slices.BinarySearch(c.hosts, ipU32); found {
return true
}
for _, net := range c.nets {
if net.Contains(ipU32) {
return true
}
}
return false
}
func Parse(prefixList []string) (*Cohort, error) {
var hosts []uint32
var nets []IPv4Net
for _, raw := range prefixList {
ipv4net, err := ParseIPv4(raw)
if err != nil {
log.Printf("skipping invalid entry: %q", raw)
continue
}
if ipv4net.prefix == 32 {
hosts = append(hosts, ipv4net.networkBE)
} else {
nets = append(nets, ipv4net)
}
}
slices.Sort(hosts)
sortNets(nets)
return &Cohort{hosts: hosts, nets: nets}, nil
}
func ParseIPv4(raw string) (ipv4net IPv4Net, err error) {
var ippre netip.Prefix
var ip netip.Addr
if strings.Contains(raw, "/") {
ippre, err = netip.ParsePrefix(raw)
if err != nil {
return ipv4net, err
}
} else {
ip, err = netip.ParseAddr(raw)
if err != nil {
return ipv4net, err
}
ippre = netip.PrefixFrom(ip, 32)
}
addr := ippre.Addr()
if !addr.Is4() {
return ipv4net, fmt.Errorf("not an IPv4 address: %s", raw)
}
ip4 := addr.As4()
prefix := uint8(ippre.Bits()) // 0–32
return NewIPv4Net(
binary.BigEndian.Uint32(ip4[:]),
prefix,
), nil
}
func LoadFile(path string) (*Cohort, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("could not load %q: %v", path, err)
}
defer f.Close()
return ParseCSV(f)
}
// LoadFiles loads and merges multiple files into one Cohort.
// Useful when hosts and networks are stored in separate files.
func LoadFiles(paths ...string) (*Cohort, error) {
var hosts []uint32
var nets []IPv4Net
for _, path := range paths {
c, err := LoadFile(path)
if err != nil {
return nil, err
}
hosts = append(hosts, c.hosts...)
nets = append(nets, c.nets...)
}
slices.Sort(hosts)
sortNets(nets)
return &Cohort{hosts: hosts, nets: nets}, nil
}
func ParseCSV(f io.Reader) (*Cohort, error) {
r := csv.NewReader(f)
r.FieldsPerRecord = -1
return ReadAll(r)
}
func ReadAll(r *csv.Reader) (*Cohort, error) {
var hosts []uint32
var nets []IPv4Net
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("csv read error: %w", err)
}
if len(record) == 0 {
continue
}
raw := strings.TrimSpace(record[0])
if raw == "" || strings.HasPrefix(raw, "#") {
continue
}
ipv4net, err := ParseIPv4(raw)
if err != nil {
log.Printf("skipping invalid entry: %q", raw)
continue
}
if ipv4net.prefix == 32 {
hosts = append(hosts, ipv4net.networkBE)
} else {
nets = append(nets, ipv4net)
}
}
slices.Sort(hosts)
sortNets(nets)
return &Cohort{hosts: hosts, nets: nets}, nil
}

View File

@ -0,0 +1,166 @@
package ipcohort_test
import (
"os"
"path/filepath"
"testing"
"github.com/therootcompany/golib/net/ipcohort"
)
func TestParseIPv4(t *testing.T) {
tests := []struct {
raw string
wantErr bool
}{
{"1.2.3.4", false},
{"1.2.3.4/32", false},
{"10.0.0.0/8", false},
{"192.168.0.0/16", false},
{"0.0.0.0/0", false},
{"", true},
{"not-an-ip", true},
{"1.2.3.4/33", true},
}
for _, tt := range tests {
_, err := ipcohort.ParseIPv4(tt.raw)
if (err != nil) != tt.wantErr {
t.Errorf("ParseIPv4(%q): got err=%v, wantErr=%v", tt.raw, err, tt.wantErr)
}
}
}
func TestContains_SingleHosts(t *testing.T) {
c, err := ipcohort.Parse([]string{"1.2.3.4", "5.6.7.8", "10.0.0.1"})
if err != nil {
t.Fatal(err)
}
hits := []string{"1.2.3.4", "5.6.7.8", "10.0.0.1"}
misses := []string{"1.2.3.5", "5.6.7.7", "10.0.0.2", "0.0.0.0"}
for _, ip := range hits {
if !c.Contains(ip) {
t.Errorf("expected %s to be in cohort", ip)
}
}
for _, ip := range misses {
if c.Contains(ip) {
t.Errorf("expected %s NOT to be in cohort", ip)
}
}
}
func TestContains_CIDRRanges(t *testing.T) {
c, err := ipcohort.Parse([]string{"10.0.0.0/8", "192.168.1.0/24"})
if err != nil {
t.Fatal(err)
}
hits := []string{
"10.0.0.0", "10.0.0.1", "10.255.255.255",
"192.168.1.0", "192.168.1.1", "192.168.1.254", "192.168.1.255",
}
misses := []string{
"9.255.255.255", "11.0.0.0",
"192.168.0.255", "192.168.2.0",
}
for _, ip := range hits {
if !c.Contains(ip) {
t.Errorf("expected %s to be in cohort (CIDR)", ip)
}
}
for _, ip := range misses {
if c.Contains(ip) {
t.Errorf("expected %s NOT to be in cohort (CIDR)", ip)
}
}
}
func TestContains_Mixed(t *testing.T) {
c, err := ipcohort.Parse([]string{"1.2.3.4", "10.0.0.0/8"})
if err != nil {
t.Fatal(err)
}
if !c.Contains("1.2.3.4") {
t.Error("host miss")
}
if !c.Contains("10.5.5.5") {
t.Error("CIDR miss")
}
if c.Contains("1.2.3.5") {
t.Error("false positive for host-adjacent")
}
}
func TestContains_FailClosed(t *testing.T) {
c, _ := ipcohort.Parse([]string{"1.2.3.4"})
// Unparseable input should return true (fail-closed).
if !c.Contains("not-an-ip") {
t.Error("expected fail-closed true for unparseable IP")
}
}
func TestContains_Empty(t *testing.T) {
c, err := ipcohort.Parse(nil)
if err != nil {
t.Fatal(err)
}
if c.Contains("1.2.3.4") {
t.Error("empty cohort should not contain anything")
}
}
func TestSize(t *testing.T) {
c, _ := ipcohort.Parse([]string{"1.2.3.4", "5.6.7.8", "10.0.0.0/8"})
if got, want := c.Size(), 3; got != want {
t.Errorf("Size() = %d, want %d", got, want)
}
}
func TestLoadFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "list.txt")
content := "# comment\n1.2.3.4\n10.0.0.0/8\n\n5.6.7.8\n"
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
c, err := ipcohort.LoadFile(path)
if err != nil {
t.Fatal(err)
}
if c.Size() != 3 {
t.Errorf("Size() = %d, want 3", c.Size())
}
if !c.Contains("1.2.3.4") {
t.Error("missing 1.2.3.4")
}
if !c.Contains("10.5.5.5") {
t.Error("missing CIDR member 10.5.5.5")
}
}
func TestLoadFiles_Merge(t *testing.T) {
dir := t.TempDir()
f1 := filepath.Join(dir, "singles.txt")
f2 := filepath.Join(dir, "networks.txt")
os.WriteFile(f1, []byte("1.2.3.4\n5.6.7.8\n"), 0o644)
os.WriteFile(f2, []byte("192.168.0.0/24\n"), 0o644)
c, err := ipcohort.LoadFiles(f1, f2)
if err != nil {
t.Fatal(err)
}
if c.Size() != 3 {
t.Errorf("Size() = %d, want 3", c.Size())
}
if !c.Contains("192.168.0.100") {
t.Error("missing merged CIDR member")
}
}