diff --git a/net/ipcohort/README.md b/net/ipcohort/README.md new file mode 100644 index 0000000..679c385 --- /dev/null +++ b/net/ipcohort/README.md @@ -0,0 +1,122 @@ +# [ipcohort](https://github.com/therootcompany/golib/tree/main/net/ipcohort) + +A memory-efficient, fast IP cohort checker for blacklists, whitelists, and ad cohorts. + +- 6 bytes per IP address (5 + 1 for alignment) +- binary search for /32 hosts, linear scan for CIDR ranges +- immutable cohorts — callers swap via `atomic.Pointer` for lock-free reads + +## Example + +Check if an IP address belongs to a cohort (such as a blacklist): + +```go +cohort, err := ipcohort.LoadFile("/srv/data/inbound.txt") +if err != nil { + log.Fatalf("load: %v", err) +} + +if cohort.Contains("92.255.85.72") { + fmt.Println("BLOCKED") + os.Exit(1) +} +fmt.Println("allowed") +``` + +## Update the list periodically: git (shallow) + +```go +import ( + "sync/atomic" + + "github.com/therootcompany/golib/net/gitshallow" + "github.com/therootcompany/golib/net/ipcohort" +) + +var cohort atomic.Pointer[ipcohort.Cohort] + +repo := gitshallow.New("https://github.com/bitwire-it/ipblocklist.git", "/srv/data/ipblocklist", 1, "") + +// Init: clone if missing, pull, load. +if _, err := repo.Init(false); err != nil { + log.Fatalf("init: %v", err) +} +c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt") +if err != nil { + log.Fatalf("load: %v", err) +} +cohort.Store(c) + +// Background: pull and reload when HEAD changes. +go func() { + ticker := time.NewTicker(47 * time.Minute) + defer ticker.Stop() + for range ticker.C { + updated, err := repo.Sync(false) + if err != nil { + log.Printf("sync: %v", err) + continue + } + if !updated { + continue + } + c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt") + if err != nil { + log.Printf("reload: %v", err) + continue + } + cohort.Store(c) + log.Printf("reloaded %d entries", cohort.Load().Size()) + } +}() +``` + +## Update the list periodically: HTTP (cache) + +```go +import ( + "sync/atomic" + + "github.com/therootcompany/golib/net/httpcache" + "github.com/therootcompany/golib/net/ipcohort" +) + +const listURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt" + +var cohort atomic.Pointer[ipcohort.Cohort] + +cacher := httpcache.New(listURL, "/srv/data/inbound.txt") + +// Init: fetch unconditionally, load. +if _, err := cacher.Fetch(); err != nil { + log.Fatalf("fetch: %v", err) +} +c, err := ipcohort.LoadFile("/srv/data/inbound.txt") +if err != nil { + log.Fatalf("load: %v", err) +} +cohort.Store(c) + +// Background: conditional GET, reload only when content changes. +go func() { + ticker := time.NewTicker(47 * time.Minute) + defer ticker.Stop() + for range ticker.C { + updated, err := cacher.Fetch() + if err != nil { + log.Printf("fetch: %v", err) + continue + } + if !updated { + continue + } + c, err := ipcohort.LoadFile("/srv/data/inbound.txt") + if err != nil { + log.Printf("reload: %v", err) + continue + } + cohort.Store(c) + log.Printf("reloaded %d entries", cohort.Load().Size()) + } +}() +``` diff --git a/net/ipcohort/cmd/ipcohort-contains/main.go b/net/ipcohort/cmd/ipcohort-contains/main.go new file mode 100644 index 0000000..62a50b0 --- /dev/null +++ b/net/ipcohort/cmd/ipcohort-contains/main.go @@ -0,0 +1,161 @@ +// ipcohort-contains checks whether one or more IP addresses appear in a set +// of cohort files (plain text, one IP/CIDR per line). +// +// Usage: +// +// ipcohort-contains [flags] ... -- ... +// ipcohort-contains [flags] --ip ... +// echo "" | ipcohort-contains ... +// +// Exit code: 0 if all queried IPs are found, 1 if any are not found, 2 on error. +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/therootcompany/golib/net/ipcohort" +) + +const version = "dev" + +func commafy(n int) string { + s := strconv.Itoa(n) + neg := "" + if n < 0 { + neg, s = "-", s[1:] + } + if len(s) <= 3 { + return neg + s + } + var b strings.Builder + head := len(s) % 3 + if head > 0 { + b.WriteString(s[:head]) + b.WriteByte(',') + } + for i := head; i < len(s); i += 3 { + b.WriteString(s[i : i+3]) + if i+3 < len(s) { + b.WriteByte(',') + } + } + return neg + b.String() +} + +type Config struct { + IP string +} + +func main() { + cfg := Config{} + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.StringVar(&cfg.IP, "ip", "", "IP address to check (alternative to -- separator)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] ... -- ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s --ip ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " echo | %s ...\n", os.Args[0]) + fs.PrintDefaults() + fmt.Fprintln(os.Stderr, "Exit: 0=all found, 1=not found, 2=error") + } + + if len(os.Args) > 1 { + switch os.Args[1] { + case "-V", "-version", "--version", "version": + fmt.Fprintf(os.Stdout, "ipcohort-contains %s\n", version) + os.Exit(0) + case "help", "-help", "--help": + fmt.Fprintf(os.Stdout, "ipcohort-contains %s\n\n", version) + fs.SetOutput(os.Stdout) + fs.Usage() + os.Exit(0) + } + } + if err := fs.Parse(os.Args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + os.Exit(2) + } + + args := fs.Args() + var filePaths, ips []string + switch { + case cfg.IP != "": + filePaths = args + ips = []string{cfg.IP} + default: + sep := -1 + for i, a := range args { + if a == "--" { + sep = i + break + } + } + if sep >= 0 { + filePaths = args[:sep] + ips = args[sep+1:] + } else { + filePaths = args + } + } + + if len(filePaths) == 0 { + fmt.Fprintln(os.Stderr, "error: at least one file path required") + fs.Usage() + os.Exit(2) + } + + fmt.Fprint(os.Stderr, "Loading cohort... ") + t := time.Now() + cohort, err := ipcohort.LoadFiles(filePaths...) + if err != nil { + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "%s (entries=%s)\n", + time.Since(t).Round(time.Millisecond), + commafy(cohort.Size()), + ) + + if len(ips) == 0 { + sc := bufio.NewScanner(os.Stdin) + for sc.Scan() { + if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") { + ips = append(ips, line) + } + } + if err := sc.Err(); err != nil { + fmt.Fprintf(os.Stderr, "error reading stdin: %v\n", err) + os.Exit(2) + } + } + + if len(ips) == 0 { + fmt.Fprintln(os.Stderr, "error: no IP addresses to check") + fs.Usage() + os.Exit(2) + } + + fmt.Fprintln(os.Stderr) + allFound := true + for _, ip := range ips { + if cohort.Contains(ip) { + fmt.Printf("%s\tFOUND\n", ip) + } else { + fmt.Printf("%s\tNOT FOUND\n", ip) + allFound = false + } + } + + if !allFound { + os.Exit(1) + } +} diff --git a/net/ipcohort/go.mod b/net/ipcohort/go.mod new file mode 100644 index 0000000..65d7ae7 --- /dev/null +++ b/net/ipcohort/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/ipcohort + +go 1.26.0 diff --git a/net/ipcohort/ipcohort.go b/net/ipcohort/ipcohort.go new file mode 100644 index 0000000..d1e03b5 --- /dev/null +++ b/net/ipcohort/ipcohort.go @@ -0,0 +1,223 @@ +package ipcohort + +import ( + "cmp" + "encoding/binary" + "encoding/csv" + "fmt" + "io" + "log" + "net/netip" + "os" + "slices" + "strings" +) + +// IPv4Net represents a subnet or single address (/32). +// 6 bytes: networkBE uint32 + prefix uint8 + shift uint8. +type IPv4Net struct { + networkBE uint32 + prefix uint8 + shift uint8 +} + +func NewIPv4Net(ip4be uint32, prefix uint8) IPv4Net { + return IPv4Net{ + networkBE: ip4be, + prefix: prefix, + shift: 32 - prefix, + } +} + +func (r IPv4Net) Contains(ip uint32) bool { + mask := uint32(0xFFFFFFFF << r.shift) + return (ip & mask) == r.networkBE +} + +// Cohort is an immutable, read-only set of IPv4 addresses and subnets. +// Contains is safe for concurrent use without locks. +// +// hosts holds sorted /32 addresses for O(log n) binary search. +// nets holds CIDR ranges (prefix < 32) for O(k) linear scan — typically small. +type Cohort struct { + hosts []uint32 + nets []IPv4Net +} + +func New() *Cohort { + return &Cohort{} +} + +func sortNets(nets []IPv4Net) { + slices.SortFunc(nets, func(a, b IPv4Net) int { + return cmp.Compare(a.networkBE, b.networkBE) + }) +} + +// Size returns the total number of entries (hosts + nets). +func (c *Cohort) Size() int { + return len(c.hosts) + len(c.nets) +} + +// Contains reports whether ipStr falls within any host or subnet in the cohort. +// Returns true on parse error (fail-closed): unparseable input is treated as +// blocked so that garbage strings never accidentally bypass a blocklist check. +// IPv6 addresses are not stored and always return false. +func (c *Cohort) Contains(ipStr string) bool { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return true // fail-closed + } + return c.ContainsAddr(ip) +} + +// ContainsAddr reports whether ip falls within any host or subnet in the cohort. +// IPv6 addresses always return false (cohort is IPv4-only). +func (c *Cohort) ContainsAddr(ip netip.Addr) bool { + if !ip.Is4() { + return false + } + ip4 := ip.As4() + ipU32 := binary.BigEndian.Uint32(ip4[:]) + + if _, found := slices.BinarySearch(c.hosts, ipU32); found { + return true + } + for _, net := range c.nets { + if net.Contains(ipU32) { + return true + } + } + return false +} + +func Parse(prefixList []string) (*Cohort, error) { + var hosts []uint32 + var nets []IPv4Net + + for _, raw := range prefixList { + ipv4net, err := ParseIPv4(raw) + if err != nil { + log.Printf("skipping invalid entry: %q", raw) + continue + } + if ipv4net.prefix == 32 { + hosts = append(hosts, ipv4net.networkBE) + } else { + nets = append(nets, ipv4net) + } + } + + slices.Sort(hosts) + sortNets(nets) + + return &Cohort{hosts: hosts, nets: nets}, nil +} + +func ParseIPv4(raw string) (ipv4net IPv4Net, err error) { + var ippre netip.Prefix + var ip netip.Addr + if strings.Contains(raw, "/") { + ippre, err = netip.ParsePrefix(raw) + if err != nil { + return ipv4net, err + } + } else { + ip, err = netip.ParseAddr(raw) + if err != nil { + return ipv4net, err + } + ippre = netip.PrefixFrom(ip, 32) + } + + addr := ippre.Addr() + if !addr.Is4() { + return ipv4net, fmt.Errorf("not an IPv4 address: %s", raw) + } + ip4 := addr.As4() + prefix := uint8(ippre.Bits()) // 0–32 + return NewIPv4Net( + binary.BigEndian.Uint32(ip4[:]), + prefix, + ), nil +} + +func LoadFile(path string) (*Cohort, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("could not load %q: %v", path, err) + } + defer f.Close() + + return ParseCSV(f) +} + +// LoadFiles loads and merges multiple files into one Cohort. +// Useful when hosts and networks are stored in separate files. +func LoadFiles(paths ...string) (*Cohort, error) { + var hosts []uint32 + var nets []IPv4Net + + for _, path := range paths { + c, err := LoadFile(path) + if err != nil { + return nil, err + } + hosts = append(hosts, c.hosts...) + nets = append(nets, c.nets...) + } + + slices.Sort(hosts) + sortNets(nets) + + return &Cohort{hosts: hosts, nets: nets}, nil +} + +func ParseCSV(f io.Reader) (*Cohort, error) { + r := csv.NewReader(f) + r.FieldsPerRecord = -1 + + return ReadAll(r) +} + +func ReadAll(r *csv.Reader) (*Cohort, error) { + var hosts []uint32 + var nets []IPv4Net + + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("csv read error: %w", err) + } + + if len(record) == 0 { + continue + } + + raw := strings.TrimSpace(record[0]) + + if raw == "" || strings.HasPrefix(raw, "#") { + continue + } + + ipv4net, err := ParseIPv4(raw) + if err != nil { + log.Printf("skipping invalid entry: %q", raw) + continue + } + + if ipv4net.prefix == 32 { + hosts = append(hosts, ipv4net.networkBE) + } else { + nets = append(nets, ipv4net) + } + } + + slices.Sort(hosts) + sortNets(nets) + + return &Cohort{hosts: hosts, nets: nets}, nil +} diff --git a/net/ipcohort/ipcohort_test.go b/net/ipcohort/ipcohort_test.go new file mode 100644 index 0000000..c5e9d53 --- /dev/null +++ b/net/ipcohort/ipcohort_test.go @@ -0,0 +1,166 @@ +package ipcohort_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/ipcohort" +) + +func TestParseIPv4(t *testing.T) { + tests := []struct { + raw string + wantErr bool + }{ + {"1.2.3.4", false}, + {"1.2.3.4/32", false}, + {"10.0.0.0/8", false}, + {"192.168.0.0/16", false}, + {"0.0.0.0/0", false}, + {"", true}, + {"not-an-ip", true}, + {"1.2.3.4/33", true}, + } + for _, tt := range tests { + _, err := ipcohort.ParseIPv4(tt.raw) + if (err != nil) != tt.wantErr { + t.Errorf("ParseIPv4(%q): got err=%v, wantErr=%v", tt.raw, err, tt.wantErr) + } + } +} + +func TestContains_SingleHosts(t *testing.T) { + c, err := ipcohort.Parse([]string{"1.2.3.4", "5.6.7.8", "10.0.0.1"}) + if err != nil { + t.Fatal(err) + } + + hits := []string{"1.2.3.4", "5.6.7.8", "10.0.0.1"} + misses := []string{"1.2.3.5", "5.6.7.7", "10.0.0.2", "0.0.0.0"} + + for _, ip := range hits { + if !c.Contains(ip) { + t.Errorf("expected %s to be in cohort", ip) + } + } + for _, ip := range misses { + if c.Contains(ip) { + t.Errorf("expected %s NOT to be in cohort", ip) + } + } +} + +func TestContains_CIDRRanges(t *testing.T) { + c, err := ipcohort.Parse([]string{"10.0.0.0/8", "192.168.1.0/24"}) + if err != nil { + t.Fatal(err) + } + + hits := []string{ + "10.0.0.0", "10.0.0.1", "10.255.255.255", + "192.168.1.0", "192.168.1.1", "192.168.1.254", "192.168.1.255", + } + misses := []string{ + "9.255.255.255", "11.0.0.0", + "192.168.0.255", "192.168.2.0", + } + + for _, ip := range hits { + if !c.Contains(ip) { + t.Errorf("expected %s to be in cohort (CIDR)", ip) + } + } + for _, ip := range misses { + if c.Contains(ip) { + t.Errorf("expected %s NOT to be in cohort (CIDR)", ip) + } + } +} + +func TestContains_Mixed(t *testing.T) { + c, err := ipcohort.Parse([]string{"1.2.3.4", "10.0.0.0/8"}) + if err != nil { + t.Fatal(err) + } + + if !c.Contains("1.2.3.4") { + t.Error("host miss") + } + if !c.Contains("10.5.5.5") { + t.Error("CIDR miss") + } + if c.Contains("1.2.3.5") { + t.Error("false positive for host-adjacent") + } +} + +func TestContains_FailClosed(t *testing.T) { + c, _ := ipcohort.Parse([]string{"1.2.3.4"}) + // Unparseable input should return true (fail-closed). + if !c.Contains("not-an-ip") { + t.Error("expected fail-closed true for unparseable IP") + } +} + + +func TestContains_Empty(t *testing.T) { + c, err := ipcohort.Parse(nil) + if err != nil { + t.Fatal(err) + } + if c.Contains("1.2.3.4") { + t.Error("empty cohort should not contain anything") + } +} + +func TestSize(t *testing.T) { + c, _ := ipcohort.Parse([]string{"1.2.3.4", "5.6.7.8", "10.0.0.0/8"}) + if got, want := c.Size(), 3; got != want { + t.Errorf("Size() = %d, want %d", got, want) + } +} + +func TestLoadFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "list.txt") + content := "# comment\n1.2.3.4\n10.0.0.0/8\n\n5.6.7.8\n" + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + c, err := ipcohort.LoadFile(path) + if err != nil { + t.Fatal(err) + } + if c.Size() != 3 { + t.Errorf("Size() = %d, want 3", c.Size()) + } + if !c.Contains("1.2.3.4") { + t.Error("missing 1.2.3.4") + } + if !c.Contains("10.5.5.5") { + t.Error("missing CIDR member 10.5.5.5") + } +} + +func TestLoadFiles_Merge(t *testing.T) { + dir := t.TempDir() + + f1 := filepath.Join(dir, "singles.txt") + f2 := filepath.Join(dir, "networks.txt") + os.WriteFile(f1, []byte("1.2.3.4\n5.6.7.8\n"), 0o644) + os.WriteFile(f2, []byte("192.168.0.0/24\n"), 0o644) + + c, err := ipcohort.LoadFiles(f1, f2) + if err != nil { + t.Fatal(err) + } + if c.Size() != 3 { + t.Errorf("Size() = %d, want 3", c.Size()) + } + if !c.Contains("192.168.0.100") { + t.Error("missing merged CIDR member") + } +} +