mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 20:58:00 +00:00
AJ ONeal
673d084bd2
dataset.Loader[T] is now func() (*T, error) — a closure capturing its own paths/config, so multi-file cases (LoadFiles(paths...)) work naturally. Dataset.Close func(*T) is called with the old value after each swap, enabling resource cleanup (e.g. geoip2.Reader.Close). Sources.Datasets() builds a dataset.Group + three typed *Dataset[ipcohort.Cohort]. main.go now uses blGroup.Run / cityDS.Run / asnDS.Run instead of hand-rolled atomic.Pointer + polling loops. containsInbound/OutBound accept *Dataset[Cohort]. nopSyncer handles file-only GeoIP paths (no download, just open).
ipcohort
A memory-efficient, fast IP cohort checker for blacklists, whitelists, and ad cohorts.
- 6 bytes per IP address (5 + 1 for alignment)
- binary search for /32 hosts, linear scan for CIDR ranges
- immutable cohorts — callers swap via
atomic.Pointerfor lock-free reads
Example
Check if an IP address belongs to a cohort (such as a blacklist):
cohort, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
if cohort.Contains("92.255.85.72") {
fmt.Println("BLOCKED")
os.Exit(1)
}
fmt.Println("allowed")
Update the list periodically: git (shallow)
import (
"sync/atomic"
"github.com/therootcompany/golib/net/gitshallow"
"github.com/therootcompany/golib/net/ipcohort"
)
var cohort atomic.Pointer[ipcohort.Cohort]
repo := gitshallow.New("https://github.com/bitwire-it/ipblocklist.git", "/srv/data/ipblocklist", 1, "")
// Init: clone if missing, pull, load.
if _, err := repo.Init(false); err != nil {
log.Fatalf("init: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: pull and reload when HEAD changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := repo.Sync(false)
if err != nil {
log.Printf("sync: %v", err)
continue
}
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()
Update the list periodically: HTTP (cache)
import (
"sync/atomic"
"github.com/therootcompany/golib/net/httpcache"
"github.com/therootcompany/golib/net/ipcohort"
)
const listURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
var cohort atomic.Pointer[ipcohort.Cohort]
cacher := httpcache.New(listURL, "/srv/data/inbound.txt")
// Init: fetch unconditionally, load.
if _, err := cacher.Fetch(); err != nil {
log.Fatalf("fetch: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: conditional GET, reload only when content changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := cacher.Fetch()
if err != nil {
log.Printf("fetch: %v", err)
continue
}
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()