root/golib
1
0
Fork 0
mirror of https://github.com/therootcompany/golib.git synced 2026-04-24 20:58:00 +00:00
Code Issues Projects Releases Wiki Activity

docs: update ipcohort README with git and HTTP periodic-update examples

Browse Source
This commit is contained in:
AJ ONeal 2026-04-19 23:42:06 -06:00
parent 4895553a91
commit 8c578ee0c6
No known key found for this signature in database
1 changed files with 102 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
net/ipcohort/README.md
View File

@ -3,51 +3,120 @@
A memory-efficient, fast IP cohort checker for blacklists, whitelists, and ad cohorts.
- 6 bytes per IP address (5 + 1 for alignment)
- binary search (not as fast as a trie, but memory is linear)
- atomic swaps for updates
- binary search for /32 hosts, linear scan for CIDR ranges
- immutable cohorts — callers swap via `atomic.Pointer` for lock-free reads
## Example
Check if an IP address belongs to a cohort (such as a blacklist):
```go
func main() {
ipStr := "92.255.85.72"
path := "/opt/github.com/bitwire-it/ipblocklist/inbound.txt"
unsorted := false
blacklist, err := ipcohort.LoadFile(path, unsorted)
cohort, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("Failed to load blacklist: %v", err)
log.Fatalf("load: %v", err)
}
if blacklist.Contains(ipStr) {
fmt.Printf("%s is BLOCKED\n", ipStr)
if cohort.Contains("92.255.85.72") {
fmt.Println("BLOCKED")
os.Exit(1)
}
fmt.Printf("%s is allowed\n", ipStr)
}
fmt.Println("allowed")
```
Update the list periodically:
## Update the list periodically: git (shallow)
```go
func backgroundUpdate(path string, c *ipcohort.Cohort) {
ticker := time.NewTicker(1 * time.Hour)
defer ticker.Stop()
import (
"sync/atomic"
for range ticker.C {
needsSort := false
nextCohort, err := ipcohort.LoadFile(path, needsSort)
"github.com/therootcompany/golib/net/gitshallow"
"github.com/therootcompany/golib/net/ipcohort"
)
var cohort atomic.Pointer[ipcohort.Cohort]
repo := gitshallow.New("https://github.com/bitwire-it/ipblocklist.git", "/srv/data/ipblocklist", 1, "")
// Init: clone if missing, pull, load.
if _, err := repo.Init(false); err != nil {
log.Fatalf("init: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Printf("reload failed: %v", err)
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: pull and reload when HEAD changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := repo.Sync(false)
if err != nil {
log.Printf("sync: %v", err)
continue
}
log.Printf("reloaded %d blacklist entries", c.Size())
c.Swap(nextCohort)
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/ipblocklist/tables/inbound/single_ips.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()
```
## Update the list periodically: HTTP (cache)
```go
import (
"sync/atomic"
"github.com/therootcompany/golib/net/httpcache"
"github.com/therootcompany/golib/net/ipcohort"
)
const listURL = "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt"
var cohort atomic.Pointer[ipcohort.Cohort]
cacher := httpcache.New(listURL, "/srv/data/inbound.txt")
// Init: fetch unconditionally, load.
if _, err := cacher.Fetch(); err != nil {
log.Fatalf("fetch: %v", err)
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Fatalf("load: %v", err)
}
cohort.Store(c)
// Background: conditional GET, reload only when content changes.
go func() {
ticker := time.NewTicker(47 * time.Minute)
defer ticker.Stop()
for range ticker.C {
updated, err := cacher.Fetch()
if err != nil {
log.Printf("fetch: %v", err)
continue
}
if !updated {
continue
}
c, err := ipcohort.LoadFile("/srv/data/inbound.txt")
if err != nil {
log.Printf("reload: %v", err)
continue
}
cohort.Store(c)
log.Printf("reloaded %d entries", cohort.Load().Size())
}
}()
```