From 34a54c2d66fc05e9eac898b7a639c01d24c5c336 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 20 Apr 2026 11:22:01 -0600 Subject: [PATCH] refactor: multi-module workspace + dataset owns Syncer interface - Each package gets its own go.mod: net/{dataset,httpcache,gitshallow,ipcohort,geoip,formmailer} - go.work with replace directives for cross-module workspace resolution - dataset.Syncer/NopSyncer moved here from httpcache; callers duck-type it - dataset.View[T] returned by Add to prevent Init/Sync/Run misuse on group members - cmd/check-ip moved from net/ipcohort/cmd/check-ip to top-level cmd/check-ip - Add net/ipcohort/cmd/ipcohort-contains for standalone cohort membership testing --- .gitignore | 3 + .../cmd => cmd}/check-ip/blacklist.go | 6 +- cmd/check-ip/go.mod | 12 ++ {net/ipcohort/cmd => cmd}/check-ip/main.go | 5 +- go.work | 21 ++++ net/dataset/dataset.go | 22 +++- net/dataset/dataset_test.go | 7 +- net/dataset/go.mod | 3 + net/formmailer/go.mod | 10 ++ net/geoip/go.mod | 8 ++ net/gitshallow/go.mod | 3 + net/httpcache/go.mod | 3 + net/httpcache/httpcache.go | 11 -- net/ipcohort/cmd/ipcohort-contains/main.go | 110 ++++++++++++++++++ net/ipcohort/go.mod | 3 + 15 files changed, 200 insertions(+), 27 deletions(-) rename {net/ipcohort/cmd => cmd}/check-ip/blacklist.go (95%) create mode 100644 cmd/check-ip/go.mod rename {net/ipcohort/cmd => cmd}/check-ip/main.go (98%) create mode 100644 go.work create mode 100644 net/dataset/go.mod create mode 100644 net/formmailer/go.mod create mode 100644 net/geoip/go.mod create mode 100644 net/gitshallow/go.mod create mode 100644 net/httpcache/go.mod create mode 100644 net/ipcohort/cmd/ipcohort-contains/main.go create mode 100644 net/ipcohort/go.mod diff --git a/.gitignore b/.gitignore index 209ec0c..f4bf406 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ credentials.tsv GeoIP.conf *.mmdb +testdata .env *.env @@ -9,6 +10,8 @@ env.* # Project binaries dist/ +check-ip +cmd/check-ip/check-ip auth/csvauth/cmd/csvauth/csvauth cmd/auth-proxy/auth-proxy cmd/httplog/httplog diff --git a/net/ipcohort/cmd/check-ip/blacklist.go b/cmd/check-ip/blacklist.go similarity index 95% rename from net/ipcohort/cmd/check-ip/blacklist.go rename to cmd/check-ip/blacklist.go index 6ab0ed6..fc0e93c 100644 --- a/net/ipcohort/cmd/check-ip/blacklist.go +++ b/cmd/check-ip/blacklist.go @@ -22,7 +22,7 @@ type Sources struct { inboundPaths []string outboundPaths []string - syncs []httpcache.Syncer // all syncable sources + syncs []dataset.Syncer // all syncable sources } func newFileSources(whitelist, inbound, outbound []string) *Sources { @@ -46,7 +46,7 @@ func newGitSources(gitURL, repoDir string, whitelist, inboundRel, outboundRel [] whitelistPaths: whitelist, inboundPaths: abs(inboundRel), outboundPaths: abs(outboundRel), - syncs: []httpcache.Syncer{repo}, + syncs: []dataset.Syncer{repo}, } } @@ -64,7 +64,7 @@ func newHTTPSources(whitelist []string, inbound, outbound []HTTPSource) *Sources } // Fetch pulls updates from all sources. Returns whether any new data arrived. -// Satisfies httpcache.Syncer. +// Satisfies dataset.Syncer. func (s *Sources) Fetch() (bool, error) { var anyUpdated bool for _, syn := range s.syncs { diff --git a/cmd/check-ip/go.mod b/cmd/check-ip/go.mod new file mode 100644 index 0000000..707679a --- /dev/null +++ b/cmd/check-ip/go.mod @@ -0,0 +1,12 @@ +module github.com/therootcompany/golib/cmd/check-ip + +go 1.26.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 + github.com/therootcompany/golib/net/dataset v0.0.0 + github.com/therootcompany/golib/net/geoip v0.0.0 + github.com/therootcompany/golib/net/gitshallow v0.0.0 + github.com/therootcompany/golib/net/httpcache v0.0.0 + github.com/therootcompany/golib/net/ipcohort v0.0.0 +) diff --git a/net/ipcohort/cmd/check-ip/main.go b/cmd/check-ip/main.go similarity index 98% rename from net/ipcohort/cmd/check-ip/main.go rename to cmd/check-ip/main.go index 2da9d17..fcc58c6 100644 --- a/net/ipcohort/cmd/check-ip/main.go +++ b/cmd/check-ip/main.go @@ -13,7 +13,6 @@ import ( "github.com/oschwald/geoip2-golang" "github.com/therootcompany/golib/net/dataset" "github.com/therootcompany/golib/net/geoip" - "github.com/therootcompany/golib/net/httpcache" "github.com/therootcompany/golib/net/ipcohort" ) @@ -194,11 +193,11 @@ func main() { // newGeoIPDataset creates a Dataset[geoip2.Reader]. If d is nil, only // opens the existing file (no download). Close is wired to Reader.Close. func newGeoIPDataset(d *geoip.Downloader, edition, path string) *dataset.Dataset[geoip2.Reader] { - var syncer httpcache.Syncer + var syncer dataset.Syncer if d != nil { syncer = d.NewCacher(edition, path) } else { - syncer = httpcache.NopSyncer{} + syncer = dataset.NopSyncer{} } ds := dataset.New(syncer, func() (*geoip2.Reader, error) { return geoip2.Open(path) diff --git a/go.work b/go.work new file mode 100644 index 0000000..64fbb52 --- /dev/null +++ b/go.work @@ -0,0 +1,21 @@ +go 1.26.1 + +use ( + . + ./cmd/check-ip + ./net/dataset + ./net/formmailer + ./net/geoip + ./net/gitshallow + ./net/httpcache + ./net/ipcohort +) + +replace ( + github.com/therootcompany/golib/net/dataset v0.0.0 => ./net/dataset + github.com/therootcompany/golib/net/formmailer v0.0.0 => ./net/formmailer + github.com/therootcompany/golib/net/geoip v0.0.0 => ./net/geoip + github.com/therootcompany/golib/net/gitshallow v0.0.0 => ./net/gitshallow + github.com/therootcompany/golib/net/httpcache v0.0.0 => ./net/httpcache + github.com/therootcompany/golib/net/ipcohort v0.0.0 => ./net/ipcohort +) diff --git a/net/dataset/dataset.go b/net/dataset/dataset.go index 99dc9e3..b84a272 100644 --- a/net/dataset/dataset.go +++ b/net/dataset/dataset.go @@ -26,10 +26,20 @@ import ( "os" "sync/atomic" "time" - - "github.com/therootcompany/golib/net/httpcache" ) +// Syncer is implemented by any value that can fetch a remote resource and +// report whether it changed. +type Syncer interface { + Fetch() (updated bool, err error) +} + +// NopSyncer is a Syncer that always reports no update and no error. +// Use for datasets backed by local files with no remote source. +type NopSyncer struct{} + +func (NopSyncer) Fetch() (bool, error) { return false, nil } + // Dataset couples a Syncer, a load function, and an atomic.Pointer[T]. // Load is safe for concurrent use without locks. type Dataset[T any] struct { @@ -39,14 +49,14 @@ type Dataset[T any] struct { // Use this for values that hold resources, e.g. func(r *geoip2.Reader) { r.Close() }. Close func(*T) - syncer httpcache.Syncer + syncer Syncer load func() (*T, error) ptr atomic.Pointer[T] } // New creates a Dataset. The syncer fetches updates; load produces the value. // load is a closure — it captures whatever paths or config it needs. -func New[T any](syncer httpcache.Syncer, load func() (*T, error)) *Dataset[T] { +func New[T any](syncer Syncer, load func() (*T, error)) *Dataset[T] { return &Dataset[T]{syncer: syncer, load: load} } @@ -115,12 +125,12 @@ type member interface { // Group ties one Syncer to multiple datasets so a single Fetch drives all // reloads — no redundant network calls when datasets share a source. type Group struct { - syncer httpcache.Syncer + syncer Syncer members []member } // NewGroup creates a Group backed by syncer. -func NewGroup(syncer httpcache.Syncer) *Group { +func NewGroup(syncer Syncer) *Group { return &Group{syncer: syncer} } diff --git a/net/dataset/dataset_test.go b/net/dataset/dataset_test.go index d06af39..ab5926b 100644 --- a/net/dataset/dataset_test.go +++ b/net/dataset/dataset_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/therootcompany/golib/net/dataset" - "github.com/therootcompany/golib/net/httpcache" ) // countSyncer counts Fetch calls and optionally reports updated. @@ -47,7 +46,7 @@ func TestDataset_Init(t *testing.T) { } func TestDataset_LoadBeforeInit(t *testing.T) { - syn := httpcache.NopSyncer{} + syn := dataset.NopSyncer{} ds := dataset.New(syn, func() (*string, error) { v := "x" return &v, nil @@ -120,7 +119,7 @@ func TestDataset_InitError(t *testing.T) { } func TestDataset_LoaderError(t *testing.T) { - syn := httpcache.NopSyncer{} + syn := dataset.NopSyncer{} ds := dataset.New(syn, func() (*string, error) { return nil, errors.New("load failed") }) @@ -273,7 +272,7 @@ func TestGroup_FetchError(t *testing.T) { } func TestGroup_LoaderError(t *testing.T) { - syn := httpcache.NopSyncer{} + syn := dataset.NopSyncer{} g := dataset.NewGroup(syn) dataset.Add(g, func() (*string, error) { return nil, errors.New("parse error") diff --git a/net/dataset/go.mod b/net/dataset/go.mod new file mode 100644 index 0000000..38698eb --- /dev/null +++ b/net/dataset/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/dataset + +go 1.26.0 diff --git a/net/formmailer/go.mod b/net/formmailer/go.mod new file mode 100644 index 0000000..99330aa --- /dev/null +++ b/net/formmailer/go.mod @@ -0,0 +1,10 @@ +module github.com/therootcompany/golib/net/formmailer + +go 1.26.0 + +require ( + github.com/phuslu/iploc v1.0.20260415 + github.com/therootcompany/golib/net/dataset v0.0.0 + github.com/therootcompany/golib/net/ipcohort v0.0.0 + golang.org/x/time v0.15.0 +) diff --git a/net/geoip/go.mod b/net/geoip/go.mod new file mode 100644 index 0000000..3a91846 --- /dev/null +++ b/net/geoip/go.mod @@ -0,0 +1,8 @@ +module github.com/therootcompany/golib/net/geoip + +go 1.26.0 + +require ( + github.com/oschwald/geoip2-golang v1.13.0 + github.com/therootcompany/golib/net/httpcache v0.0.0 +) diff --git a/net/gitshallow/go.mod b/net/gitshallow/go.mod new file mode 100644 index 0000000..01121e1 --- /dev/null +++ b/net/gitshallow/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/gitshallow + +go 1.26.0 diff --git a/net/httpcache/go.mod b/net/httpcache/go.mod new file mode 100644 index 0000000..b8157ee --- /dev/null +++ b/net/httpcache/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/httpcache + +go 1.26.0 diff --git a/net/httpcache/httpcache.go b/net/httpcache/httpcache.go index 436cd9f..99fb353 100644 --- a/net/httpcache/httpcache.go +++ b/net/httpcache/httpcache.go @@ -16,17 +16,6 @@ const ( defaultTimeout = 5 * time.Minute // overall including body read ) -// Syncer is implemented by any value that can fetch a remote resource and -// report whether it changed. Both *Cacher and *gitshallow.Repo satisfy this. -type Syncer interface { - Fetch() (updated bool, err error) -} - -// NopSyncer is a Syncer that always reports no update and no error. -// Use for datasets backed by local files managed externally (no download). -type NopSyncer struct{} - -func (NopSyncer) Fetch() (bool, error) { return false, nil } // Cacher fetches a URL to a local file, using ETag/Last-Modified to skip // unchanged responses. diff --git a/net/ipcohort/cmd/ipcohort-contains/main.go b/net/ipcohort/cmd/ipcohort-contains/main.go new file mode 100644 index 0000000..a59f6db --- /dev/null +++ b/net/ipcohort/cmd/ipcohort-contains/main.go @@ -0,0 +1,110 @@ +// ipcohort-contains checks whether one or more IP addresses appear in a set +// of cohort files (plain text, one IP/CIDR per line). +// +// Usage: +// +// ipcohort-contains [flags] ... -- ... +// ipcohort-contains [flags] -ip ... +// +// Examples: +// +// ipcohort-contains networks.txt single_ips.txt -- 1.2.3.4 5.6.7.8 +// ipcohort-contains -ip 1.2.3.4 single_ips.txt +// echo "1.2.3.4" | ipcohort-contains networks.txt +// +// Exit code: 0 if all queried IPs are found, 1 if any are not found, 2 on error. +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "strings" + + "github.com/therootcompany/golib/net/ipcohort" +) + +func main() { + ipFlag := flag.String("ip", "", "IP address to check (alternative to -- separator)") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] ... -- ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s -ip ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " echo | %s ...\n", os.Args[0]) + fmt.Fprintln(os.Stderr, "Flags:") + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, "Exit: 0=all found, 1=not found, 2=error") + } + flag.Parse() + + args := flag.Args() + var filePaths, ips []string + + switch { + case *ipFlag != "": + filePaths = args + ips = []string{*ipFlag} + default: + // Split args at "--" + sep := -1 + for i, a := range args { + if a == "--" { + sep = i + break + } + } + if sep >= 0 { + filePaths = args[:sep] + ips = args[sep+1:] + } else { + filePaths = args + } + } + + if len(filePaths) == 0 { + fmt.Fprintln(os.Stderr, "error: at least one file path required") + flag.Usage() + os.Exit(2) + } + + cohort, err := ipcohort.LoadFiles(filePaths...) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(2) + } + + // If no IPs from flags/args, read from stdin. + if len(ips) == 0 { + sc := bufio.NewScanner(os.Stdin) + for sc.Scan() { + if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") { + ips = append(ips, line) + } + } + if err := sc.Err(); err != nil { + fmt.Fprintf(os.Stderr, "error reading stdin: %v\n", err) + os.Exit(2) + } + } + + if len(ips) == 0 { + fmt.Fprintln(os.Stderr, "error: no IP addresses to check") + flag.Usage() + os.Exit(2) + } + + allFound := true + for _, ip := range ips { + found := cohort.Contains(ip) + if found { + fmt.Printf("%s\tFOUND\n", ip) + } else { + fmt.Printf("%s\tNOT FOUND\n", ip) + allFound = false + } + } + + if !allFound { + os.Exit(1) + } +} diff --git a/net/ipcohort/go.mod b/net/ipcohort/go.mod new file mode 100644 index 0000000..65d7ae7 --- /dev/null +++ b/net/ipcohort/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/ipcohort + +go 1.26.0