From 1110ab7e2a2553d8b60a6be4d58e6ba328f7d65f Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 8 Sep 2026 22:12:05 -0600 Subject: [PATCH] feat(iplist): load cached local and remote TSV sources --- net/iplist/go.mod | 10 +++ net/iplist/go.sum | 6 ++ net/iplist/iplist.go | 133 ++++++++++++++++++++++++++++++++++++++ net/iplist/iplist_test.go | 42 ++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 net/iplist/go.mod create mode 100644 net/iplist/go.sum create mode 100644 net/iplist/iplist.go create mode 100644 net/iplist/iplist_test.go diff --git a/net/iplist/go.mod b/net/iplist/go.mod new file mode 100644 index 0000000..d93ff05 --- /dev/null +++ b/net/iplist/go.mod @@ -0,0 +1,10 @@ +module github.com/therootcompany/golib/net/iplist + +go 1.26.5 + +require ( + github.com/therootcompany/golib/https v0.9.0 + github.com/therootcompany/golib/net/httpcache v0.5.1 +) + +require golang.org/x/sync v0.20.0 // indirect diff --git a/net/iplist/go.sum b/net/iplist/go.sum new file mode 100644 index 0000000..9f9d3ba --- /dev/null +++ b/net/iplist/go.sum @@ -0,0 +1,6 @@ +github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= +github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= +github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= +github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= diff --git a/net/iplist/iplist.go b/net/iplist/iplist.go new file mode 100644 index 0000000..07b2371 --- /dev/null +++ b/net/iplist/iplist.go @@ -0,0 +1,133 @@ +// Package iplist loads newline-delimited IP policy entries from local TSV +// files and cached HTTP(S) URLs. +package iplist + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/therootcompany/golib/https" + "github.com/therootcompany/golib/net/httpcache" +) + +const MaxNestedSources = 2 + +const ( + defaultMaxAge = time.Hour + defaultMaxBytes = 10 << 20 +) + +// Load reads source, which may be a local TSV path or an HTTP(S) URL. Entries +// that are themselves HTTP(S) URLs are recursively expanded up to +// MaxNestedSources levels. URL responses are cached below cacheDir and the +// cache remains available across process restarts. +func Load(ctx context.Context, source, cacheDir string) ([]string, error) { + return load(ctx, strings.TrimSpace(source), cacheDir, make(map[string]struct{}), 0) +} + +func load(ctx context.Context, source, cacheDir string, seen map[string]struct{}, depth int) ([]string, error) { + if source == "" { + return nil, nil + } + if depth > MaxNestedSources { + return nil, fmt.Errorf("source nesting exceeds %d levels", MaxNestedSources) + } + entries, err := readSource(ctx, source, cacheDir, seen) + if err != nil { + return nil, err + } + result := make([]string, 0, len(entries)) + for _, entry := range entries { + parsed, parseErr := url.Parse(entry) + if parseErr == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { + nested, err := load(ctx, entry, cacheDir, seen, depth+1) + if err != nil { + return nil, err + } + result = append(result, nested...) + continue + } + result = append(result, entry) + } + return result, nil +} + +func readSource(ctx context.Context, source, cacheDir string, seen map[string]struct{}) ([]string, error) { + parsed, err := url.Parse(source) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { + body, err := os.ReadFile(source) + if err != nil { + return nil, fmt.Errorf("read source %q: %w", source, err) + } + return Parse(strings.NewReader(string(body))) + } + key := parsed.String() + if _, ok := seen[key]; ok { + return nil, fmt.Errorf("source cycle detected at %q", safeURL(source)) + } + seen[key] = struct{}{} + defer delete(seen, key) + + var header http.Header + if parsed.User != nil { + header = make(http.Header) + password, _ := parsed.User.Password() + header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(parsed.User.Username()+":"+password))) + parsed.User = nil + } + digest := sha256.Sum256([]byte(key)) + path := filepath.Join(cacheDir, "ip-sources", hex.EncodeToString(digest[:8])+".tsv") + cache := httpcache.NewWith(parsed.String(), path, https.NewInternalClient()) + cache.Header = header + cache.MaxAge = defaultMaxAge + cache.MaxBytes = defaultMaxBytes + if _, fetchErr := cache.Fetch(ctx); fetchErr != nil { + if _, statErr := os.Stat(path); statErr != nil { + return nil, fmt.Errorf("fetch %s: %w", safeURL(source), fetchErr) + } + } + body, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read cached source %s: %w", safeURL(source), err) + } + return Parse(strings.NewReader(string(body))) +} + +// Parse reads the first TSV column. Blank lines, comments, and an optional +// network header are ignored. Other columns are labels and are discarded. +func Parse(input *strings.Reader) ([]string, error) { + scanner := bufio.NewScanner(input) + entries := make([]string, 0) + for line := 1; scanner.Scan(); line++ { + value := strings.TrimSpace(strings.SplitN(scanner.Text(), "\t", 2)[0]) + if value == "" || strings.HasPrefix(value, "#") || (line == 1 && strings.EqualFold(value, "network")) { + continue + } + entries = append(entries, value) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return entries, nil +} + +func safeURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + parsed.User = nil + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String() +} diff --git a/net/iplist/iplist_test.go b/net/iplist/iplist_test.go new file mode 100644 index 0000000..c9512a6 --- /dev/null +++ b/net/iplist/iplist_test.go @@ -0,0 +1,42 @@ +package iplist + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseTSV(t *testing.T) { + got, err := Parse(strings.NewReader("network\tnote\n# comment\n192.0.2.1\tlocal\nexample.com\n")) + if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" { + t.Fatalf("entries=%#v err=%v", got, err) + } +} + +func TestLoadURLAuthCacheAndNested(t *testing.T) { + var authOK bool + inner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("192.0.2.2\n")) })) + defer inner.Close() + outer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authOK = r.Header.Get("Authorization") == "Basic dTpw" + _, _ = w.Write([]byte("network\n" + inner.URL + "\n")) + })) + defer outer.Close() + + cacheDir := t.TempDir() + got, err := Load(context.Background(), strings.Replace(outer.URL, "http://", "http://u:p@", 1), cacheDir) + if err != nil || !authOK || len(got) != 1 || got[0] != "192.0.2.2" { + t.Fatalf("auth=%v entries=%#v err=%v", authOK, got, err) + } + matches, err := filepath.Glob(filepath.Join(cacheDir, "ip-sources", "*.tsv")) + if err != nil || len(matches) != 2 { + t.Fatalf("cache files=%v err=%v", matches, err) + } + if _, err := os.Stat(matches[0] + ".meta"); err != nil { + t.Fatal(err) + } +}