feat(iplist): load cached local and remote TSV sources

net/iplist/go.mod

@@ -0,0 +1,10 @@
+module github.com/therootcompany/golib/net/iplist
+
+go 1.26.5
+
+require (
+	github.com/therootcompany/golib/https v0.9.0
+	github.com/therootcompany/golib/net/httpcache v0.5.1
+)
+
+require golang.org/x/sync v0.20.0 // indirect

net/iplist/go.sum

@@ -0,0 +1,6 @@
+github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
+github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
+github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
+github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=

net/iplist/iplist.go

@@ -0,0 +1,133 @@
+// Package iplist loads newline-delimited IP policy entries from local TSV
+// files and cached HTTP(S) URLs.
+package iplist
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/therootcompany/golib/https"
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+const MaxNestedSources = 2
+
+const (
+	defaultMaxAge   = time.Hour
+	defaultMaxBytes = 10 << 20
+)
+
+// Load reads source, which may be a local TSV path or an HTTP(S) URL. Entries
+// that are themselves HTTP(S) URLs are recursively expanded up to
+// MaxNestedSources levels. URL responses are cached below cacheDir and the
+// cache remains available across process restarts.
+func Load(ctx context.Context, source, cacheDir string) ([]string, error) {
+	return load(ctx, strings.TrimSpace(source), cacheDir, make(map[string]struct{}), 0)
+}
+
+func load(ctx context.Context, source, cacheDir string, seen map[string]struct{}, depth int) ([]string, error) {
+	if source == "" {
+		return nil, nil
+	}
+	if depth > MaxNestedSources {
+		return nil, fmt.Errorf("source nesting exceeds %d levels", MaxNestedSources)
+	}
+	entries, err := readSource(ctx, source, cacheDir, seen)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		parsed, parseErr := url.Parse(entry)
+		if parseErr == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
+			nested, err := load(ctx, entry, cacheDir, seen, depth+1)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, nested...)
+			continue
+		}
+		result = append(result, entry)
+	}
+	return result, nil
+}
+
+func readSource(ctx context.Context, source, cacheDir string, seen map[string]struct{}) ([]string, error) {
+	parsed, err := url.Parse(source)
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		body, err := os.ReadFile(source)
+		if err != nil {
+			return nil, fmt.Errorf("read source %q: %w", source, err)
+		}
+		return Parse(strings.NewReader(string(body)))
+	}
+	key := parsed.String()
+	if _, ok := seen[key]; ok {
+		return nil, fmt.Errorf("source cycle detected at %q", safeURL(source))
+	}
+	seen[key] = struct{}{}
+	defer delete(seen, key)
+
+	var header http.Header
+	if parsed.User != nil {
+		header = make(http.Header)
+		password, _ := parsed.User.Password()
+		header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(parsed.User.Username()+":"+password)))
+		parsed.User = nil
+	}
+	digest := sha256.Sum256([]byte(key))
+	path := filepath.Join(cacheDir, "ip-sources", hex.EncodeToString(digest[:8])+".tsv")
+	cache := httpcache.NewWith(parsed.String(), path, https.NewInternalClient())
+	cache.Header = header
+	cache.MaxAge = defaultMaxAge
+	cache.MaxBytes = defaultMaxBytes
+	if _, fetchErr := cache.Fetch(ctx); fetchErr != nil {
+		if _, statErr := os.Stat(path); statErr != nil {
+			return nil, fmt.Errorf("fetch %s: %w", safeURL(source), fetchErr)
+		}
+	}
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read cached source %s: %w", safeURL(source), err)
+	}
+	return Parse(strings.NewReader(string(body)))
+}
+
+// Parse reads the first TSV column. Blank lines, comments, and an optional
+// network header are ignored. Other columns are labels and are discarded.
+func Parse(input *strings.Reader) ([]string, error) {
+	scanner := bufio.NewScanner(input)
+	entries := make([]string, 0)
+	for line := 1; scanner.Scan(); line++ {
+		value := strings.TrimSpace(strings.SplitN(scanner.Text(), "\t", 2)[0])
+		if value == "" || strings.HasPrefix(value, "#") || (line == 1 && strings.EqualFold(value, "network")) {
+			continue
+		}
+		entries = append(entries, value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+func safeURL(raw string) string {
+	parsed, err := url.Parse(raw)
+	if err != nil {
+		return "<unparseable URL>"
+	}
+	parsed.User = nil
+	parsed.RawQuery = ""
+	parsed.Fragment = ""
+	return parsed.String()
+}

net/iplist/iplist_test.go

@@ -0,0 +1,42 @@
+package iplist
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestParseTSV(t *testing.T) {
+	got, err := Parse(strings.NewReader("network\tnote\n# comment\n192.0.2.1\tlocal\nexample.com\n"))
+	if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" {
+		t.Fatalf("entries=%#v err=%v", got, err)
+	}
+}
+
+func TestLoadURLAuthCacheAndNested(t *testing.T) {
+	var authOK bool
+	inner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("192.0.2.2\n")) }))
+	defer inner.Close()
+	outer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		authOK = r.Header.Get("Authorization") == "Basic dTpw"
+		_, _ = w.Write([]byte("network\n" + inner.URL + "\n"))
+	}))
+	defer outer.Close()
+
+	cacheDir := t.TempDir()
+	got, err := Load(context.Background(), strings.Replace(outer.URL, "http://", "http://u:p@", 1), cacheDir)
+	if err != nil || !authOK || len(got) != 1 || got[0] != "192.0.2.2" {
+		t.Fatalf("auth=%v entries=%#v err=%v", authOK, got, err)
+	}
+	matches, err := filepath.Glob(filepath.Join(cacheDir, "ip-sources", "*.tsv"))
+	if err != nil || len(matches) != 2 {
+		t.Fatalf("cache files=%v err=%v", matches, err)
+	}
+	if _, err := os.Stat(matches[0] + ".meta"); err != nil {
+		t.Fatal(err)
+	}
+}