feat(iplist): load and validate IP policy entries from TSV/CSV sources

Add net/iplist for shared IP-policy source loading. - Accept local TSV/CSV file paths or HTTP(S) URLs (including Google Sheets). - Auto-detect TSV vs CSV format from file content. - Send Accept: text/tab-separated-values header for content negotiation. - Parse the first column and ignore comments (#) and optional "network" header. - Expand nested list URLs up to MaxNestedSources (2) levels. - Support Basic Auth from URL userinfo without sending credentials in the URL. - Persist cached URL responses across restarts via httpcache. - Refresh periodically and preserve the known-good cache on fetch or parse failure. - Validate entries as IP address, CIDR range, or domain name (Validate()). - Use golib/https.NewInternalClient() through httpcache.NewWith. - Add README with usage examples and entry format documentation.
2026-11-04 06:55:00 +00:00 · 2026-09-08 22:46:17 -06:00 · 2026-09-08 22:46:17 -06:00 · c41ac3732d
commit c41ac3732d
parent a16ec976bb
5 changed files with 529 additions and 0 deletions
--- a/net/iplist/README.md
+++ b/net/iplist/README.md
@ -0,0 +1,84 @@
+# net/iplist
+
+Load and validate IP policy entries from local TSV/CSV files, cached HTTP(S)
+URLs, and Google Sheets.
+
+## Features
+
+- **Local files**: TSV or CSV, auto-detected by content.
+- **HTTP(S) URLs**: fetched with conditional GET (ETag / Last-Modified),
+  cached to disk, and refreshed periodically. The cache survives process
+  restarts.
+- **Google Sheets**: edit/share URLs are recognised and fetched as CSV
+  exports automatically.
+- **Nested sources**: entries that are themselves HTTP(S) URLs are
+  recursively expanded up to `MaxNestedSources` (2) levels deep.
+- **Basic Auth**: credentials embedded in URL userinfo
+  (`https://user:pass@host/path`) are extracted and sent via the
+  `Authorization` header. The URL is stripped of credentials before
+  being used as a cache key or included in error messages.
+- **Content negotiation**: sends `Accept: text/tab-separated-values,
+  text/csv;q=0.5` so well-behaved servers return TSV. Format is
+  auto-detected from the response content (TSV if the first data line
+  contains a tab, otherwise CSV).
+- **Entry validation**: each non-URL entry must be an IP address, CIDR
+  range, or domain name. Invalid entries cause an error.
+
+## Usage
+
+```go
+import "github.com/therootcompany/golib/net/iplist"
+
+// Load from a local TSV file.
+entries, err := iplist.Load(ctx, "/etc/policy/allowlist.tsv", "/var/cache/iplist")
+
+// Load from an HTTPS URL with Basic Auth.
+entries, err := iplist.Load(ctx, "https://user:pass@policy.example.com/allowlist.tsv", "/var/cache/iplist")
+
+// Load from a Google Sheet.
+entries, err := iplist.Load(ctx, "https://docs.google.com/spreadsheets/d/1AbC.../edit?gid=123#gid=123", "/var/cache/iplist")
+```
+
+## Entry format
+
+Each row in a TSV or CSV source contributes its first column as an entry.
+Blank lines, comments (`#`), and an optional `network` header on the first
+line are ignored. Other columns are treated as labels and discarded.
+
+```
+# Allowlist
+network              note
+192.0.2.1            primary
+10.0.0.0/8           internal
+example.com          partner
+https://other.example.com/more-entries.tsv
+```
+
+In this example the URL on the last line will be fetched and its entries
+expanded inline (counted as one nesting level).
+
+## Validation
+
+Entries must be one of:
+
+| Type       | Examples                          |
+|------------|-----------------------------------|
+| IP address | `192.0.2.1`, `2001:db8::1`       |
+| CIDR range | `10.0.0.0/8`, `2001:db8::/32`    |
+| Domain     | `example.com`, `sub.example.com`  |
+| Source URL | `https://host/path.tsv` (expanded)|
+
+Use `iplist.Validate(s)` to check a single entry without loading a file.
+
+## Cache resilience
+
+On fetch failure the previous cache file is still used. If a fetch
+succeeds but the new content fails to parse, the previous cache is
+restored and the `.meta` sidecar is removed so the next call retries
+from scratch.
+
+## Dependencies
+
+- [`golib/https`](../https) — internal HTTP client
+- [`golib/io/transform/gsheet2csv`](../io/transform/gsheet2csv) — Google Sheet URL parsing
+- [`golib/net/httpcache`](../httpcache) — conditional GET caching with disk persistence
--- a/net/iplist/go.mod
+++ b/net/iplist/go.mod
@ -0,0 +1,11 @@
+module github.com/therootcompany/golib/net/iplist
+
+go 1.26.5
+
+require (
+	github.com/therootcompany/golib/https v0.9.0
+	github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0
+	github.com/therootcompany/golib/net/httpcache v0.5.1
+)
+
+require golang.org/x/sync v0.20.0 // indirect
--- a/net/iplist/go.sum
+++ b/net/iplist/go.sum
@ -0,0 +1,8 @@
+github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c=
+github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY=
+github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo=
+github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o=
+github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY=
+github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
--- a/net/iplist/iplist.go
+++ b/net/iplist/iplist.go
@ -0,0 +1,304 @@
+// Package iplist loads and validates IP policy entries from local TSV/CSV
+// files, cached HTTP(S) URLs, and Google Sheets.
+//
+// Each entry must be an IP address, CIDR range, domain name, or HTTP(S)
+// URL pointing to another TSV/CSV source. URL entries are recursively
+// expanded up to [MaxNestedSources] levels.
+package iplist
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/csv"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/therootcompany/golib/https"
+	"github.com/therootcompany/golib/io/transform/gsheet2csv"
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+// MaxNestedSources is the maximum nesting depth for URL sources whose
+// entries are themselves URLs. The top-level source is depth 0; a URL
+// entry at depth 0 is expanded at depth 1, and so on. A source at
+// depth MaxNestedSources+1 triggers an error.
+//
+// In other words, this limits *nesting* (URL-within-URL), not the total
+// number of sources fetched. Sibling URLs at the same depth are all
+// fetched.
+const MaxNestedSources = 2
+
+const (
+	defaultMaxAge   = time.Hour
+	defaultMaxBytes = 10 << 20
+)
+
+// Load reads source, which may be a local TSV/CSV file path or an HTTP(S)
+// URL (including Google Sheets edit/share URLs). Entries that are
+// themselves HTTP(S) URLs are recursively expanded up to
+// [MaxNestedSources] levels. URL responses are cached below cacheDir and
+// the cache remains available across process restarts.
+//
+// Each non-URL entry must be an IP address, CIDR range, or domain name.
+// Invalid entries cause an error.
+func Load(ctx context.Context, source, cacheDir string) ([]string, error) {
+	return load(ctx, strings.TrimSpace(source), cacheDir, make(map[string]struct{}), 0)
+}
+
+func load(ctx context.Context, source, cacheDir string, seen map[string]struct{}, depth int) ([]string, error) {
+	if source == "" {
+		return nil, nil
+	}
+	if depth > MaxNestedSources {
+		return nil, fmt.Errorf("source nesting exceeds %d levels", MaxNestedSources)
+	}
+	entries, err := readSource(ctx, source, cacheDir, seen)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if isURL(entry) {
+			nested, err := load(ctx, entry, cacheDir, seen, depth+1)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, nested...)
+			continue
+		}
+		if !Validate(entry) {
+			return nil, fmt.Errorf("invalid entry %q: must be an IP address, CIDR range, domain name, or HTTP(S) URL", entry)
+		}
+		result = append(result, entry)
+	}
+	return result, nil
+}
+
+func isURL(s string) bool {
+	parsed, err := url.Parse(s)
+	if err != nil {
+		return false
+	}
+	return parsed.Scheme == "http" || parsed.Scheme == "https"
+}
+
+// Validate reports whether s is a valid IP address, CIDR range, or domain
+// name. URL entries (HTTP(S) sources to be expanded) are not considered
+// valid by this function — they are handled separately by Load.
+func Validate(s string) bool {
+	if s == "" {
+		return false
+	}
+	// IP address (IPv4 or IPv6).
+	if net.ParseIP(s) != nil {
+		return true
+	}
+	// CIDR range.
+	if _, _, err := net.ParseCIDR(s); err == nil {
+		return true
+	}
+	// Domain name.
+	return isDomain(s)
+}
+
+// isDomain reports whether s looks like a domain name: it must contain at
+// least one dot, have no scheme, and consist of valid DNS label characters.
+func isDomain(s string) bool {
+	if s == "" || !strings.Contains(s, ".") {
+		return false
+	}
+	parsed, err := url.Parse(s)
+	if err != nil {
+		return false
+	}
+	// Reject anything with a scheme — URLs are handled separately.
+	if parsed.Scheme != "" {
+		return false
+	}
+	// Reject whitespace, slashes, colons, and other characters that don't
+	// belong in a bare domain name.
+	if strings.ContainsAny(s, " \t\r\n/:@?") {
+		return false
+	}
+	// Each label must be non-empty and contain only letters, digits, and
+	// hyphens (RFC 1035).
+	labels := strings.Split(s, ".")
+	for _, label := range labels {
+		if label == "" {
+			return false
+		}
+		for _, r := range label {
+			if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-') {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func readSource(ctx context.Context, source, cacheDir string, seen map[string]struct{}) ([]string, error) {
+	parsed, err := url.Parse(source)
+	if err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
+		if docid, gid := gsheet2csv.ParseIDs(source); docid != "" {
+			source = gsheet2csv.ToCSVURL(docid, gid)
+			parsed, err = url.Parse(source)
+		}
+	}
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		body, err := os.ReadFile(source)
+		if err != nil {
+			return nil, fmt.Errorf("read source %q: %w", source, err)
+		}
+		return Parse(bytes.NewReader(body))
+	}
+	key := parsed.String()
+	if _, ok := seen[key]; ok {
+		return nil, fmt.Errorf("source cycle detected at %q", safeURL(source))
+	}
+	seen[key] = struct{}{}
+	defer delete(seen, key)
+
+	// Collect headers: Basic Auth from URL userinfo plus Accept for
+	// content negotiation.
+	header := make(http.Header)
+	if parsed.User != nil {
+		password, _ := parsed.User.Password()
+		header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(parsed.User.Username()+":"+password)))
+		parsed.User = nil
+	}
+	// Prefer TSV but accept CSV and generic content.
+	header.Set("Accept", "text/tab-separated-values, text/csv;q=0.5, */*;q=0.1")
+
+	digest := sha256.Sum256([]byte(key))
+	cachePath := filepath.Join(cacheDir, "ip-sources", hex.EncodeToString(digest[:8])+".tsv")
+
+	// Ensure the cache directory exists before httpcache tries to write.
+	if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil {
+		return nil, fmt.Errorf("create cache dir for %s: %w", safeURL(source), err)
+	}
+
+	// previous is the last-known-good cache content. If the fetch fails or
+	// the new content can't be parsed, we fall back to (or restore) it.
+	previous, previousErr := os.ReadFile(cachePath)
+
+	cache := httpcache.NewWith(parsed.String(), cachePath, https.NewInternalClient())
+	cache.Header = header
+	cache.MaxAge = defaultMaxAge
+	cache.MaxBytes = defaultMaxBytes
+	if _, fetchErr := cache.Fetch(ctx); fetchErr != nil {
+		// No previous cache to fall back on — propagate the fetch error.
+		if previousErr != nil {
+			return nil, fmt.Errorf("fetch %s: %w", safeURL(source), fetchErr)
+		}
+		// Previous cache exists — fall through and read it instead.
+	}
+	body, err := os.ReadFile(cachePath)
+	if err != nil {
+		return nil, fmt.Errorf("read cached source %s: %w", safeURL(source), err)
+	}
+	entries, err := Parse(bytes.NewReader(body))
+	if err != nil {
+		// Fetch succeeded but produced unparseable content. If we had a
+		// previous cache, restore it so the next call can retry cleanly.
+		// On the first-ever run (no previous cache) there is nothing to
+		// restore, so we just return the parse error.
+		if previousErr == nil {
+			_ = os.WriteFile(cachePath, previous, 0o600)
+			_ = os.Remove(cachePath + ".meta")
+		}
+		return nil, fmt.Errorf("parse cached source %s: %w", safeURL(source), err)
+	}
+	return entries, nil
+}
+
+// Parse reads TSV or CSV data from r and returns the first column of each
+// row. Blank lines, comments (lines starting with #), and an optional
+// "network" header on the first line are ignored. Other columns are
+// treated as labels and discarded.
+//
+// The format is auto-detected: if the first non-empty, non-comment line
+// contains a tab character, TSV is assumed; otherwise CSV is used.
+func Parse(r io.Reader) ([]string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("read input: %w", err)
+	}
+	if isTSV(data) {
+		return parseTSV(bytes.NewReader(data))
+	}
+	return parseCSV(bytes.NewReader(data))
+}
+
+// isTSV heuristically detects TSV format by checking whether the first
+// non-empty, non-comment line contains a tab character.
+func isTSV(data []byte) bool {
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		trimmed := bytes.TrimSpace(line)
+		if len(trimmed) == 0 || bytes.HasPrefix(trimmed, []byte("#")) {
+			continue
+		}
+		return bytes.ContainsRune(line, '\t')
+	}
+	return false
+}
+
+func parseTSV(r io.Reader) ([]string, error) {
+	scanner := bufio.NewScanner(r)
+	entries := make([]string, 0)
+	for line := 1; scanner.Scan(); line++ {
+		col, _, _ := strings.Cut(scanner.Text(), "\t")
+		value := strings.TrimSpace(col)
+		if value == "" || strings.HasPrefix(value, "#") || (line == 1 && strings.EqualFold(value, "network")) {
+			continue
+		}
+		entries = append(entries, value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+func parseCSV(r io.Reader) ([]string, error) {
+	cr := csv.NewReader(r)
+	cr.FieldsPerRecord = -1 // allow variable column counts
+	cr.Comment = '#'
+	records, err := cr.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	entries := make([]string, 0, len(records))
+	for i, record := range records {
+		if len(record) == 0 {
+			continue
+		}
+		value := strings.TrimSpace(record[0])
+		if value == "" || (i == 0 && strings.EqualFold(value, "network")) {
+			continue
+		}
+		entries = append(entries, value)
+	}
+	return entries, nil
+}
+
+func safeURL(raw string) string {
+	parsed, err := url.Parse(raw)
+	if err != nil {
+		return "<unparseable URL>"
+	}
+	parsed.User = nil
+	parsed.RawQuery = ""
+	parsed.Fragment = ""
+	return parsed.String()
+}
--- a/net/iplist/iplist_test.go
+++ b/net/iplist/iplist_test.go
@ -0,0 +1,122 @@
+package iplist
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestParseTSV(t *testing.T) {
+	got, err := Parse(strings.NewReader("network\tnote\n# comment\n192.0.2.1\tlocal\nexample.com\n"))
+	if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" {
+		t.Fatalf("entries=%#v err=%v", got, err)
+	}
+}
+
+func TestParseCSV(t *testing.T) {
+	got, err := Parse(strings.NewReader("network,note\n# comment\n192.0.2.1,local\nexample.com\n"))
+	if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" {
+		t.Fatalf("entries=%#v err=%v", got, err)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	tests := []struct {
+		entry string
+		want  bool
+	}{
+		{"192.0.2.1", true},
+		{"2001:db8::1", true},
+		{"10.0.0.0/8", true},
+		{"2001:db8::/32", true},
+		{"example.com", true},
+		{"sub.example.com", true},
+		{"", false},
+		{"not valid", false},
+		{"has space.com", false},
+		{"/missing/domain", false},
+		{"http://example.com", false}, // URLs are not valid entries
+	}
+
+	for _, tt := range tests {
+		if got := Validate(tt.entry); got != tt.want {
+			t.Errorf("Validate(%q) = %v, want %v", tt.entry, got, tt.want)
+		}
+	}
+}
+
+func TestLoadRejectsInvalidEntry(t *testing.T) {
+	dir := t.TempDir()
+	// Write a local file with an invalid entry.
+	path := filepath.Join(dir, "bad.tsv")
+	if err := os.WriteFile(path, []byte("192.0.2.1\nnot valid entry\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	_, err := Load(t.Context(), path, dir)
+	if err == nil {
+		t.Fatal("expected error for invalid entry, got nil")
+	}
+}
+
+func TestLoadURLAuthCacheAndNested(t *testing.T) {
+	var authOK bool
+	inner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("192.0.2.2\n"))
+	}))
+	defer inner.Close()
+	outer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		authOK = r.Header.Get("Authorization") == "Basic dTpw"
+		_, _ = w.Write([]byte("network\n" + inner.URL + "\n"))
+	}))
+	defer outer.Close()
+
+	cacheDir := t.TempDir()
+	got, err := Load(t.Context(), strings.Replace(outer.URL, "http://", "http://u:p@", 1), cacheDir)
+	if err != nil || !authOK || len(got) != 1 || got[0] != "192.0.2.2" {
+		t.Fatalf("auth=%v entries=%#v err=%v", authOK, got, err)
+	}
+	matches, err := filepath.Glob(filepath.Join(cacheDir, "ip-sources", "*.tsv"))
+	if err != nil || len(matches) != 2 {
+		t.Fatalf("cache files=%v err=%v", matches, err)
+	}
+	if _, err := os.Stat(matches[0] + ".meta"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLoadAcceptHeader(t *testing.T) {
+	var acceptOK bool
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		acceptOK = strings.Contains(r.Header.Get("Accept"), "text/tab-separated-values")
+		_, _ = w.Write([]byte("192.0.2.1\n"))
+	}))
+	defer srv.Close()
+
+	cacheDir := t.TempDir()
+	if _, err := Load(t.Context(), srv.URL, cacheDir); err != nil {
+		t.Fatal(err)
+	}
+	if !acceptOK {
+		t.Fatal("expected Accept header to contain text/tab-separated-values")
+	}
+}
+
+func TestLoadCSVDetection(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/csv")
+		_, _ = w.Write([]byte("network,note\n192.0.2.1,local\n10.0.0.0/8,internal\n"))
+	}))
+	defer srv.Close()
+
+	cacheDir := t.TempDir()
+	got, err := Load(t.Context(), srv.URL, cacheDir)
+	if err != nil {
+		t.Fatalf("err=%v", err)
+	}
+	if len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "10.0.0.0/8" {
+		t.Fatalf("entries=%#v", got)
+	}
+}