diff --git a/net/iplist/README.md b/net/iplist/README.md new file mode 100644 index 0000000..5f8bbce --- /dev/null +++ b/net/iplist/README.md @@ -0,0 +1,84 @@ +# net/iplist + +Load and validate IP policy entries from local TSV/CSV files, cached HTTP(S) +URLs, and Google Sheets. + +## Features + +- **Local files**: TSV or CSV, auto-detected by content. +- **HTTP(S) URLs**: fetched with conditional GET (ETag / Last-Modified), + cached to disk, and refreshed periodically. The cache survives process + restarts. +- **Google Sheets**: edit/share URLs are recognised and fetched as CSV + exports automatically. +- **Nested sources**: entries that are themselves HTTP(S) URLs are + recursively expanded up to `MaxNestedSources` (2) levels deep. +- **Basic Auth**: credentials embedded in URL userinfo + (`https://user:pass@host/path`) are extracted and sent via the + `Authorization` header. The URL is stripped of credentials before + being used as a cache key or included in error messages. +- **Content negotiation**: sends `Accept: text/tab-separated-values, + text/csv;q=0.5` so well-behaved servers return TSV. Format is + auto-detected from the response content (TSV if the first data line + contains a tab, otherwise CSV). +- **Entry validation**: each non-URL entry must be an IP address, CIDR + range, or domain name. Invalid entries cause an error. + +## Usage + +```go +import "github.com/therootcompany/golib/net/iplist" + +// Load from a local TSV file. +entries, err := iplist.Load(ctx, "/etc/policy/allowlist.tsv", "/var/cache/iplist") + +// Load from an HTTPS URL with Basic Auth. +entries, err := iplist.Load(ctx, "https://user:pass@policy.example.com/allowlist.tsv", "/var/cache/iplist") + +// Load from a Google Sheet. +entries, err := iplist.Load(ctx, "https://docs.google.com/spreadsheets/d/1AbC.../edit?gid=123#gid=123", "/var/cache/iplist") +``` + +## Entry format + +Each row in a TSV or CSV source contributes its first column as an entry. +Blank lines, comments (`#`), and an optional `network` header on the first +line are ignored. Other columns are treated as labels and discarded. + +``` +# Allowlist +network note +192.0.2.1 primary +10.0.0.0/8 internal +example.com partner +https://other.example.com/more-entries.tsv +``` + +In this example the URL on the last line will be fetched and its entries +expanded inline (counted as one nesting level). + +## Validation + +Entries must be one of: + +| Type | Examples | +|------------|-----------------------------------| +| IP address | `192.0.2.1`, `2001:db8::1` | +| CIDR range | `10.0.0.0/8`, `2001:db8::/32` | +| Domain | `example.com`, `sub.example.com` | +| Source URL | `https://host/path.tsv` (expanded)| + +Use `iplist.Validate(s)` to check a single entry without loading a file. + +## Cache resilience + +On fetch failure the previous cache file is still used. If a fetch +succeeds but the new content fails to parse, the previous cache is +restored and the `.meta` sidecar is removed so the next call retries +from scratch. + +## Dependencies + +- [`golib/https`](../https) — internal HTTP client +- [`golib/io/transform/gsheet2csv`](../io/transform/gsheet2csv) — Google Sheet URL parsing +- [`golib/net/httpcache`](../httpcache) — conditional GET caching with disk persistence \ No newline at end of file diff --git a/net/iplist/go.mod b/net/iplist/go.mod new file mode 100644 index 0000000..02a3996 --- /dev/null +++ b/net/iplist/go.mod @@ -0,0 +1,11 @@ +module github.com/therootcompany/golib/net/iplist + +go 1.26.5 + +require ( + github.com/therootcompany/golib/https v0.9.0 + github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 + github.com/therootcompany/golib/net/httpcache v0.5.1 +) + +require golang.org/x/sync v0.20.0 // indirect diff --git a/net/iplist/go.sum b/net/iplist/go.sum new file mode 100644 index 0000000..3304d22 --- /dev/null +++ b/net/iplist/go.sum @@ -0,0 +1,8 @@ +github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= +github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o= +github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= +github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= diff --git a/net/iplist/iplist.go b/net/iplist/iplist.go new file mode 100644 index 0000000..ad8cb56 --- /dev/null +++ b/net/iplist/iplist.go @@ -0,0 +1,304 @@ +// Package iplist loads and validates IP policy entries from local TSV/CSV +// files, cached HTTP(S) URLs, and Google Sheets. +// +// Each entry must be an IP address, CIDR range, domain name, or HTTP(S) +// URL pointing to another TSV/CSV source. URL entries are recursively +// expanded up to [MaxNestedSources] levels. +package iplist + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/csv" + "encoding/hex" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/therootcompany/golib/https" + "github.com/therootcompany/golib/io/transform/gsheet2csv" + "github.com/therootcompany/golib/net/httpcache" +) + +// MaxNestedSources is the maximum nesting depth for URL sources whose +// entries are themselves URLs. The top-level source is depth 0; a URL +// entry at depth 0 is expanded at depth 1, and so on. A source at +// depth MaxNestedSources+1 triggers an error. +// +// In other words, this limits *nesting* (URL-within-URL), not the total +// number of sources fetched. Sibling URLs at the same depth are all +// fetched. +const MaxNestedSources = 2 + +const ( + defaultMaxAge = time.Hour + defaultMaxBytes = 10 << 20 +) + +// Load reads source, which may be a local TSV/CSV file path or an HTTP(S) +// URL (including Google Sheets edit/share URLs). Entries that are +// themselves HTTP(S) URLs are recursively expanded up to +// [MaxNestedSources] levels. URL responses are cached below cacheDir and +// the cache remains available across process restarts. +// +// Each non-URL entry must be an IP address, CIDR range, or domain name. +// Invalid entries cause an error. +func Load(ctx context.Context, source, cacheDir string) ([]string, error) { + return load(ctx, strings.TrimSpace(source), cacheDir, make(map[string]struct{}), 0) +} + +func load(ctx context.Context, source, cacheDir string, seen map[string]struct{}, depth int) ([]string, error) { + if source == "" { + return nil, nil + } + if depth > MaxNestedSources { + return nil, fmt.Errorf("source nesting exceeds %d levels", MaxNestedSources) + } + entries, err := readSource(ctx, source, cacheDir, seen) + if err != nil { + return nil, err + } + result := make([]string, 0, len(entries)) + for _, entry := range entries { + if isURL(entry) { + nested, err := load(ctx, entry, cacheDir, seen, depth+1) + if err != nil { + return nil, err + } + result = append(result, nested...) + continue + } + if !Validate(entry) { + return nil, fmt.Errorf("invalid entry %q: must be an IP address, CIDR range, domain name, or HTTP(S) URL", entry) + } + result = append(result, entry) + } + return result, nil +} + +func isURL(s string) bool { + parsed, err := url.Parse(s) + if err != nil { + return false + } + return parsed.Scheme == "http" || parsed.Scheme == "https" +} + +// Validate reports whether s is a valid IP address, CIDR range, or domain +// name. URL entries (HTTP(S) sources to be expanded) are not considered +// valid by this function — they are handled separately by Load. +func Validate(s string) bool { + if s == "" { + return false + } + // IP address (IPv4 or IPv6). + if net.ParseIP(s) != nil { + return true + } + // CIDR range. + if _, _, err := net.ParseCIDR(s); err == nil { + return true + } + // Domain name. + return isDomain(s) +} + +// isDomain reports whether s looks like a domain name: it must contain at +// least one dot, have no scheme, and consist of valid DNS label characters. +func isDomain(s string) bool { + if s == "" || !strings.Contains(s, ".") { + return false + } + parsed, err := url.Parse(s) + if err != nil { + return false + } + // Reject anything with a scheme — URLs are handled separately. + if parsed.Scheme != "" { + return false + } + // Reject whitespace, slashes, colons, and other characters that don't + // belong in a bare domain name. + if strings.ContainsAny(s, " \t\r\n/:@?") { + return false + } + // Each label must be non-empty and contain only letters, digits, and + // hyphens (RFC 1035). + labels := strings.Split(s, ".") + for _, label := range labels { + if label == "" { + return false + } + for _, r := range label { + if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-') { + return false + } + } + } + return true +} + +func readSource(ctx context.Context, source, cacheDir string, seen map[string]struct{}) ([]string, error) { + parsed, err := url.Parse(source) + if err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { + if docid, gid := gsheet2csv.ParseIDs(source); docid != "" { + source = gsheet2csv.ToCSVURL(docid, gid) + parsed, err = url.Parse(source) + } + } + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { + body, err := os.ReadFile(source) + if err != nil { + return nil, fmt.Errorf("read source %q: %w", source, err) + } + return Parse(bytes.NewReader(body)) + } + key := parsed.String() + if _, ok := seen[key]; ok { + return nil, fmt.Errorf("source cycle detected at %q", safeURL(source)) + } + seen[key] = struct{}{} + defer delete(seen, key) + + // Collect headers: Basic Auth from URL userinfo plus Accept for + // content negotiation. + header := make(http.Header) + if parsed.User != nil { + password, _ := parsed.User.Password() + header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(parsed.User.Username()+":"+password))) + parsed.User = nil + } + // Prefer TSV but accept CSV and generic content. + header.Set("Accept", "text/tab-separated-values, text/csv;q=0.5, */*;q=0.1") + + digest := sha256.Sum256([]byte(key)) + cachePath := filepath.Join(cacheDir, "ip-sources", hex.EncodeToString(digest[:8])+".tsv") + + // Ensure the cache directory exists before httpcache tries to write. + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + return nil, fmt.Errorf("create cache dir for %s: %w", safeURL(source), err) + } + + // previous is the last-known-good cache content. If the fetch fails or + // the new content can't be parsed, we fall back to (or restore) it. + previous, previousErr := os.ReadFile(cachePath) + + cache := httpcache.NewWith(parsed.String(), cachePath, https.NewInternalClient()) + cache.Header = header + cache.MaxAge = defaultMaxAge + cache.MaxBytes = defaultMaxBytes + if _, fetchErr := cache.Fetch(ctx); fetchErr != nil { + // No previous cache to fall back on — propagate the fetch error. + if previousErr != nil { + return nil, fmt.Errorf("fetch %s: %w", safeURL(source), fetchErr) + } + // Previous cache exists — fall through and read it instead. + } + body, err := os.ReadFile(cachePath) + if err != nil { + return nil, fmt.Errorf("read cached source %s: %w", safeURL(source), err) + } + entries, err := Parse(bytes.NewReader(body)) + if err != nil { + // Fetch succeeded but produced unparseable content. If we had a + // previous cache, restore it so the next call can retry cleanly. + // On the first-ever run (no previous cache) there is nothing to + // restore, so we just return the parse error. + if previousErr == nil { + _ = os.WriteFile(cachePath, previous, 0o600) + _ = os.Remove(cachePath + ".meta") + } + return nil, fmt.Errorf("parse cached source %s: %w", safeURL(source), err) + } + return entries, nil +} + +// Parse reads TSV or CSV data from r and returns the first column of each +// row. Blank lines, comments (lines starting with #), and an optional +// "network" header on the first line are ignored. Other columns are +// treated as labels and discarded. +// +// The format is auto-detected: if the first non-empty, non-comment line +// contains a tab character, TSV is assumed; otherwise CSV is used. +func Parse(r io.Reader) ([]string, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("read input: %w", err) + } + if isTSV(data) { + return parseTSV(bytes.NewReader(data)) + } + return parseCSV(bytes.NewReader(data)) +} + +// isTSV heuristically detects TSV format by checking whether the first +// non-empty, non-comment line contains a tab character. +func isTSV(data []byte) bool { + for _, line := range bytes.Split(data, []byte("\n")) { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 || bytes.HasPrefix(trimmed, []byte("#")) { + continue + } + return bytes.ContainsRune(line, '\t') + } + return false +} + +func parseTSV(r io.Reader) ([]string, error) { + scanner := bufio.NewScanner(r) + entries := make([]string, 0) + for line := 1; scanner.Scan(); line++ { + col, _, _ := strings.Cut(scanner.Text(), "\t") + value := strings.TrimSpace(col) + if value == "" || strings.HasPrefix(value, "#") || (line == 1 && strings.EqualFold(value, "network")) { + continue + } + entries = append(entries, value) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return entries, nil +} + +func parseCSV(r io.Reader) ([]string, error) { + cr := csv.NewReader(r) + cr.FieldsPerRecord = -1 // allow variable column counts + cr.Comment = '#' + records, err := cr.ReadAll() + if err != nil { + return nil, err + } + entries := make([]string, 0, len(records)) + for i, record := range records { + if len(record) == 0 { + continue + } + value := strings.TrimSpace(record[0]) + if value == "" || (i == 0 && strings.EqualFold(value, "network")) { + continue + } + entries = append(entries, value) + } + return entries, nil +} + +func safeURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + parsed.User = nil + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String() +} diff --git a/net/iplist/iplist_test.go b/net/iplist/iplist_test.go new file mode 100644 index 0000000..272ec3f --- /dev/null +++ b/net/iplist/iplist_test.go @@ -0,0 +1,122 @@ +package iplist + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseTSV(t *testing.T) { + got, err := Parse(strings.NewReader("network\tnote\n# comment\n192.0.2.1\tlocal\nexample.com\n")) + if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" { + t.Fatalf("entries=%#v err=%v", got, err) + } +} + +func TestParseCSV(t *testing.T) { + got, err := Parse(strings.NewReader("network,note\n# comment\n192.0.2.1,local\nexample.com\n")) + if err != nil || len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "example.com" { + t.Fatalf("entries=%#v err=%v", got, err) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + entry string + want bool + }{ + {"192.0.2.1", true}, + {"2001:db8::1", true}, + {"10.0.0.0/8", true}, + {"2001:db8::/32", true}, + {"example.com", true}, + {"sub.example.com", true}, + {"", false}, + {"not valid", false}, + {"has space.com", false}, + {"/missing/domain", false}, + {"http://example.com", false}, // URLs are not valid entries + } + + for _, tt := range tests { + if got := Validate(tt.entry); got != tt.want { + t.Errorf("Validate(%q) = %v, want %v", tt.entry, got, tt.want) + } + } +} + +func TestLoadRejectsInvalidEntry(t *testing.T) { + dir := t.TempDir() + // Write a local file with an invalid entry. + path := filepath.Join(dir, "bad.tsv") + if err := os.WriteFile(path, []byte("192.0.2.1\nnot valid entry\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := Load(t.Context(), path, dir) + if err == nil { + t.Fatal("expected error for invalid entry, got nil") + } +} + +func TestLoadURLAuthCacheAndNested(t *testing.T) { + var authOK bool + inner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("192.0.2.2\n")) + })) + defer inner.Close() + outer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authOK = r.Header.Get("Authorization") == "Basic dTpw" + _, _ = w.Write([]byte("network\n" + inner.URL + "\n")) + })) + defer outer.Close() + + cacheDir := t.TempDir() + got, err := Load(t.Context(), strings.Replace(outer.URL, "http://", "http://u:p@", 1), cacheDir) + if err != nil || !authOK || len(got) != 1 || got[0] != "192.0.2.2" { + t.Fatalf("auth=%v entries=%#v err=%v", authOK, got, err) + } + matches, err := filepath.Glob(filepath.Join(cacheDir, "ip-sources", "*.tsv")) + if err != nil || len(matches) != 2 { + t.Fatalf("cache files=%v err=%v", matches, err) + } + if _, err := os.Stat(matches[0] + ".meta"); err != nil { + t.Fatal(err) + } +} + +func TestLoadAcceptHeader(t *testing.T) { + var acceptOK bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + acceptOK = strings.Contains(r.Header.Get("Accept"), "text/tab-separated-values") + _, _ = w.Write([]byte("192.0.2.1\n")) + })) + defer srv.Close() + + cacheDir := t.TempDir() + if _, err := Load(t.Context(), srv.URL, cacheDir); err != nil { + t.Fatal(err) + } + if !acceptOK { + t.Fatal("expected Accept header to contain text/tab-separated-values") + } +} + +func TestLoadCSVDetection(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/csv") + _, _ = w.Write([]byte("network,note\n192.0.2.1,local\n10.0.0.0/8,internal\n")) + })) + defer srv.Close() + + cacheDir := t.TempDir() + got, err := Load(t.Context(), srv.URL, cacheDir) + if err != nil { + t.Fatalf("err=%v", err) + } + if len(got) != 2 || got[0] != "192.0.2.1" || got[1] != "10.0.0.0/8" { + t.Fatalf("entries=%#v", got) + } +}