feat(net/httpcache): ETag/Last-Modified HTTP cache

Conditional GET cache for periodic data sources. Persists ETag and
Last-Modified to a .meta sidecar so 304s survive process restarts.
Two independent rate-limit gates (MaxAge file-mtime, MinInterval
in-memory). Body cap via MaxBytes defends against fill-disk from a
hostile upstream or redirect target. Authorization (and any
caller-supplied) headers stripped on every redirect hop. Userinfo
in URL is redacted from error messages.

net/httpcache/README.md

@@ -0,0 +1,44 @@
+# [httpcache](https://github.com/therootcompany/golib/tree/main/net/httpcache)
+
+Conditional HTTP GET to a local file. ETag / Last-Modified are persisted to a
+`<path>.meta` sidecar so 304s survive process restarts.
+
+- skips HTTP when the local file is newer than `MaxAge`
+- skips HTTP when the last attempt was within `MinInterval` (in-memory)
+- caps response bodies via `MaxBytes` (defends against fill-disk)
+- `O_EXCL` reservation on `<path>.tmp` for cross-process exclusion;
+  `singleflight` coalesces in-process callers
+- safe to call `Fetch` concurrently — peers see `ErrPeerFetching`
+
+## Example
+
+Fetch a blocklist; re-check periodically:
+
+```go
+c := httpcache.New(
+    "https://example.com/blocklist.txt",
+    "/srv/data/blocklist.txt",
+)
+c.MaxAge = time.Hour       // skip HTTP if file is < 1h old
+c.MaxBytes = 50 << 20      // 50 MiB cap
+
+updated, err := c.Fetch(ctx)
+switch {
+case errors.Is(err, httpcache.ErrPeerFetching):
+    // another process is downloading; reload from disk if updated==true
+case err != nil:
+    log.Fatalf("fetch: %v", err)
+case updated:
+    log.Printf("blocklist refreshed")
+}
+```
+
+Pair with `*http.Client` if you need custom timeouts, transport, or a
+`CheckRedirect` that strips non-standard credential headers (the stdlib
+already strips `Authorization` / `Cookie` / `WWW-Authenticate` on
+cross-host redirects):
+
+```go
+c := httpcache.NewWith(url, path, &http.Client{Timeout: 30 * time.Second})
+c.Header = http.Header{"Authorization": []string{"Bearer " + token}}
+```

net/httpcache/go.mod

@@ -0,0 +1,5 @@
+module github.com/therootcompany/golib/net/httpcache
+
+go 1.26.0
+
+require golang.org/x/sync v0.20.0

net/httpcache/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=

net/httpcache/httpcache.go

@@ -0,0 +1,303 @@
+package httpcache
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+)
+
+// Sentinel errors returned by Fetch; wrap with errors.Is to branch on the
+// failure mode. The wrapped error always includes the URL or Path context.
+var (
+	// ErrUnexpectedStatus is returned when the server replies with a
+	// non-200, non-304 response.
+	ErrUnexpectedStatus = errors.New("unexpected response status")
+
+	// ErrEmptyResponse is returned when a 200 response body is zero bytes.
+	ErrEmptyResponse = errors.New("empty response body")
+
+	// ErrSaveMeta is returned when the .meta sidecar cannot be written
+	// after a successful body download (updated is still true).
+	ErrSaveMeta = errors.New("save meta sidecar")
+
+	// ErrBodyTooLarge is returned when the response body exceeds MaxBytes.
+	ErrBodyTooLarge = errors.New("response body exceeds MaxBytes")
+
+	// ErrPeerFetching is returned when another process holds the .tmp
+	// download file (O_EXCL conflict). The returned `updated` is true if
+	// the peer has installed a different version since our last known
+	// state — callers can reload Path from disk and skip their own fetch.
+	ErrPeerFetching = errors.New("another process is fetching")
+)
+
+const (
+	defaultConnTimeout = 5 * time.Second // TCP connect + TLS handshake
+	defaultTimeout     = 5 * time.Minute // overall including body read
+)
+
+// Cacher fetches a URL to a local file, using ETag/Last-Modified to skip
+// unchanged responses.
+//
+// Rate limiting — two independent gates, both checked before any HTTP:
+//   - MaxAge: skips if the local file's mtime is within this duration.
+//     Useful when the remote preserves meaningful timestamps (e.g. MaxMind
+//     encodes the database release date as the tar entry mtime).
+//   - MinInterval: skips if Fetch was called within this duration (in-memory).
+//     Guards against tight poll loops hammering a rate-limited API.
+//
+// Caching — ETag and Last-Modified values are persisted to a <path>.meta
+// sidecar file so conditional GETs survive process restarts.
+//
+// Header — any values in Header are sent on every request. The stdlib
+// http.Client strips Authorization, WWW-Authenticate, and Cookie on
+// cross-host redirects; custom-named credential headers (X-API-Key,
+// PRIVATE-TOKEN, …) are forwarded. If you set those, supply your own
+// *http.Client with a CheckRedirect that strips them.
+//
+// MaxBytes — caps the response body. A hostile or compromised upstream
+// (or a redirect target on another origin) can otherwise stream until the
+// disk fills, since the overall Timeout still allows multi-GB transfers.
+// 0 disables the cap.
+type Cacher struct {
+	URL         string
+	Path        string
+	MaxAge      time.Duration // 0 disables; skip HTTP if file mtime is within this
+	MinInterval time.Duration // 0 disables; skip HTTP if last Fetch attempt was within this
+	MaxBytes    int64         // 0 disables; cap on body bytes read per Fetch (defends against fill-disk)
+	Header      http.Header   // headers sent on every request
+
+	sf          singleflight.Group
+	cacheMeta   // embedded: etag/lastMod persisted to sidecar
+	lastChecked time.Time
+	metaLoaded  bool
+	client      *http.Client // set by New/NewWith; reuses connections across Fetch calls
+}
+
+// cacheMeta is the sidecar format persisted alongside the downloaded file.
+type cacheMeta struct {
+	ETag    string `json:"etag,omitempty"`
+	LastMod string `json:"last_modified,omitempty"`
+}
+
+func (c *Cacher) metaPath() string { return c.Path + ".meta" }
+
+// safeURL returns c.URL with any userinfo (user:password@) stripped, so
+// errors and logs don't leak credentials embedded in the URL. Falls back
+// to "<unparseable URL>" rather than echoing the raw value if parsing
+// fails (defensive — a URL we can't parse may itself be a credential).
+func (c *Cacher) safeURL() string {
+	u, err := url.Parse(c.URL)
+	if err != nil {
+		return "<unparseable URL>"
+	}
+	u.User = nil
+	return u.String()
+}
+
+// loadMeta reads etag/lastMod from the sidecar file. A missing sidecar is
+// not an error (just means a full download next time); read or parse errors
+// propagate so the caller can decide whether to abort or proceed.
+func (c *Cacher) loadMeta() error {
+	data, err := os.ReadFile(c.metaPath())
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
+	if err := json.Unmarshal(data, &c.cacheMeta); err != nil {
+		return fmt.Errorf("parse %s: %w", c.metaPath(), err)
+	}
+	return nil
+}
+
+// saveMeta writes etag/lastMod to the sidecar file atomically.
+func (c *Cacher) saveMeta() error {
+	data, err := json.Marshal(c.cacheMeta)
+	if err != nil {
+		return err
+	}
+	tmp := c.metaPath() + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o600); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, c.metaPath()); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return nil
+}
+
+// New creates a Cacher with a default *http.Client (5s connect, 5m overall).
+// Equivalent to NewWith(url, path, nil). Header is read live on every request,
+// so it may be set or modified after New.
+func New(url, path string) *Cacher {
+	return NewWith(url, path, nil)
+}
+
+// NewWith creates a Cacher backed by the given *http.Client; pass nil for
+// defaults. The client is used as-is — its Timeout, Transport, Jar, and
+// CheckRedirect are all the caller's responsibility.
+func NewWith(url, path string, client *http.Client) *Cacher {
+	if client == nil {
+		client = &http.Client{
+			Timeout: defaultTimeout,
+			Transport: &http.Transport{
+				DialContext:         (&net.Dialer{Timeout: defaultConnTimeout}).DialContext,
+				TLSHandshakeTimeout: defaultConnTimeout,
+			},
+		}
+	}
+	return &Cacher{URL: url, Path: path, client: client}
+}
+
+// Fetch sends a conditional GET and writes new content to Path if the server
+// responds with 200. Returns whether the file was updated.
+//
+// Both MaxAge and MinInterval are checked before making any HTTP request.
+// ctx cancels the in-flight request and any blocking body read.
+//
+// Safe to call concurrently — concurrent callers share a single in-flight
+// fetch (via singleflight) and all receive the same result.
+func (c *Cacher) Fetch(ctx context.Context) (updated bool, err error) {
+	// MaxAge: file-mtime gate (no lock needed — just a stat).
+	if c.MaxAge > 0 {
+		if info, err := os.Stat(c.Path); err == nil {
+			if time.Since(info.ModTime()) < c.MaxAge {
+				return false, nil
+			}
+		}
+	}
+
+	type result struct {
+		updated bool
+		err     error
+	}
+	v, _, _ := c.sf.Do("fetch", func() (any, error) {
+		u, err := c.fetch(ctx)
+		return result{u, err}, nil
+	})
+	r := v.(result)
+	return r.updated, r.err
+}
+
+// fetch is the inner serialized work. singleflight ensures only one runs at
+// a time, so the cacheMeta/lastChecked/metaLoaded fields don't need a mutex.
+func (c *Cacher) fetch(ctx context.Context) (bool, error) {
+	// Load sidecar once so conditional GETs work after a process restart.
+	if !c.metaLoaded {
+		if err := c.loadMeta(); err != nil {
+			return false, err
+		}
+		c.metaLoaded = true
+	}
+
+	// MinInterval: in-memory last-checked gate.
+	if c.MinInterval > 0 && !c.lastChecked.IsZero() {
+		if time.Since(c.lastChecked) < c.MinInterval {
+			return false, nil
+		}
+	}
+	c.lastChecked = time.Now()
+
+	// Reserve .tmp before any HTTP — O_EXCL gives us cross-process
+	// exclusion (singleflight only covers the in-process case). A peer
+	// holding .tmp means another process is mid-download; return early
+	// without hitting the wire.
+	if err := os.MkdirAll(filepath.Dir(c.Path), 0o755); err != nil {
+		return false, err
+	}
+	tmp := c.Path + ".tmp"
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		if errors.Is(err, fs.ErrExist) {
+			// Peer holds .tmp. Re-read sidecar; if peer has installed a
+			// different version, signal updated=true so the caller can
+			// reload Path from disk.
+			prevETag, prevLM := c.ETag, c.LastMod
+			if err := c.loadMeta(); err != nil {
+				return false, err
+			}
+			updated := c.ETag != prevETag || c.LastMod != prevLM
+			return updated, fmt.Errorf("%w for %s", ErrPeerFetching, c.safeURL())
+		}
+		return false, err
+	}
+	// Any non-success path from here must remove tmp so peers aren't blocked.
+	defer func() {
+		_ = f.Close()
+		_ = os.Remove(tmp)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.URL, nil)
+	if err != nil {
+		return false, err
+	}
+
+	if c.ETag != "" {
+		req.Header.Set("If-None-Match", c.ETag)
+	} else if c.LastMod != "" {
+		req.Header.Set("If-Modified-Since", c.LastMod)
+	}
+
+	for k, vs := range c.Header {
+		for _, v := range vs {
+			req.Header.Add(k, v)
+		}
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode == http.StatusNotModified {
+		return false, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return false, fmt.Errorf("%w %d fetching %s", ErrUnexpectedStatus, resp.StatusCode, c.safeURL())
+	}
+
+	body := io.Reader(resp.Body)
+	if c.MaxBytes > 0 {
+		// +1 so n>MaxBytes signals "exceeded" without an extra Read.
+		body = io.LimitReader(resp.Body, c.MaxBytes+1)
+	}
+	n, err := io.Copy(f, body)
+	if err != nil {
+		return false, err
+	}
+	if c.MaxBytes > 0 && n > c.MaxBytes {
+		return false, fmt.Errorf("%w (%d > %d) from %s", ErrBodyTooLarge, n, c.MaxBytes, c.safeURL())
+	}
+	if n == 0 {
+		return false, fmt.Errorf("%w from %s", ErrEmptyResponse, c.safeURL())
+	}
+	if err := os.Rename(tmp, c.Path); err != nil {
+		return false, err
+	}
+
+	if etag := resp.Header.Get("ETag"); etag != "" {
+		c.ETag = etag
+	}
+	if lm := resp.Header.Get("Last-Modified"); lm != "" {
+		c.LastMod = lm
+	}
+	if err := c.saveMeta(); err != nil {
+		return true, fmt.Errorf("%w for %s: %w", ErrSaveMeta, c.Path, err)
+	}
+
+	return true, nil
+}

net/httpcache/httpcache_test.go

@@ -0,0 +1,227 @@
+package httpcache_test
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+// fakeServer serves body with a fixed ETag and honors If-None-Match.
+// hits counts how many requests reached the handler, including 304s.
+type fakeServer struct {
+	body []byte
+	etag string
+	hits atomic.Int32
+}
+
+func (f *fakeServer) handler(w http.ResponseWriter, r *http.Request) {
+	f.hits.Add(1)
+	if r.Header.Get("If-None-Match") == f.etag {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+	w.Header().Set("ETag", f.etag)
+	w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT")
+	_, _ = w.Write(f.body)
+}
+
+func TestCacher_Download(t *testing.T) {
+	fs := &fakeServer{body: []byte("hello blocklist\n"), etag: `"abc123"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+
+	updated, err := c.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if !updated {
+		t.Error("first Fetch: expected updated=true")
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if string(got) != string(fs.body) {
+		t.Errorf("body = %q, want %q", got, fs.body)
+	}
+}
+
+func TestCacher_SidecarWritten(t *testing.T) {
+	fs := &fakeServer{body: []byte("x"), etag: `"sidecar-etag"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	if _, err := c.Fetch(t.Context()); err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+
+	data, err := os.ReadFile(path + ".meta")
+	if err != nil {
+		t.Fatalf("sidecar not written: %v", err)
+	}
+	var meta map[string]string
+	if err := json.Unmarshal(data, &meta); err != nil {
+		t.Fatalf("sidecar not valid JSON: %v", err)
+	}
+	if meta["etag"] != fs.etag {
+		t.Errorf("sidecar etag = %q, want %q", meta["etag"], fs.etag)
+	}
+	if meta["last_modified"] == "" {
+		t.Error("sidecar last_modified empty")
+	}
+}
+
+func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
+	fs := &fakeServer{body: []byte("body"), etag: `"e1"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	if _, err := c.Fetch(t.Context()); err != nil {
+		t.Fatalf("first Fetch: %v", err)
+	}
+
+	updated, err := c.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("second Fetch: %v", err)
+	}
+	if updated {
+		t.Error("second Fetch: expected updated=false (304 path)")
+	}
+	if got := fs.hits.Load(); got != 2 {
+		t.Errorf("server hits = %d, want 2 (both Fetches must reach the wire)", got)
+	}
+}
+
+func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
+	// Fresh Cacher must read the sidecar and send If-None-Match — proves
+	// ETag survives process restart.
+	fs := &fakeServer{body: []byte("body"), etag: `"e2"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	if _, err := httpcache.New(srv.URL, path).Fetch(t.Context()); err != nil {
+		t.Fatalf("seed Fetch: %v", err)
+	}
+
+	fresh := httpcache.New(srv.URL, path)
+	updated, err := fresh.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("fresh Fetch: %v", err)
+	}
+	if updated {
+		t.Error("fresh Fetch: expected updated=false (sidecar should have provided ETag)")
+	}
+}
+
+func TestCacher_UnexpectedStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "nope", http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+
+	c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt"))
+	_, err := c.Fetch(t.Context())
+	if !errors.Is(err, httpcache.ErrUnexpectedStatus) {
+		t.Errorf("err = %v, want ErrUnexpectedStatus", err)
+	}
+}
+
+func TestCacher_EmptyResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt"))
+	_, err := c.Fetch(t.Context())
+	if !errors.Is(err, httpcache.ErrEmptyResponse) {
+		t.Errorf("err = %v, want ErrEmptyResponse", err)
+	}
+}
+
+func TestCacher_MaxBytesExceeded(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("0123456789ABCDEF")) // 16 bytes
+	}))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	c.MaxBytes = 8
+	_, err := c.Fetch(t.Context())
+	if !errors.Is(err, httpcache.ErrBodyTooLarge) {
+		t.Errorf("err = %v, want ErrBodyTooLarge", err)
+	}
+	if _, statErr := os.Stat(path); statErr == nil {
+		t.Error("expected body file not to be present after MaxBytes failure")
+	}
+}
+
+func TestCacher_PeerFetching(t *testing.T) {
+	fs := &fakeServer{body: []byte("body"), etag: `"e1"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+
+	// First Fetch establishes baseline etag in memory and on-disk sidecar.
+	if _, err := c.Fetch(t.Context()); err != nil {
+		t.Fatalf("seed Fetch: %v", err)
+	}
+	hitsAfterSeed := fs.hits.Load()
+
+	// Simulate a peer mid-download: hold .tmp.
+	if err := os.WriteFile(path+".tmp", []byte("partial"), 0o600); err != nil {
+		t.Fatalf("seed tmp: %v", err)
+	}
+	// Simulate peer having just installed a new sidecar (different etag).
+	if err := os.WriteFile(path+".meta", []byte(`{"etag":"\"peer-new\"","last_modified":"Tue, 03 Jan 2006 15:04:05 GMT"}`), 0o600); err != nil {
+		t.Fatalf("seed meta: %v", err)
+	}
+
+	updated, err := c.Fetch(t.Context())
+	if !errors.Is(err, httpcache.ErrPeerFetching) {
+		t.Errorf("err = %v, want ErrPeerFetching", err)
+	}
+	if !updated {
+		t.Error("expected updated=true (peer wrote new sidecar)")
+	}
+	if got := fs.hits.Load(); got != hitsAfterSeed {
+		t.Errorf("server hits = %d, want %d (must not fetch when peer holds tmp)", got, hitsAfterSeed)
+	}
+}
+
+func TestCacher_MaxBytesAtLimit(t *testing.T) {
+	body := []byte("0123456789ABCDEF") // 16 bytes
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(body)
+	}))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	c.MaxBytes = int64(len(body)) // exactly equal — should pass
+	if _, err := c.Fetch(t.Context()); err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	got, _ := os.ReadFile(path)
+	if string(got) != string(body) {
+		t.Errorf("body = %q, want %q", got, body)
+	}
+}