feat(net/httpcache): ETag/Last-Modified HTTP cache for periodic data

New package for fetching files from HTTP with conditional-GET semantics
and a sidecar metadata file that persists ETag/Last-Modified across
restarts. Splits connection vs download timeouts, supports custom auth
headers (stripped on cross-origin redirect), and exposes sentinel errors
for callers branching on failure mode. Integration tests included.

net/httpcache/go.mod

@@ -0,0 +1,3 @@
+module github.com/therootcompany/golib/net/httpcache
+
+go 1.26.0

net/httpcache/httpcache.go

@@ -0,0 +1,247 @@
+package httpcache
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// Sentinel errors returned by Fetch; wrap with errors.Is to branch on the
+// failure mode. The wrapped error always includes the URL or Path context.
+var (
+	// ErrUnexpectedStatus is returned when the server replies with a
+	// non-200, non-304 response.
+	ErrUnexpectedStatus = errors.New("unexpected response status")
+
+	// ErrEmptyResponse is returned when a 200 response body is zero bytes.
+	ErrEmptyResponse = errors.New("empty response body")
+
+	// ErrSaveMeta is returned when the .meta sidecar cannot be written
+	// after a successful body download (updated is still true).
+	ErrSaveMeta = errors.New("save meta sidecar")
+)
+
+// BasicAuth returns an HTTP Basic Authorization header value:
+// "Basic " + base64(user:pass). Pair with the "Authorization" header in
+// Cacher.Header.
+func BasicAuth(user, pass string) string {
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass))
+}
+
+// Bearer returns a Bearer Authorization header value: "Bearer " + token.
+// Pair with the "Authorization" header in Cacher.Header.
+func Bearer(token string) string {
+	return "Bearer " + token
+}
+
+const (
+	defaultConnTimeout = 5 * time.Second // TCP connect + TLS handshake
+	defaultTimeout     = 5 * time.Minute // overall including body read
+)
+
+
+// Cacher fetches a URL to a local file, using ETag/Last-Modified to skip
+// unchanged responses.
+//
+// Rate limiting — two independent gates, both checked before any HTTP:
+//   - MaxAge: skips if the local file's mtime is within this duration.
+//     Useful when the remote preserves meaningful timestamps (e.g. MaxMind
+//     encodes the database release date as the tar entry mtime).
+//   - MinInterval: skips if Fetch was called within this duration (in-memory).
+//     Guards against tight poll loops hammering a rate-limited API.
+//
+// Caching — ETag and Last-Modified values are persisted to a <path>.meta
+// sidecar file so conditional GETs survive process restarts.
+//
+// Header — any values in Header are sent on every request. Authorization
+// headers are stripped before following redirects so presigned targets
+// (e.g. S3/R2 URLs) never receive credentials. The BasicAuth and Bearer
+// helpers produce Authorization values for the common cases.
+type Cacher struct {
+	URL         string
+	Path        string
+	ConnTimeout time.Duration // 0 uses 5s;  caps TCP connect + TLS handshake
+	Timeout     time.Duration // 0 uses 5m;  caps overall request including body read
+	MaxAge      time.Duration // 0 disables; skip HTTP if file mtime is within this
+	MinInterval time.Duration // 0 disables; skip HTTP if last Fetch attempt was within this
+	Header      http.Header   // headers sent on every request (Authorization is stripped on redirect)
+
+	mu          sync.Mutex
+	etag        string
+	lastMod     string
+	lastChecked time.Time
+	metaLoaded  bool
+}
+
+// cacheMeta is the sidecar format persisted alongside the downloaded file.
+type cacheMeta struct {
+	ETag    string `json:"etag,omitempty"`
+	LastMod string `json:"last_modified,omitempty"`
+}
+
+func (c *Cacher) metaPath() string { return c.Path + ".meta" }
+
+// loadMeta reads etag/lastMod from the sidecar file. Errors are silently
+// ignored — a missing or corrupt sidecar just means a full download next time.
+func (c *Cacher) loadMeta() {
+	data, err := os.ReadFile(c.metaPath())
+	if err != nil {
+		return
+	}
+	var m cacheMeta
+	if err := json.Unmarshal(data, &m); err != nil {
+		return
+	}
+	c.etag = m.ETag
+	c.lastMod = m.LastMod
+}
+
+// saveMeta writes etag/lastMod to the sidecar file atomically.
+func (c *Cacher) saveMeta() error {
+	m := cacheMeta{ETag: c.etag, LastMod: c.lastMod}
+	data, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	tmp := c.metaPath() + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, c.metaPath()); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return nil
+}
+
+// New creates a Cacher that fetches URL and writes it to path.
+func New(url, path string) *Cacher {
+	return &Cacher{URL: url, Path: path}
+}
+
+// Fetch sends a conditional GET and writes new content to Path if the server
+// responds with 200. Returns whether the file was updated.
+//
+// Both MaxAge and MinInterval are checked before making any HTTP request.
+func (c *Cacher) Fetch() (updated bool, err error) {
+	// MaxAge: file-mtime gate (no lock needed — just a stat).
+	if c.MaxAge > 0 {
+		if info, err := os.Stat(c.Path); err == nil {
+			if time.Since(info.ModTime()) < c.MaxAge {
+				return false, nil
+			}
+		}
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Load sidecar once so conditional GETs work after a process restart.
+	if !c.metaLoaded {
+		c.loadMeta()
+		c.metaLoaded = true
+	}
+
+	// MinInterval: in-memory last-checked gate.
+	if c.MinInterval > 0 && !c.lastChecked.IsZero() {
+		if time.Since(c.lastChecked) < c.MinInterval {
+			return false, nil
+		}
+	}
+	c.lastChecked = time.Now()
+
+	connTimeout := c.ConnTimeout
+	if connTimeout == 0 {
+		connTimeout = defaultConnTimeout
+	}
+	timeout := c.Timeout
+	if timeout == 0 {
+		timeout = defaultTimeout
+	}
+
+	req, err := http.NewRequest(http.MethodGet, c.URL, nil)
+	if err != nil {
+		return false, err
+	}
+
+	if c.etag != "" {
+		req.Header.Set("If-None-Match", c.etag)
+	} else if c.lastMod != "" {
+		req.Header.Set("If-Modified-Since", c.lastMod)
+	}
+
+	transport := &http.Transport{
+		DialContext:         (&net.Dialer{Timeout: connTimeout}).DialContext,
+		TLSHandshakeTimeout: connTimeout,
+	}
+
+	for k, vs := range c.Header {
+		for _, v := range vs {
+			req.Header.Add(k, v)
+		}
+	}
+
+	client := &http.Client{Timeout: timeout, Transport: transport}
+	// Strip Authorization before following any redirect — redirect targets
+	// (e.g. presigned S3/R2 URLs) must not receive our credentials.
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		req.Header.Del("Authorization")
+		return nil
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotModified {
+		return false, nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return false, fmt.Errorf("%w %d fetching %s", ErrUnexpectedStatus, resp.StatusCode, c.URL)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(c.Path), 0o755); err != nil {
+		return false, err
+	}
+	tmp := c.Path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return false, err
+	}
+	n, err := io.Copy(f, resp.Body)
+	f.Close()
+	if err != nil {
+		os.Remove(tmp)
+		return false, err
+	}
+	if n == 0 {
+		os.Remove(tmp)
+		return false, fmt.Errorf("%w from %s", ErrEmptyResponse, c.URL)
+	}
+	if err := os.Rename(tmp, c.Path); err != nil {
+		os.Remove(tmp)
+		return false, err
+	}
+
+	if etag := resp.Header.Get("ETag"); etag != "" {
+		c.etag = etag
+	}
+	if lm := resp.Header.Get("Last-Modified"); lm != "" {
+		c.lastMod = lm
+	}
+	if err := c.saveMeta(); err != nil {
+		return true, fmt.Errorf("%w for %s: %w", ErrSaveMeta, c.Path, err)
+	}
+
+	return true, nil
+}

net/httpcache/httpcache_test.go

@@ -0,0 +1,209 @@
+package httpcache_test
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+
+	"github.com/therootcompany/golib/net/httpcache"
+)
+
+// fakeServer serves body with a fixed ETag and honors If-None-Match.
+// hits counts how many requests reached the handler, including 304s.
+type fakeServer struct {
+	body []byte
+	etag string
+	hits atomic.Int32
+}
+
+func (f *fakeServer) handler(w http.ResponseWriter, r *http.Request) {
+	f.hits.Add(1)
+	if r.Header.Get("If-None-Match") == f.etag {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+	w.Header().Set("ETag", f.etag)
+	w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT")
+	_, _ = w.Write(f.body)
+}
+
+func TestCacher_Download(t *testing.T) {
+	fs := &fakeServer{body: []byte("hello blocklist\n"), etag: `"abc123"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+
+	updated, err := c.Fetch()
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if !updated {
+		t.Error("first Fetch: expected updated=true")
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if string(got) != string(fs.body) {
+		t.Errorf("body = %q, want %q", got, fs.body)
+	}
+}
+
+func TestCacher_SidecarWritten(t *testing.T) {
+	fs := &fakeServer{body: []byte("x"), etag: `"sidecar-etag"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	if _, err := c.Fetch(); err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+
+	data, err := os.ReadFile(path + ".meta")
+	if err != nil {
+		t.Fatalf("sidecar not written: %v", err)
+	}
+	var meta map[string]string
+	if err := json.Unmarshal(data, &meta); err != nil {
+		t.Fatalf("sidecar not valid JSON: %v", err)
+	}
+	if meta["etag"] != fs.etag {
+		t.Errorf("sidecar etag = %q, want %q", meta["etag"], fs.etag)
+	}
+	if meta["last_modified"] == "" {
+		t.Error("sidecar last_modified empty")
+	}
+}
+
+func TestCacher_ConditionalGet_SameCacher(t *testing.T) {
+	fs := &fakeServer{body: []byte("body"), etag: `"e1"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	if _, err := c.Fetch(); err != nil {
+		t.Fatalf("first Fetch: %v", err)
+	}
+
+	updated, err := c.Fetch()
+	if err != nil {
+		t.Fatalf("second Fetch: %v", err)
+	}
+	if updated {
+		t.Error("second Fetch: expected updated=false (304 path)")
+	}
+	if got := fs.hits.Load(); got != 2 {
+		t.Errorf("server hits = %d, want 2 (both Fetches must reach the wire)", got)
+	}
+}
+
+func TestCacher_ConditionalGet_FreshCacher(t *testing.T) {
+	// Fresh Cacher must read the sidecar and send If-None-Match — proves
+	// ETag survives process restart.
+	fs := &fakeServer{body: []byte("body"), etag: `"e2"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	if _, err := httpcache.New(srv.URL, path).Fetch(); err != nil {
+		t.Fatalf("seed Fetch: %v", err)
+	}
+
+	fresh := httpcache.New(srv.URL, path)
+	updated, err := fresh.Fetch()
+	if err != nil {
+		t.Fatalf("fresh Fetch: %v", err)
+	}
+	if updated {
+		t.Error("fresh Fetch: expected updated=false (sidecar should have provided ETag)")
+	}
+}
+
+func TestCacher_BodyChanges_Redownloads(t *testing.T) {
+	fs := &fakeServer{body: []byte("v1"), etag: `"v1"`}
+	srv := httptest.NewServer(http.HandlerFunc(fs.handler))
+	defer srv.Close()
+
+	path := filepath.Join(t.TempDir(), "data.txt")
+	c := httpcache.New(srv.URL, path)
+	if _, err := c.Fetch(); err != nil {
+		t.Fatalf("initial Fetch: %v", err)
+	}
+
+	fs.body = []byte("v2-longer-body")
+	fs.etag = `"v2"`
+
+	updated, err := c.Fetch()
+	if err != nil {
+		t.Fatalf("re-Fetch: %v", err)
+	}
+	if !updated {
+		t.Error("expected updated=true after server-side change")
+	}
+	got, _ := os.ReadFile(path)
+	if string(got) != "v2-longer-body" {
+		t.Errorf("body = %q, want v2-longer-body", got)
+	}
+}
+
+func TestCacher_UnexpectedStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "nope", http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+
+	c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt"))
+	_, err := c.Fetch()
+	if !errors.Is(err, httpcache.ErrUnexpectedStatus) {
+		t.Errorf("err = %v, want ErrUnexpectedStatus", err)
+	}
+}
+
+func TestCacher_EmptyResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt"))
+	_, err := c.Fetch()
+	if !errors.Is(err, httpcache.ErrEmptyResponse) {
+		t.Errorf("err = %v, want ErrEmptyResponse", err)
+	}
+}
+
+func TestCacher_AuthHeaderStrippedOnRedirect(t *testing.T) {
+	// Final hop should not see Authorization. Initial hop should.
+	var initialAuth, finalAuth string
+	final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		finalAuth = r.Header.Get("Authorization")
+		_, _ = w.Write([]byte("ok"))
+	}))
+	defer final.Close()
+	first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		initialAuth = r.Header.Get("Authorization")
+		http.Redirect(w, r, final.URL, http.StatusFound)
+	}))
+	defer first.Close()
+
+	c := httpcache.New(first.URL, filepath.Join(t.TempDir(), "data.txt"))
+	c.Header = http.Header{"Authorization": []string{"Bearer secret-token"}}
+	if _, err := c.Fetch(); err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if initialAuth != "Bearer secret-token" {
+		t.Errorf("initial Authorization = %q, want Bearer secret-token", initialAuth)
+	}
+	if finalAuth != "" {
+		t.Errorf("final Authorization = %q, want empty (must be stripped on redirect)", finalAuth)
+	}
+}