From b43d378a66f57e9406707f0dfcd1f587130e48f4 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 30 Apr 2026 16:48:44 -0600 Subject: [PATCH] feat(net/httpcache): ETag/Last-Modified HTTP cache for periodic data New package for fetching files from HTTP with conditional-GET semantics and a sidecar metadata file that persists ETag/Last-Modified across restarts. Splits connection vs download timeouts, supports custom auth headers (stripped on cross-origin redirect), and exposes sentinel errors for callers branching on failure mode. Integration tests included. --- net/httpcache/go.mod | 3 + net/httpcache/httpcache.go | 247 ++++++++++++++++++++ net/httpcache/httpcache_integration_test.go | 145 ++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 net/httpcache/go.mod create mode 100644 net/httpcache/httpcache.go create mode 100644 net/httpcache/httpcache_integration_test.go diff --git a/net/httpcache/go.mod b/net/httpcache/go.mod new file mode 100644 index 0000000..b8157ee --- /dev/null +++ b/net/httpcache/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/httpcache + +go 1.26.0 diff --git a/net/httpcache/httpcache.go b/net/httpcache/httpcache.go new file mode 100644 index 0000000..952b537 --- /dev/null +++ b/net/httpcache/httpcache.go @@ -0,0 +1,247 @@ +package httpcache + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +// Sentinel errors returned by Fetch; wrap with errors.Is to branch on the +// failure mode. The wrapped error always includes the URL or Path context. +var ( + // ErrUnexpectedStatus is returned when the server replies with a + // non-200, non-304 response. + ErrUnexpectedStatus = errors.New("unexpected response status") + + // ErrEmptyResponse is returned when a 200 response body is zero bytes. + ErrEmptyResponse = errors.New("empty response body") + + // ErrSaveMeta is returned when the .meta sidecar cannot be written + // after a successful body download (updated is still true). + ErrSaveMeta = errors.New("save meta sidecar") +) + +// BasicAuth returns an HTTP Basic Authorization header value: +// "Basic " + base64(user:pass). Pair with the "Authorization" header in +// Cacher.Header. +func BasicAuth(user, pass string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass)) +} + +// Bearer returns a Bearer Authorization header value: "Bearer " + token. +// Pair with the "Authorization" header in Cacher.Header. +func Bearer(token string) string { + return "Bearer " + token +} + +const ( + defaultConnTimeout = 5 * time.Second // TCP connect + TLS handshake + defaultTimeout = 5 * time.Minute // overall including body read +) + + +// Cacher fetches a URL to a local file, using ETag/Last-Modified to skip +// unchanged responses. +// +// Rate limiting — two independent gates, both checked before any HTTP: +// - MaxAge: skips if the local file's mtime is within this duration. +// Useful when the remote preserves meaningful timestamps (e.g. MaxMind +// encodes the database release date as the tar entry mtime). +// - MinInterval: skips if Fetch was called within this duration (in-memory). +// Guards against tight poll loops hammering a rate-limited API. +// +// Caching — ETag and Last-Modified values are persisted to a .meta +// sidecar file so conditional GETs survive process restarts. +// +// Header — any values in Header are sent on every request. Authorization +// headers are stripped before following redirects so presigned targets +// (e.g. S3/R2 URLs) never receive credentials. The BasicAuth and Bearer +// helpers produce Authorization values for the common cases. +type Cacher struct { + URL string + Path string + ConnTimeout time.Duration // 0 uses 5s; caps TCP connect + TLS handshake + Timeout time.Duration // 0 uses 5m; caps overall request including body read + MaxAge time.Duration // 0 disables; skip HTTP if file mtime is within this + MinInterval time.Duration // 0 disables; skip HTTP if last Fetch attempt was within this + Header http.Header // headers sent on every request (Authorization is stripped on redirect) + + mu sync.Mutex + etag string + lastMod string + lastChecked time.Time + metaLoaded bool +} + +// cacheMeta is the sidecar format persisted alongside the downloaded file. +type cacheMeta struct { + ETag string `json:"etag,omitempty"` + LastMod string `json:"last_modified,omitempty"` +} + +func (c *Cacher) metaPath() string { return c.Path + ".meta" } + +// loadMeta reads etag/lastMod from the sidecar file. Errors are silently +// ignored — a missing or corrupt sidecar just means a full download next time. +func (c *Cacher) loadMeta() { + data, err := os.ReadFile(c.metaPath()) + if err != nil { + return + } + var m cacheMeta + if err := json.Unmarshal(data, &m); err != nil { + return + } + c.etag = m.ETag + c.lastMod = m.LastMod +} + +// saveMeta writes etag/lastMod to the sidecar file atomically. +func (c *Cacher) saveMeta() error { + m := cacheMeta{ETag: c.etag, LastMod: c.lastMod} + data, err := json.Marshal(m) + if err != nil { + return err + } + tmp := c.metaPath() + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + if err := os.Rename(tmp, c.metaPath()); err != nil { + _ = os.Remove(tmp) + return err + } + return nil +} + +// New creates a Cacher that fetches URL and writes it to path. +func New(url, path string) *Cacher { + return &Cacher{URL: url, Path: path} +} + +// Fetch sends a conditional GET and writes new content to Path if the server +// responds with 200. Returns whether the file was updated. +// +// Both MaxAge and MinInterval are checked before making any HTTP request. +func (c *Cacher) Fetch() (updated bool, err error) { + // MaxAge: file-mtime gate (no lock needed — just a stat). + if c.MaxAge > 0 { + if info, err := os.Stat(c.Path); err == nil { + if time.Since(info.ModTime()) < c.MaxAge { + return false, nil + } + } + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Load sidecar once so conditional GETs work after a process restart. + if !c.metaLoaded { + c.loadMeta() + c.metaLoaded = true + } + + // MinInterval: in-memory last-checked gate. + if c.MinInterval > 0 && !c.lastChecked.IsZero() { + if time.Since(c.lastChecked) < c.MinInterval { + return false, nil + } + } + c.lastChecked = time.Now() + + connTimeout := c.ConnTimeout + if connTimeout == 0 { + connTimeout = defaultConnTimeout + } + timeout := c.Timeout + if timeout == 0 { + timeout = defaultTimeout + } + + req, err := http.NewRequest(http.MethodGet, c.URL, nil) + if err != nil { + return false, err + } + + if c.etag != "" { + req.Header.Set("If-None-Match", c.etag) + } else if c.lastMod != "" { + req.Header.Set("If-Modified-Since", c.lastMod) + } + + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: connTimeout}).DialContext, + TLSHandshakeTimeout: connTimeout, + } + + for k, vs := range c.Header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + + client := &http.Client{Timeout: timeout, Transport: transport} + // Strip Authorization before following any redirect — redirect targets + // (e.g. presigned S3/R2 URLs) must not receive our credentials. + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + req.Header.Del("Authorization") + return nil + } + + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotModified { + return false, nil + } + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("%w %d fetching %s", ErrUnexpectedStatus, resp.StatusCode, c.URL) + } + + if err := os.MkdirAll(filepath.Dir(c.Path), 0o755); err != nil { + return false, err + } + tmp := c.Path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return false, err + } + n, err := io.Copy(f, resp.Body) + f.Close() + if err != nil { + os.Remove(tmp) + return false, err + } + if n == 0 { + os.Remove(tmp) + return false, fmt.Errorf("%w from %s", ErrEmptyResponse, c.URL) + } + if err := os.Rename(tmp, c.Path); err != nil { + os.Remove(tmp) + return false, err + } + + if etag := resp.Header.Get("ETag"); etag != "" { + c.etag = etag + } + if lm := resp.Header.Get("Last-Modified"); lm != "" { + c.lastMod = lm + } + if err := c.saveMeta(); err != nil { + return true, fmt.Errorf("%w for %s: %w", ErrSaveMeta, c.Path, err) + } + + return true, nil +} diff --git a/net/httpcache/httpcache_integration_test.go b/net/httpcache/httpcache_integration_test.go new file mode 100644 index 0000000..ff5a7b1 --- /dev/null +++ b/net/httpcache/httpcache_integration_test.go @@ -0,0 +1,145 @@ +//go:build integration + +package httpcache_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/httpcache" +) + +var inboundSources = []struct { + name string + url string +}{ + { + "inbound_single_ips", + "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/single_ips.txt", + }, + { + "inbound_networks", + "https://github.com/bitwire-it/ipblocklist/raw/refs/heads/main/tables/inbound/networks.txt", + }, +} + +func testdataDir(t *testing.T) string { + t.Helper() + dir, _ := filepath.Abs(".") + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata") + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find module root (go.mod)") + } + dir = parent + } +} + +func TestCacher_Download(t *testing.T) { + for _, src := range inboundSources { + t.Run(src.name, func(t *testing.T) { + path := filepath.Join(testdataDir(t), src.name+".txt") + os.Remove(path) + os.Remove(path + ".meta") + + c := httpcache.New(src.url, path) + updated, err := c.Fetch() + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if !updated { + t.Error("first Fetch: expected updated=true") + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("file not created: %v", err) + } + if info.Size() == 0 { + t.Error("downloaded file is empty") + } + t.Logf("downloaded %d bytes to %s", info.Size(), path) + }) + } +} + +func TestCacher_SidecarWritten(t *testing.T) { + for _, src := range inboundSources { + t.Run(src.name, func(t *testing.T) { + path := filepath.Join(testdataDir(t), src.name+".txt") + os.Remove(path) + os.Remove(path + ".meta") + + c := httpcache.New(src.url, path) + if _, err := c.Fetch(); err != nil { + t.Fatalf("Fetch: %v", err) + } + + data, err := os.ReadFile(path + ".meta") + if err != nil { + t.Fatalf("sidecar not written: %v", err) + } + var meta map[string]string + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("sidecar not valid JSON: %v", err) + } + if meta["etag"] == "" && meta["last_modified"] == "" { + t.Error("sidecar has neither etag nor last_modified") + } + t.Logf("sidecar: %s", data) + }) + } +} + +func TestCacher_ConditionalGet_SameCacher(t *testing.T) { + for _, src := range inboundSources { + t.Run(src.name, func(t *testing.T) { + path := filepath.Join(testdataDir(t), src.name+".txt") + + c := httpcache.New(src.url, path) + if _, err := c.Fetch(); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + + updated, err := c.Fetch() + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if updated { + t.Error("second Fetch on same cacher: expected updated=false") + } + t.Log("same-cacher conditional GET correctly skipped re-download") + }) + } +} + +func TestCacher_ConditionalGet_FreshCacher(t *testing.T) { + for _, src := range inboundSources { + t.Run(src.name, func(t *testing.T) { + path := filepath.Join(testdataDir(t), src.name+".txt") + + first := httpcache.New(src.url, path) + if _, err := first.Fetch(); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + if _, err := os.Stat(path + ".meta"); err != nil { + t.Fatalf("sidecar missing after first fetch: %v", err) + } + + // New Cacher with no in-memory state — must read sidecar. + fresh := httpcache.New(src.url, path) + updated, err := fresh.Fetch() + if err != nil { + t.Fatalf("fresh-cacher Fetch: %v", err) + } + if updated { + t.Error("fresh-cacher Fetch: expected updated=false (sidecar should have provided ETag)") + } + t.Log("fresh-cacher conditional GET correctly used sidecar ETag") + }) + } +}