diff --git a/net/httpcache/README.md b/net/httpcache/README.md new file mode 100644 index 0000000..6d1bbb7 --- /dev/null +++ b/net/httpcache/README.md @@ -0,0 +1,44 @@ +# [httpcache](https://github.com/therootcompany/golib/tree/main/net/httpcache) + +Conditional HTTP GET to a local file. ETag / Last-Modified are persisted to a +`.meta` sidecar so 304s survive process restarts. + +- skips HTTP when the local file is newer than `MaxAge` +- skips HTTP when the last attempt was within `MinInterval` (in-memory) +- caps response bodies via `MaxBytes` (defends against fill-disk) +- `O_EXCL` reservation on `.tmp` for cross-process exclusion; + `singleflight` coalesces in-process callers +- safe to call `Fetch` concurrently — peers see `ErrPeerFetching` + +## Example + +Fetch a blocklist; re-check periodically: + +```go +c := httpcache.New( + "https://example.com/blocklist.txt", + "/srv/data/blocklist.txt", +) +c.MaxAge = time.Hour // skip HTTP if file is < 1h old +c.MaxBytes = 50 << 20 // 50 MiB cap + +updated, err := c.Fetch(ctx) +switch { +case errors.Is(err, httpcache.ErrPeerFetching): + // another process is downloading; reload from disk if updated==true +case err != nil: + log.Fatalf("fetch: %v", err) +case updated: + log.Printf("blocklist refreshed") +} +``` + +Pair with `*http.Client` if you need custom timeouts, transport, or a +`CheckRedirect` that strips non-standard credential headers (the stdlib +already strips `Authorization` / `Cookie` / `WWW-Authenticate` on +cross-host redirects): + +```go +c := httpcache.NewWith(url, path, &http.Client{Timeout: 30 * time.Second}) +c.Header = http.Header{"Authorization": []string{"Bearer " + token}} +``` diff --git a/net/httpcache/go.mod b/net/httpcache/go.mod new file mode 100644 index 0000000..8408a97 --- /dev/null +++ b/net/httpcache/go.mod @@ -0,0 +1,5 @@ +module github.com/therootcompany/golib/net/httpcache + +go 1.26.0 + +require golang.org/x/sync v0.20.0 diff --git a/net/httpcache/go.sum b/net/httpcache/go.sum new file mode 100644 index 0000000..733d716 --- /dev/null +++ b/net/httpcache/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= diff --git a/net/httpcache/httpcache.go b/net/httpcache/httpcache.go new file mode 100644 index 0000000..fcd5b18 --- /dev/null +++ b/net/httpcache/httpcache.go @@ -0,0 +1,303 @@ +package httpcache + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "golang.org/x/sync/singleflight" +) + +// Sentinel errors returned by Fetch; wrap with errors.Is to branch on the +// failure mode. The wrapped error always includes the URL or Path context. +var ( + // ErrUnexpectedStatus is returned when the server replies with a + // non-200, non-304 response. + ErrUnexpectedStatus = errors.New("unexpected response status") + + // ErrEmptyResponse is returned when a 200 response body is zero bytes. + ErrEmptyResponse = errors.New("empty response body") + + // ErrSaveMeta is returned when the .meta sidecar cannot be written + // after a successful body download (updated is still true). + ErrSaveMeta = errors.New("save meta sidecar") + + // ErrBodyTooLarge is returned when the response body exceeds MaxBytes. + ErrBodyTooLarge = errors.New("response body exceeds MaxBytes") + + // ErrPeerFetching is returned when another process holds the .tmp + // download file (O_EXCL conflict). The returned `updated` is true if + // the peer has installed a different version since our last known + // state — callers can reload Path from disk and skip their own fetch. + ErrPeerFetching = errors.New("another process is fetching") +) + +const ( + defaultConnTimeout = 5 * time.Second // TCP connect + TLS handshake + defaultTimeout = 5 * time.Minute // overall including body read +) + +// Cacher fetches a URL to a local file, using ETag/Last-Modified to skip +// unchanged responses. +// +// Rate limiting — two independent gates, both checked before any HTTP: +// - MaxAge: skips if the local file's mtime is within this duration. +// Useful when the remote preserves meaningful timestamps (e.g. MaxMind +// encodes the database release date as the tar entry mtime). +// - MinInterval: skips if Fetch was called within this duration (in-memory). +// Guards against tight poll loops hammering a rate-limited API. +// +// Caching — ETag and Last-Modified values are persisted to a .meta +// sidecar file so conditional GETs survive process restarts. +// +// Header — any values in Header are sent on every request. The stdlib +// http.Client strips Authorization, WWW-Authenticate, and Cookie on +// cross-host redirects; custom-named credential headers (X-API-Key, +// PRIVATE-TOKEN, …) are forwarded. If you set those, supply your own +// *http.Client with a CheckRedirect that strips them. +// +// MaxBytes — caps the response body. A hostile or compromised upstream +// (or a redirect target on another origin) can otherwise stream until the +// disk fills, since the overall Timeout still allows multi-GB transfers. +// 0 disables the cap. +type Cacher struct { + URL string + Path string + MaxAge time.Duration // 0 disables; skip HTTP if file mtime is within this + MinInterval time.Duration // 0 disables; skip HTTP if last Fetch attempt was within this + MaxBytes int64 // 0 disables; cap on body bytes read per Fetch (defends against fill-disk) + Header http.Header // headers sent on every request + + sf singleflight.Group + cacheMeta // embedded: etag/lastMod persisted to sidecar + lastChecked time.Time + metaLoaded bool + client *http.Client // set by New/NewWith; reuses connections across Fetch calls +} + +// cacheMeta is the sidecar format persisted alongside the downloaded file. +type cacheMeta struct { + ETag string `json:"etag,omitempty"` + LastMod string `json:"last_modified,omitempty"` +} + +func (c *Cacher) metaPath() string { return c.Path + ".meta" } + +// safeURL returns c.URL with any userinfo (user:password@) stripped, so +// errors and logs don't leak credentials embedded in the URL. Falls back +// to "" rather than echoing the raw value if parsing +// fails (defensive — a URL we can't parse may itself be a credential). +func (c *Cacher) safeURL() string { + u, err := url.Parse(c.URL) + if err != nil { + return "" + } + u.User = nil + return u.String() +} + +// loadMeta reads etag/lastMod from the sidecar file. A missing sidecar is +// not an error (just means a full download next time); read or parse errors +// propagate so the caller can decide whether to abort or proceed. +func (c *Cacher) loadMeta() error { + data, err := os.ReadFile(c.metaPath()) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if err := json.Unmarshal(data, &c.cacheMeta); err != nil { + return fmt.Errorf("parse %s: %w", c.metaPath(), err) + } + return nil +} + +// saveMeta writes etag/lastMod to the sidecar file atomically. +func (c *Cacher) saveMeta() error { + data, err := json.Marshal(c.cacheMeta) + if err != nil { + return err + } + tmp := c.metaPath() + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + if err := os.Rename(tmp, c.metaPath()); err != nil { + _ = os.Remove(tmp) + return err + } + return nil +} + +// New creates a Cacher with a default *http.Client (5s connect, 5m overall). +// Equivalent to NewWith(url, path, nil). Header is read live on every request, +// so it may be set or modified after New. +func New(url, path string) *Cacher { + return NewWith(url, path, nil) +} + +// NewWith creates a Cacher backed by the given *http.Client; pass nil for +// defaults. The client is used as-is — its Timeout, Transport, Jar, and +// CheckRedirect are all the caller's responsibility. +func NewWith(url, path string, client *http.Client) *Cacher { + if client == nil { + client = &http.Client{ + Timeout: defaultTimeout, + Transport: &http.Transport{ + DialContext: (&net.Dialer{Timeout: defaultConnTimeout}).DialContext, + TLSHandshakeTimeout: defaultConnTimeout, + }, + } + } + return &Cacher{URL: url, Path: path, client: client} +} + +// Fetch sends a conditional GET and writes new content to Path if the server +// responds with 200. Returns whether the file was updated. +// +// Both MaxAge and MinInterval are checked before making any HTTP request. +// ctx cancels the in-flight request and any blocking body read. +// +// Safe to call concurrently — concurrent callers share a single in-flight +// fetch (via singleflight) and all receive the same result. +func (c *Cacher) Fetch(ctx context.Context) (updated bool, err error) { + // MaxAge: file-mtime gate (no lock needed — just a stat). + if c.MaxAge > 0 { + if info, err := os.Stat(c.Path); err == nil { + if time.Since(info.ModTime()) < c.MaxAge { + return false, nil + } + } + } + + type result struct { + updated bool + err error + } + v, _, _ := c.sf.Do("fetch", func() (any, error) { + u, err := c.fetch(ctx) + return result{u, err}, nil + }) + r := v.(result) + return r.updated, r.err +} + +// fetch is the inner serialized work. singleflight ensures only one runs at +// a time, so the cacheMeta/lastChecked/metaLoaded fields don't need a mutex. +func (c *Cacher) fetch(ctx context.Context) (bool, error) { + // Load sidecar once so conditional GETs work after a process restart. + if !c.metaLoaded { + if err := c.loadMeta(); err != nil { + return false, err + } + c.metaLoaded = true + } + + // MinInterval: in-memory last-checked gate. + if c.MinInterval > 0 && !c.lastChecked.IsZero() { + if time.Since(c.lastChecked) < c.MinInterval { + return false, nil + } + } + c.lastChecked = time.Now() + + // Reserve .tmp before any HTTP — O_EXCL gives us cross-process + // exclusion (singleflight only covers the in-process case). A peer + // holding .tmp means another process is mid-download; return early + // without hitting the wire. + if err := os.MkdirAll(filepath.Dir(c.Path), 0o755); err != nil { + return false, err + } + tmp := c.Path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, fs.ErrExist) { + // Peer holds .tmp. Re-read sidecar; if peer has installed a + // different version, signal updated=true so the caller can + // reload Path from disk. + prevETag, prevLM := c.ETag, c.LastMod + if err := c.loadMeta(); err != nil { + return false, err + } + updated := c.ETag != prevETag || c.LastMod != prevLM + return updated, fmt.Errorf("%w for %s", ErrPeerFetching, c.safeURL()) + } + return false, err + } + // Any non-success path from here must remove tmp so peers aren't blocked. + defer func() { + _ = f.Close() + _ = os.Remove(tmp) + }() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.URL, nil) + if err != nil { + return false, err + } + + if c.ETag != "" { + req.Header.Set("If-None-Match", c.ETag) + } else if c.LastMod != "" { + req.Header.Set("If-Modified-Since", c.LastMod) + } + + for k, vs := range c.Header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + + resp, err := c.client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified { + return false, nil + } + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("%w %d fetching %s", ErrUnexpectedStatus, resp.StatusCode, c.safeURL()) + } + + body := io.Reader(resp.Body) + if c.MaxBytes > 0 { + // +1 so n>MaxBytes signals "exceeded" without an extra Read. + body = io.LimitReader(resp.Body, c.MaxBytes+1) + } + n, err := io.Copy(f, body) + if err != nil { + return false, err + } + if c.MaxBytes > 0 && n > c.MaxBytes { + return false, fmt.Errorf("%w (%d > %d) from %s", ErrBodyTooLarge, n, c.MaxBytes, c.safeURL()) + } + if n == 0 { + return false, fmt.Errorf("%w from %s", ErrEmptyResponse, c.safeURL()) + } + if err := os.Rename(tmp, c.Path); err != nil { + return false, err + } + + if etag := resp.Header.Get("ETag"); etag != "" { + c.ETag = etag + } + if lm := resp.Header.Get("Last-Modified"); lm != "" { + c.LastMod = lm + } + if err := c.saveMeta(); err != nil { + return true, fmt.Errorf("%w for %s: %w", ErrSaveMeta, c.Path, err) + } + + return true, nil +} diff --git a/net/httpcache/httpcache_test.go b/net/httpcache/httpcache_test.go new file mode 100644 index 0000000..0d4e4b9 --- /dev/null +++ b/net/httpcache/httpcache_test.go @@ -0,0 +1,227 @@ +package httpcache_test + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/therootcompany/golib/net/httpcache" +) + +// fakeServer serves body with a fixed ETag and honors If-None-Match. +// hits counts how many requests reached the handler, including 304s. +type fakeServer struct { + body []byte + etag string + hits atomic.Int32 +} + +func (f *fakeServer) handler(w http.ResponseWriter, r *http.Request) { + f.hits.Add(1) + if r.Header.Get("If-None-Match") == f.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", f.etag) + w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT") + _, _ = w.Write(f.body) +} + +func TestCacher_Download(t *testing.T) { + fs := &fakeServer{body: []byte("hello blocklist\n"), etag: `"abc123"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + + updated, err := c.Fetch(t.Context()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if !updated { + t.Error("first Fetch: expected updated=true") + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(got) != string(fs.body) { + t.Errorf("body = %q, want %q", got, fs.body) + } +} + +func TestCacher_SidecarWritten(t *testing.T) { + fs := &fakeServer{body: []byte("x"), etag: `"sidecar-etag"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + if _, err := c.Fetch(t.Context()); err != nil { + t.Fatalf("Fetch: %v", err) + } + + data, err := os.ReadFile(path + ".meta") + if err != nil { + t.Fatalf("sidecar not written: %v", err) + } + var meta map[string]string + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("sidecar not valid JSON: %v", err) + } + if meta["etag"] != fs.etag { + t.Errorf("sidecar etag = %q, want %q", meta["etag"], fs.etag) + } + if meta["last_modified"] == "" { + t.Error("sidecar last_modified empty") + } +} + +func TestCacher_ConditionalGet_SameCacher(t *testing.T) { + fs := &fakeServer{body: []byte("body"), etag: `"e1"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + if _, err := c.Fetch(t.Context()); err != nil { + t.Fatalf("first Fetch: %v", err) + } + + updated, err := c.Fetch(t.Context()) + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if updated { + t.Error("second Fetch: expected updated=false (304 path)") + } + if got := fs.hits.Load(); got != 2 { + t.Errorf("server hits = %d, want 2 (both Fetches must reach the wire)", got) + } +} + +func TestCacher_ConditionalGet_FreshCacher(t *testing.T) { + // Fresh Cacher must read the sidecar and send If-None-Match — proves + // ETag survives process restart. + fs := &fakeServer{body: []byte("body"), etag: `"e2"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + if _, err := httpcache.New(srv.URL, path).Fetch(t.Context()); err != nil { + t.Fatalf("seed Fetch: %v", err) + } + + fresh := httpcache.New(srv.URL, path) + updated, err := fresh.Fetch(t.Context()) + if err != nil { + t.Fatalf("fresh Fetch: %v", err) + } + if updated { + t.Error("fresh Fetch: expected updated=false (sidecar should have provided ETag)") + } +} + +func TestCacher_UnexpectedStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusInternalServerError) + })) + defer srv.Close() + + c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt")) + _, err := c.Fetch(t.Context()) + if !errors.Is(err, httpcache.ErrUnexpectedStatus) { + t.Errorf("err = %v, want ErrUnexpectedStatus", err) + } +} + +func TestCacher_EmptyResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt")) + _, err := c.Fetch(t.Context()) + if !errors.Is(err, httpcache.ErrEmptyResponse) { + t.Errorf("err = %v, want ErrEmptyResponse", err) + } +} + +func TestCacher_MaxBytesExceeded(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("0123456789ABCDEF")) // 16 bytes + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.MaxBytes = 8 + _, err := c.Fetch(t.Context()) + if !errors.Is(err, httpcache.ErrBodyTooLarge) { + t.Errorf("err = %v, want ErrBodyTooLarge", err) + } + if _, statErr := os.Stat(path); statErr == nil { + t.Error("expected body file not to be present after MaxBytes failure") + } +} + +func TestCacher_PeerFetching(t *testing.T) { + fs := &fakeServer{body: []byte("body"), etag: `"e1"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + + // First Fetch establishes baseline etag in memory and on-disk sidecar. + if _, err := c.Fetch(t.Context()); err != nil { + t.Fatalf("seed Fetch: %v", err) + } + hitsAfterSeed := fs.hits.Load() + + // Simulate a peer mid-download: hold .tmp. + if err := os.WriteFile(path+".tmp", []byte("partial"), 0o600); err != nil { + t.Fatalf("seed tmp: %v", err) + } + // Simulate peer having just installed a new sidecar (different etag). + if err := os.WriteFile(path+".meta", []byte(`{"etag":"\"peer-new\"","last_modified":"Tue, 03 Jan 2006 15:04:05 GMT"}`), 0o600); err != nil { + t.Fatalf("seed meta: %v", err) + } + + updated, err := c.Fetch(t.Context()) + if !errors.Is(err, httpcache.ErrPeerFetching) { + t.Errorf("err = %v, want ErrPeerFetching", err) + } + if !updated { + t.Error("expected updated=true (peer wrote new sidecar)") + } + if got := fs.hits.Load(); got != hitsAfterSeed { + t.Errorf("server hits = %d, want %d (must not fetch when peer holds tmp)", got, hitsAfterSeed) + } +} + +func TestCacher_MaxBytesAtLimit(t *testing.T) { + body := []byte("0123456789ABCDEF") // 16 bytes + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.MaxBytes = int64(len(body)) // exactly equal — should pass + if _, err := c.Fetch(t.Context()); err != nil { + t.Fatalf("Fetch: %v", err) + } + got, _ := os.ReadFile(path) + if string(got) != string(body) { + t.Errorf("body = %q, want %q", got, body) + } +}