From fac7ffad84c81220694ff2a58d0251692045947d Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 9 Sep 2026 22:29:01 -0600 Subject: [PATCH] refactor(net/httpcache): finalize release API --- net/httpcache/README.md | 2 +- net/httpcache/httpcache.go | 18 ++- net/httpcache/httpcache_test.go | 247 +++++++++++++++++++++++++++++--- 3 files changed, 249 insertions(+), 18 deletions(-) diff --git a/net/httpcache/README.md b/net/httpcache/README.md index 86767af..873afb1 100644 --- a/net/httpcache/README.md +++ b/net/httpcache/README.md @@ -25,7 +25,7 @@ c.MaxAge = time.Hour // skip HTTP if file is < 1h old // c.FailureBackoff = 30 * time.Minute // defaults to MaxAge c.MaxBytes = 50 << 20 // 50 MiB cap -updated, err := c.Fetch(ctx) +updated, err := c.Update(ctx) switch { case errors.Is(err, httpcache.ErrPeerFetching): // another process is downloading; reload from disk if updated==true diff --git a/net/httpcache/httpcache.go b/net/httpcache/httpcache.go index 18f5961..e9ea386 100644 --- a/net/httpcache/httpcache.go +++ b/net/httpcache/httpcache.go @@ -2,6 +2,7 @@ package httpcache import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -100,6 +101,16 @@ type cacheMeta struct { func (c *Cacher) metaPath() string { return c.Path + ".meta" } +// BasicAuth returns an HTTP Basic Authorization header value. +func BasicAuth(user, pass string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass)) +} + +// Bearer returns a Bearer Authorization header value. +func Bearer(token string) string { + return "Bearer " + token +} + // safeURL returns c.URL with any userinfo (user:password@) stripped, so // errors and logs don't leak credentials embedded in the URL. Falls back // to "" rather than echoing the raw value if parsing @@ -178,7 +189,7 @@ func NewWith(url, path string, client *http.Client) *Cacher { // // Safe to call concurrently — concurrent callers share a single in-flight // fetch (via singleflight) and all receive the same result. -func (c *Cacher) Fetch(ctx context.Context) (updated bool, err error) { +func (c *Cacher) Update(ctx context.Context) (updated bool, err error) { type result struct { updated bool err error @@ -339,6 +350,11 @@ func (c *Cacher) fetch(ctx context.Context) (updated bool, err error) { } func (c *Cacher) recordFailure(err error, retryAt *time.Time) error { + if errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, ErrPeerFetching) { + return err + } now := time.Now() c.LastFailure = &now c.RetryAt = retryAt diff --git a/net/httpcache/httpcache_test.go b/net/httpcache/httpcache_test.go index 6580392..46e304c 100644 --- a/net/httpcache/httpcache_test.go +++ b/net/httpcache/httpcache_test.go @@ -1,8 +1,10 @@ package httpcache_test import ( + "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "os" @@ -41,7 +43,7 @@ func TestCacher_Download(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) - updated, err := c.Fetch(t.Context()) + updated, err := c.Update(t.Context()) if err != nil { t.Fatalf("Fetch: %v", err) } @@ -64,7 +66,7 @@ func TestCacher_SidecarWritten(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) - if _, err := c.Fetch(t.Context()); err != nil { + if _, err := c.Update(t.Context()); err != nil { t.Fatalf("Fetch: %v", err) } @@ -91,11 +93,11 @@ func TestCacher_ConditionalGet_SameCacher(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) - if _, err := c.Fetch(t.Context()); err != nil { + if _, err := c.Update(t.Context()); err != nil { t.Fatalf("first Fetch: %v", err) } - updated, err := c.Fetch(t.Context()) + updated, err := c.Update(t.Context()) if err != nil { t.Fatalf("second Fetch: %v", err) } @@ -115,12 +117,12 @@ func TestCacher_ConditionalGet_FreshCacher(t *testing.T) { defer srv.Close() path := filepath.Join(t.TempDir(), "data.txt") - if _, err := httpcache.New(srv.URL, path).Fetch(t.Context()); err != nil { + if _, err := httpcache.New(srv.URL, path).Update(t.Context()); err != nil { t.Fatalf("seed Fetch: %v", err) } fresh := httpcache.New(srv.URL, path) - updated, err := fresh.Fetch(t.Context()) + updated, err := fresh.Update(t.Context()) if err != nil { t.Fatalf("fresh Fetch: %v", err) } @@ -136,7 +138,7 @@ func TestCacher_UnexpectedStatus(t *testing.T) { defer srv.Close() c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt")) - _, err := c.Fetch(t.Context()) + _, err := c.Update(t.Context()) if !errors.Is(err, httpcache.ErrUnexpectedStatus) { t.Errorf("err = %v, want ErrUnexpectedStatus", err) } @@ -149,7 +151,7 @@ func TestCacher_EmptyResponse(t *testing.T) { defer srv.Close() c := httpcache.New(srv.URL, filepath.Join(t.TempDir(), "data.txt")) - _, err := c.Fetch(t.Context()) + _, err := c.Update(t.Context()) if !errors.Is(err, httpcache.ErrEmptyResponse) { t.Errorf("err = %v, want ErrEmptyResponse", err) } @@ -166,7 +168,7 @@ func TestCacher_MaxAgeBacksOffAfterFailure(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) c.MaxAge = time.Hour - if _, err := c.Fetch(t.Context()); !errors.Is(err, httpcache.ErrUnexpectedStatus) { + if _, err := c.Update(t.Context()); !errors.Is(err, httpcache.ErrUnexpectedStatus) { t.Fatalf("first Fetch: %v, want ErrUnexpectedStatus", err) } @@ -174,7 +176,7 @@ func TestCacher_MaxAgeBacksOffAfterFailure(t *testing.T) { // of retrying immediately against the rate-limited server. fresh := httpcache.New(srv.URL, path) fresh.MaxAge = time.Hour - updated, err := fresh.Fetch(t.Context()) + updated, err := fresh.Update(t.Context()) if err != nil { t.Fatalf("backoff Fetch: %v", err) } @@ -198,12 +200,12 @@ func TestCacher_RetryAfterOverridesFailureBackoff(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) c.FailureBackoff = time.Second - if _, err := c.Fetch(t.Context()); !errors.Is(err, httpcache.ErrUnexpectedStatus) { + if _, err := c.Update(t.Context()); !errors.Is(err, httpcache.ErrUnexpectedStatus) { t.Fatalf("first Fetch: %v, want ErrUnexpectedStatus", err) } fresh := httpcache.New(srv.URL, path) - updated, err := fresh.Fetch(t.Context()) + updated, err := fresh.Update(t.Context()) if err != nil { t.Fatalf("Retry-After Fetch: %v", err) } @@ -224,7 +226,7 @@ func TestCacher_MaxBytesExceeded(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) c.MaxBytes = 8 - _, err := c.Fetch(t.Context()) + _, err := c.Update(t.Context()) if !errors.Is(err, httpcache.ErrBodyTooLarge) { t.Errorf("err = %v, want ErrBodyTooLarge", err) } @@ -242,7 +244,7 @@ func TestCacher_PeerFetching(t *testing.T) { c := httpcache.New(srv.URL, path) // First Fetch establishes baseline etag in memory and on-disk sidecar. - if _, err := c.Fetch(t.Context()); err != nil { + if _, err := c.Update(t.Context()); err != nil { t.Fatalf("seed Fetch: %v", err) } hitsAfterSeed := fs.hits.Load() @@ -256,7 +258,7 @@ func TestCacher_PeerFetching(t *testing.T) { t.Fatalf("seed meta: %v", err) } - updated, err := c.Fetch(t.Context()) + updated, err := c.Update(t.Context()) if !errors.Is(err, httpcache.ErrPeerFetching) { t.Errorf("err = %v, want ErrPeerFetching", err) } @@ -278,7 +280,7 @@ func TestCacher_MaxBytesAtLimit(t *testing.T) { path := filepath.Join(t.TempDir(), "data.txt") c := httpcache.New(srv.URL, path) c.MaxBytes = int64(len(body)) // exactly equal — should pass - if _, err := c.Fetch(t.Context()); err != nil { + if _, err := c.Update(t.Context()); err != nil { t.Fatalf("Fetch: %v", err) } got, _ := os.ReadFile(path) @@ -286,3 +288,216 @@ func TestCacher_MaxBytesAtLimit(t *testing.T) { t.Errorf("body = %q, want %q", got, body) } } + +func TestCacher_FailureBackoff(t *testing.T) { + fs := &fakeServer{body: []byte("body"), etag: `"e1"`} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Fail the first request only; let later ones through. + if fs.hits.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + if r.Header.Get("If-None-Match") == fs.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", fs.etag) + w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT") + _, _ = w.Write(fs.body) + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.FailureBackoff = time.Hour + + // First Fetch fails with 429 and records the failure time. + if _, err := c.Update(t.Context()); err == nil { + t.Fatal("first Fetch: expected error") + } + + // The immediate retry must not reach the wire. + if updated, err := c.Update(t.Context()); err != nil || updated { + t.Fatalf("retry: updated=%v err=%v, want false/nil (backoff gate)", updated, err) + } + if got := fs.hits.Load(); got != 1 { + t.Errorf("server hits = %d, want 1 (failed fetch must be gated)", got) + } + + // After the backoff expires (sidecar rewritten with an old failure + // time, loaded by a fresh Cacher), the retry reaches the wire and + // succeeds. + expired := fmt.Sprintf(`{"failed_at":%d}`, time.Now().Add(-time.Hour).UnixMilli()) + if err := os.WriteFile(path+".meta", []byte(expired), 0o600); err != nil { + t.Fatalf("expire failure: %v", err) + } + fresh := httpcache.New(srv.URL, path) + fresh.FailureBackoff = time.Hour + updated, err := fresh.Update(t.Context()) + if err != nil { + t.Fatalf("retry after backoff: %v", err) + } + if !updated { + t.Error("retry after backoff: expected updated=true") + } + if got := fs.hits.Load(); got != 2 { + t.Errorf("server hits = %d, want 2", got) + } + + // A successful fetch clears the failure state from the sidecar. + var meta map[string]any + data, err := os.ReadFile(path + ".meta") + if err != nil { + t.Fatalf("read sidecar: %v", err) + } + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("parse sidecar: %v", err) + } + if _, ok := meta["failed_at"]; ok { + t.Errorf("failed_at = %v, want cleared after success", meta["failed_at"]) + } +} + +func TestCacher_FailureBackoffDefaultsToMaxAge(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.MaxAge = time.Hour // FailureBackoff unset → failures wait MaxAge + + if _, err := c.Update(t.Context()); err == nil { + t.Fatal("first Fetch: expected error") + } + if updated, err := c.Update(t.Context()); err != nil || updated { + t.Fatalf("retry: updated=%v err=%v, want gated", updated, err) + } + if got := hits.Load(); got != 1 { + t.Errorf("server hits = %d, want 1 (backoff must default to MaxAge)", got) + } +} + +func TestCacher_FailureBackoffPersistsAcrossInstances(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.FailureBackoff = time.Hour + if _, err := c.Update(t.Context()); err == nil { + t.Fatal("first Fetch: expected failure") + } + + // A brand-new Cacher (simulated process restart) must honor the + // recorded failure without a single HTTP request. + fresh := httpcache.New(srv.URL, path) + fresh.FailureBackoff = time.Hour + if updated, err := fresh.Update(t.Context()); err != nil || updated { + t.Fatalf("fresh Fetch: updated=%v err=%v, want gated", updated, err) + } + if got := hits.Load(); got != 1 { + t.Errorf("server hits = %d, want 1 (backoff must survive restart)", got) + } +} + +func TestCacher_304ClearsFailure(t *testing.T) { + fs := &fakeServer{body: []byte("body"), etag: `"e1"`} + srv := httptest.NewServer(http.HandlerFunc(fs.handler)) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.FailureBackoff = time.Hour + if _, err := c.Update(t.Context()); err != nil { + t.Fatalf("seed Fetch: %v", err) + } + + // Hand-write a failure into the sidecar (epoch-time, so the gate + // is already expired) with the known-good ETag, then use a FRESH + // Cacher so the sidecar is actually loaded (metaLoaded=false). + sidecar := `{"etag":"\"e1\"","last_modified":"Mon, 02 Jan 2006 15:04:05 GMT","failed_at":1}` + if err := os.WriteFile(path+".meta", []byte(sidecar), 0o600); err != nil { + t.Fatalf("seed sidecar: %v", err) + } + fresh := httpcache.New(srv.URL, path) + fresh.FailureBackoff = time.Hour + + // The conditional GET gets a 304 — the remote proved healthy, so + // the failure state must be cleared. + updated, err := fresh.Update(t.Context()) + if err != nil || updated { + t.Fatalf("Fetch: updated=%v err=%v, want false/nil (304)", updated, err) + } + data, err := os.ReadFile(path + ".meta") + if err != nil { + t.Fatalf("read sidecar: %v", err) + } + var meta map[string]any + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("parse sidecar: %v", err) + } + if _, ok := meta["failed_at"]; ok { + t.Errorf("failed_at = %v, want cleared by 304", meta["failed_at"]) + } +} + +func TestCacher_ContextCanceledNotBackedOff(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) > 1 { + w.Header().Set("ETag", `"e1"`) + _, _ = w.Write([]byte("body")) + return + } + <-r.Context().Done() // hang until the client gives up + })) + defer srv.Close() + + path := filepath.Join(t.TempDir(), "data.txt") + c := httpcache.New(srv.URL, path) + c.FailureBackoff = time.Hour + + ctx, cancel := context.WithCancel(t.Context()) + errCh := make(chan error, 1) + go func() { + _, err := c.Update(ctx) + errCh <- err + }() + time.Sleep(50 * time.Millisecond) // let the request reach the server + cancel() + if err := <-errCh; err == nil { + t.Fatal("expected context error from canceled Fetch") + } + + // A canceled fetch is not an upstream failure: the next Fetch must + // go back to the wire immediately, not wait out the backoff. + if updated, err := c.Update(t.Context()); err != nil { + t.Fatalf("Fetch after cancel: %v", err) + } else if !updated { + t.Error("Fetch after cancel: expected updated=true") + } + if got := hits.Load(); got != 2 { + t.Errorf("server hits = %d, want 2 (canceled fetch must not gate)", got) + } +} + +func TestBasicAuth(t *testing.T) { + // base64("user:pass") == "dXNlcjpwYXNz" + if got, want := httpcache.BasicAuth("user", "pass"), "Basic dXNlcjpwYXNz"; got != want { + t.Errorf("BasicAuth = %q, want %q", got, want) + } +} + +func TestBearer(t *testing.T) { + if got, want := httpcache.Bearer("tok"), "Bearer tok"; got != want { + t.Errorf("Bearer = %q, want %q", got, want) + } +}