mirror of
https://github.com/therootcompany/golib.git
synced 2026-11-04 06:55:00 +00:00
feat(httpcache): persist failure time — failed fetches wait MaxAge before retrying
This commit is contained in:
parent
ac7047c77c
commit
9054c7aff7
@ -2,6 +2,7 @@ package httpcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -100,6 +101,16 @@ type cacheMeta struct {
|
||||
|
||||
func (c *Cacher) metaPath() string { return c.Path + ".meta" }
|
||||
|
||||
// BasicAuth returns an HTTP Basic Authorization header value.
|
||||
func BasicAuth(user, pass string) string {
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass))
|
||||
}
|
||||
|
||||
// Bearer returns a Bearer Authorization header value.
|
||||
func Bearer(token string) string {
|
||||
return "Bearer " + token
|
||||
}
|
||||
|
||||
// safeURL returns c.URL with any userinfo (user:password@) stripped, so
|
||||
// errors and logs don't leak credentials embedded in the URL. Falls back
|
||||
// to "<unparseable URL>" rather than echoing the raw value if parsing
|
||||
@ -339,6 +350,11 @@ func (c *Cacher) fetch(ctx context.Context) (updated bool, err error) {
|
||||
}
|
||||
|
||||
func (c *Cacher) recordFailure(err error, retryAt *time.Time) error {
|
||||
if errors.Is(err, context.Canceled) ||
|
||||
errors.Is(err, context.DeadlineExceeded) ||
|
||||
errors.Is(err, ErrPeerFetching) {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
c.LastFailure = &now
|
||||
c.RetryAt = retryAt
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
package httpcache_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@ -286,3 +288,216 @@ func TestCacher_MaxBytesAtLimit(t *testing.T) {
|
||||
t.Errorf("body = %q, want %q", got, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacher_FailureBackoff(t *testing.T) {
|
||||
fs := &fakeServer{body: []byte("body"), etag: `"e1"`}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Fail the first request only; let later ones through.
|
||||
if fs.hits.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("If-None-Match") == fs.etag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", fs.etag)
|
||||
w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT")
|
||||
_, _ = w.Write(fs.body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "data.txt")
|
||||
c := httpcache.New(srv.URL, path)
|
||||
c.FailureBackoff = time.Hour
|
||||
|
||||
// First Fetch fails with 429 and records the failure time.
|
||||
if _, err := c.Fetch(t.Context()); err == nil {
|
||||
t.Fatal("first Fetch: expected error")
|
||||
}
|
||||
|
||||
// The immediate retry must not reach the wire.
|
||||
if updated, err := c.Fetch(t.Context()); err != nil || updated {
|
||||
t.Fatalf("retry: updated=%v err=%v, want false/nil (backoff gate)", updated, err)
|
||||
}
|
||||
if got := fs.hits.Load(); got != 1 {
|
||||
t.Errorf("server hits = %d, want 1 (failed fetch must be gated)", got)
|
||||
}
|
||||
|
||||
// After the backoff expires (sidecar rewritten with an old failure
|
||||
// time, loaded by a fresh Cacher), the retry reaches the wire and
|
||||
// succeeds.
|
||||
expired := fmt.Sprintf(`{"failed_at":%d}`, time.Now().Add(-time.Hour).UnixMilli())
|
||||
if err := os.WriteFile(path+".meta", []byte(expired), 0o600); err != nil {
|
||||
t.Fatalf("expire failure: %v", err)
|
||||
}
|
||||
fresh := httpcache.New(srv.URL, path)
|
||||
fresh.FailureBackoff = time.Hour
|
||||
updated, err := fresh.Fetch(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("retry after backoff: %v", err)
|
||||
}
|
||||
if !updated {
|
||||
t.Error("retry after backoff: expected updated=true")
|
||||
}
|
||||
if got := fs.hits.Load(); got != 2 {
|
||||
t.Errorf("server hits = %d, want 2", got)
|
||||
}
|
||||
|
||||
// A successful fetch clears the failure state from the sidecar.
|
||||
var meta map[string]any
|
||||
data, err := os.ReadFile(path + ".meta")
|
||||
if err != nil {
|
||||
t.Fatalf("read sidecar: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
t.Fatalf("parse sidecar: %v", err)
|
||||
}
|
||||
if _, ok := meta["failed_at"]; ok {
|
||||
t.Errorf("failed_at = %v, want cleared after success", meta["failed_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacher_FailureBackoffDefaultsToMaxAge(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "data.txt")
|
||||
c := httpcache.New(srv.URL, path)
|
||||
c.MaxAge = time.Hour // FailureBackoff unset → failures wait MaxAge
|
||||
|
||||
if _, err := c.Fetch(t.Context()); err == nil {
|
||||
t.Fatal("first Fetch: expected error")
|
||||
}
|
||||
if updated, err := c.Fetch(t.Context()); err != nil || updated {
|
||||
t.Fatalf("retry: updated=%v err=%v, want gated", updated, err)
|
||||
}
|
||||
if got := hits.Load(); got != 1 {
|
||||
t.Errorf("server hits = %d, want 1 (backoff must default to MaxAge)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacher_FailureBackoffPersistsAcrossInstances(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "data.txt")
|
||||
c := httpcache.New(srv.URL, path)
|
||||
c.FailureBackoff = time.Hour
|
||||
if _, err := c.Fetch(t.Context()); err == nil {
|
||||
t.Fatal("first Fetch: expected failure")
|
||||
}
|
||||
|
||||
// A brand-new Cacher (simulated process restart) must honor the
|
||||
// recorded failure without a single HTTP request.
|
||||
fresh := httpcache.New(srv.URL, path)
|
||||
fresh.FailureBackoff = time.Hour
|
||||
if updated, err := fresh.Fetch(t.Context()); err != nil || updated {
|
||||
t.Fatalf("fresh Fetch: updated=%v err=%v, want gated", updated, err)
|
||||
}
|
||||
if got := hits.Load(); got != 1 {
|
||||
t.Errorf("server hits = %d, want 1 (backoff must survive restart)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacher_304ClearsFailure(t *testing.T) {
|
||||
fs := &fakeServer{body: []byte("body"), etag: `"e1"`}
|
||||
srv := httptest.NewServer(http.HandlerFunc(fs.handler))
|
||||
defer srv.Close()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "data.txt")
|
||||
c := httpcache.New(srv.URL, path)
|
||||
c.FailureBackoff = time.Hour
|
||||
if _, err := c.Fetch(t.Context()); err != nil {
|
||||
t.Fatalf("seed Fetch: %v", err)
|
||||
}
|
||||
|
||||
// Hand-write a failure into the sidecar (epoch-time, so the gate
|
||||
// is already expired) with the known-good ETag, then use a FRESH
|
||||
// Cacher so the sidecar is actually loaded (metaLoaded=false).
|
||||
sidecar := `{"etag":"\"e1\"","last_modified":"Mon, 02 Jan 2006 15:04:05 GMT","failed_at":1}`
|
||||
if err := os.WriteFile(path+".meta", []byte(sidecar), 0o600); err != nil {
|
||||
t.Fatalf("seed sidecar: %v", err)
|
||||
}
|
||||
fresh := httpcache.New(srv.URL, path)
|
||||
fresh.FailureBackoff = time.Hour
|
||||
|
||||
// The conditional GET gets a 304 — the remote proved healthy, so
|
||||
// the failure state must be cleared.
|
||||
updated, err := fresh.Fetch(t.Context())
|
||||
if err != nil || updated {
|
||||
t.Fatalf("Fetch: updated=%v err=%v, want false/nil (304)", updated, err)
|
||||
}
|
||||
data, err := os.ReadFile(path + ".meta")
|
||||
if err != nil {
|
||||
t.Fatalf("read sidecar: %v", err)
|
||||
}
|
||||
var meta map[string]any
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
t.Fatalf("parse sidecar: %v", err)
|
||||
}
|
||||
if _, ok := meta["failed_at"]; ok {
|
||||
t.Errorf("failed_at = %v, want cleared by 304", meta["failed_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacher_ContextCanceledNotBackedOff(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if hits.Add(1) > 1 {
|
||||
w.Header().Set("ETag", `"e1"`)
|
||||
_, _ = w.Write([]byte("body"))
|
||||
return
|
||||
}
|
||||
<-r.Context().Done() // hang until the client gives up
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "data.txt")
|
||||
c := httpcache.New(srv.URL, path)
|
||||
c.FailureBackoff = time.Hour
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.Fetch(ctx)
|
||||
errCh <- err
|
||||
}()
|
||||
time.Sleep(50 * time.Millisecond) // let the request reach the server
|
||||
cancel()
|
||||
if err := <-errCh; err == nil {
|
||||
t.Fatal("expected context error from canceled Fetch")
|
||||
}
|
||||
|
||||
// A canceled fetch is not an upstream failure: the next Fetch must
|
||||
// go back to the wire immediately, not wait out the backoff.
|
||||
if updated, err := c.Fetch(t.Context()); err != nil {
|
||||
t.Fatalf("Fetch after cancel: %v", err)
|
||||
} else if !updated {
|
||||
t.Error("Fetch after cancel: expected updated=true")
|
||||
}
|
||||
if got := hits.Load(); got != 2 {
|
||||
t.Errorf("server hits = %d, want 2 (canceled fetch must not gate)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicAuth(t *testing.T) {
|
||||
// base64("user:pass") == "dXNlcjpwYXNz"
|
||||
if got, want := httpcache.BasicAuth("user", "pass"), "Basic dXNlcjpwYXNz"; got != want {
|
||||
t.Errorf("BasicAuth = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearer(t *testing.T) {
|
||||
if got, want := httpcache.Bearer("tok"), "Bearer tok"; got != want {
|
||||
t.Errorf("Bearer = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user