mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 12:48:00 +00:00
feat(dataset): add PollFiles fetcher for local-file sources
Stats the given paths and reports updated when any size/modtime changes since the last call. First call always reports true so the initial Load populates views. check-ip uses it for --inbound/--outbound so edits to local lists get picked up by Group.Tick without a restart.
This commit is contained in:
parent
7b798a739a
commit
3b5812ffcd
@ -292,7 +292,10 @@ func openBlocklists(cfg Config) (
|
||||
func newBlocklistFetcher(cfg Config) (fetcher dataset.Fetcher, inPaths, outPaths []string, err error) {
|
||||
switch {
|
||||
case cfg.Inbound != "" || cfg.Outbound != "":
|
||||
return dataset.NopFetcher{}, splitCSV(cfg.Inbound), splitCSV(cfg.Outbound), nil
|
||||
inPaths := splitCSV(cfg.Inbound)
|
||||
outPaths := splitCSV(cfg.Outbound)
|
||||
all := append(append([]string(nil), inPaths...), outPaths...)
|
||||
return dataset.PollFiles(all...), inPaths, outPaths, nil
|
||||
|
||||
case cfg.GitURL != "":
|
||||
dir, err := cacheDir(cfg.DataDir)
|
||||
|
||||
@ -16,6 +16,8 @@ package dataset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
@ -32,12 +34,52 @@ type FetcherFunc func() (bool, error)
|
||||
|
||||
func (f FetcherFunc) Fetch() (bool, error) { return f() }
|
||||
|
||||
// NopFetcher always reports no update. Use for groups backed by local files
|
||||
// that don't need a refresh cycle.
|
||||
// NopFetcher always reports no update. Use for groups whose source never
|
||||
// changes (test fixtures, embedded data).
|
||||
type NopFetcher struct{}
|
||||
|
||||
func (NopFetcher) Fetch() (bool, error) { return false, nil }
|
||||
|
||||
// PollFiles returns a Fetcher that stat's the given paths and reports
|
||||
// "updated" whenever any file's size or modtime has changed since the last
|
||||
// call. The first call always reports updated=true.
|
||||
//
|
||||
// Use for Group's whose source is local files that may be edited out of band
|
||||
// (e.g. a user-provided --inbound list) — pair with Group.Tick to pick up
|
||||
// changes automatically.
|
||||
func PollFiles(paths ...string) Fetcher {
|
||||
return &filePoller{paths: paths, stats: make(map[string]fileStat, len(paths))}
|
||||
}
|
||||
|
||||
type fileStat struct {
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
type filePoller struct {
|
||||
mu sync.Mutex
|
||||
paths []string
|
||||
stats map[string]fileStat
|
||||
}
|
||||
|
||||
func (p *filePoller) Fetch() (bool, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
changed := false
|
||||
for _, path := range p.paths {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
cur := fileStat{size: info.Size(), modTime: info.ModTime()}
|
||||
if prev, ok := p.stats[path]; !ok || prev != cur {
|
||||
changed = true
|
||||
p.stats[path] = cur
|
||||
}
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// Group ties one Fetcher to one or more views. A Load call fetches once and,
|
||||
// on the first call or when the source reports a change, reloads every view
|
||||
// and atomically swaps its current value.
|
||||
|
||||
@ -2,8 +2,10 @@ package dataset_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/therootcompany/golib/sync/dataset"
|
||||
)
|
||||
@ -127,6 +129,49 @@ func TestGroup_LoaderError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := dir + "/a.txt"
|
||||
b := dir + "/b.txt"
|
||||
if err := os.WriteFile(a, []byte("1"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(b, []byte("2"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p := dataset.PollFiles(a, b)
|
||||
|
||||
if u, err := p.Fetch(); err != nil || !u {
|
||||
t.Fatalf("first Fetch: updated=%v err=%v, want true/nil", u, err)
|
||||
}
|
||||
if u, err := p.Fetch(); err != nil || u {
|
||||
t.Fatalf("unchanged Fetch: updated=%v err=%v, want false/nil", u, err)
|
||||
}
|
||||
|
||||
// Bump mtime + change contents on b.
|
||||
future := time.Now().Add(2 * time.Second)
|
||||
if err := os.WriteFile(b, []byte("22"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(b, future, future); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u, err := p.Fetch(); err != nil || !u {
|
||||
t.Errorf("after change: updated=%v err=%v, want true/nil", u, err)
|
||||
}
|
||||
if u, err := p.Fetch(); err != nil || u {
|
||||
t.Errorf("steady Fetch: updated=%v err=%v, want false/nil", u, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollFiles_MissingFile(t *testing.T) {
|
||||
p := dataset.PollFiles(t.TempDir() + "/nope.txt")
|
||||
if _, err := p.Fetch(); err == nil {
|
||||
t.Error("expected error for missing file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherFunc(t *testing.T) {
|
||||
var called bool
|
||||
f := dataset.FetcherFunc(func() (bool, error) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user