feat(dataset): add PollFiles fetcher for local-file sources

Stats the given paths and reports updated when any size/modtime
changes since the last call. First call always reports true so the
initial Load populates views.

check-ip uses it for --inbound/--outbound so edits to local lists
get picked up by Group.Tick without a restart.
This commit is contained in:
AJ ONeal 2026-04-20 15:39:23 -06:00
parent 7b798a739a
commit 3b5812ffcd
No known key found for this signature in database
3 changed files with 93 additions and 3 deletions

View File

@ -292,7 +292,10 @@ func openBlocklists(cfg Config) (
func newBlocklistFetcher(cfg Config) (fetcher dataset.Fetcher, inPaths, outPaths []string, err error) {
switch {
case cfg.Inbound != "" || cfg.Outbound != "":
return dataset.NopFetcher{}, splitCSV(cfg.Inbound), splitCSV(cfg.Outbound), nil
inPaths := splitCSV(cfg.Inbound)
outPaths := splitCSV(cfg.Outbound)
all := append(append([]string(nil), inPaths...), outPaths...)
return dataset.PollFiles(all...), inPaths, outPaths, nil
case cfg.GitURL != "":
dir, err := cacheDir(cfg.DataDir)

View File

@ -16,6 +16,8 @@ package dataset
import (
"context"
"os"
"sync"
"sync/atomic"
"time"
)
@ -32,12 +34,52 @@ type FetcherFunc func() (bool, error)
func (f FetcherFunc) Fetch() (bool, error) { return f() }
// NopFetcher always reports no update. Use for groups backed by local files
// that don't need a refresh cycle.
// NopFetcher always reports no update. Use for groups whose source never
// changes (test fixtures, embedded data).
type NopFetcher struct{}
func (NopFetcher) Fetch() (bool, error) { return false, nil }
// PollFiles returns a Fetcher that stat's the given paths and reports
// "updated" whenever any file's size or modtime has changed since the last
// call. The first call always reports updated=true.
//
// Use for Group's whose source is local files that may be edited out of band
// (e.g. a user-provided --inbound list) — pair with Group.Tick to pick up
// changes automatically.
func PollFiles(paths ...string) Fetcher {
return &filePoller{paths: paths, stats: make(map[string]fileStat, len(paths))}
}
type fileStat struct {
size int64
modTime time.Time
}
type filePoller struct {
mu sync.Mutex
paths []string
stats map[string]fileStat
}
func (p *filePoller) Fetch() (bool, error) {
p.mu.Lock()
defer p.mu.Unlock()
changed := false
for _, path := range p.paths {
info, err := os.Stat(path)
if err != nil {
return false, err
}
cur := fileStat{size: info.Size(), modTime: info.ModTime()}
if prev, ok := p.stats[path]; !ok || prev != cur {
changed = true
p.stats[path] = cur
}
}
return changed, nil
}
// Group ties one Fetcher to one or more views. A Load call fetches once and,
// on the first call or when the source reports a change, reloads every view
// and atomically swaps its current value.

View File

@ -2,8 +2,10 @@ package dataset_test
import (
"errors"
"os"
"sync/atomic"
"testing"
"time"
"github.com/therootcompany/golib/sync/dataset"
)
@ -127,6 +129,49 @@ func TestGroup_LoaderError(t *testing.T) {
}
}
func TestPollFiles(t *testing.T) {
dir := t.TempDir()
a := dir + "/a.txt"
b := dir + "/b.txt"
if err := os.WriteFile(a, []byte("1"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(b, []byte("2"), 0o644); err != nil {
t.Fatal(err)
}
p := dataset.PollFiles(a, b)
if u, err := p.Fetch(); err != nil || !u {
t.Fatalf("first Fetch: updated=%v err=%v, want true/nil", u, err)
}
if u, err := p.Fetch(); err != nil || u {
t.Fatalf("unchanged Fetch: updated=%v err=%v, want false/nil", u, err)
}
// Bump mtime + change contents on b.
future := time.Now().Add(2 * time.Second)
if err := os.WriteFile(b, []byte("22"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(b, future, future); err != nil {
t.Fatal(err)
}
if u, err := p.Fetch(); err != nil || !u {
t.Errorf("after change: updated=%v err=%v, want true/nil", u, err)
}
if u, err := p.Fetch(); err != nil || u {
t.Errorf("steady Fetch: updated=%v err=%v, want false/nil", u, err)
}
}
func TestPollFiles_MissingFile(t *testing.T) {
p := dataset.PollFiles(t.TempDir() + "/nope.txt")
if _, err := p.Fetch(); err == nil {
t.Error("expected error for missing file")
}
}
func TestFetcherFunc(t *testing.T) {
var called bool
f := dataset.FetcherFunc(func() (bool, error) {