mirror of
https://github.com/therootcompany/golib.git
synced 2026-04-24 20:58:00 +00:00
feat(dataset): add PollFiles fetcher for local-file sources
Stats the given paths and reports updated when any size/modtime changes since the last call. First call always reports true so the initial Load populates views. check-ip uses it for --inbound/--outbound so edits to local lists get picked up by Group.Tick without a restart.
This commit is contained in:
parent
7b798a739a
commit
3b5812ffcd
@ -292,7 +292,10 @@ func openBlocklists(cfg Config) (
|
|||||||
func newBlocklistFetcher(cfg Config) (fetcher dataset.Fetcher, inPaths, outPaths []string, err error) {
|
func newBlocklistFetcher(cfg Config) (fetcher dataset.Fetcher, inPaths, outPaths []string, err error) {
|
||||||
switch {
|
switch {
|
||||||
case cfg.Inbound != "" || cfg.Outbound != "":
|
case cfg.Inbound != "" || cfg.Outbound != "":
|
||||||
return dataset.NopFetcher{}, splitCSV(cfg.Inbound), splitCSV(cfg.Outbound), nil
|
inPaths := splitCSV(cfg.Inbound)
|
||||||
|
outPaths := splitCSV(cfg.Outbound)
|
||||||
|
all := append(append([]string(nil), inPaths...), outPaths...)
|
||||||
|
return dataset.PollFiles(all...), inPaths, outPaths, nil
|
||||||
|
|
||||||
case cfg.GitURL != "":
|
case cfg.GitURL != "":
|
||||||
dir, err := cacheDir(cfg.DataDir)
|
dir, err := cacheDir(cfg.DataDir)
|
||||||
|
|||||||
@ -16,6 +16,8 @@ package dataset
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@ -32,12 +34,52 @@ type FetcherFunc func() (bool, error)
|
|||||||
|
|
||||||
func (f FetcherFunc) Fetch() (bool, error) { return f() }
|
func (f FetcherFunc) Fetch() (bool, error) { return f() }
|
||||||
|
|
||||||
// NopFetcher always reports no update. Use for groups backed by local files
|
// NopFetcher always reports no update. Use for groups whose source never
|
||||||
// that don't need a refresh cycle.
|
// changes (test fixtures, embedded data).
|
||||||
type NopFetcher struct{}
|
type NopFetcher struct{}
|
||||||
|
|
||||||
func (NopFetcher) Fetch() (bool, error) { return false, nil }
|
func (NopFetcher) Fetch() (bool, error) { return false, nil }
|
||||||
|
|
||||||
|
// PollFiles returns a Fetcher that stat's the given paths and reports
|
||||||
|
// "updated" whenever any file's size or modtime has changed since the last
|
||||||
|
// call. The first call always reports updated=true.
|
||||||
|
//
|
||||||
|
// Use for Group's whose source is local files that may be edited out of band
|
||||||
|
// (e.g. a user-provided --inbound list) — pair with Group.Tick to pick up
|
||||||
|
// changes automatically.
|
||||||
|
func PollFiles(paths ...string) Fetcher {
|
||||||
|
return &filePoller{paths: paths, stats: make(map[string]fileStat, len(paths))}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fileStat struct {
|
||||||
|
size int64
|
||||||
|
modTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type filePoller struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
paths []string
|
||||||
|
stats map[string]fileStat
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *filePoller) Fetch() (bool, error) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
changed := false
|
||||||
|
for _, path := range p.paths {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
cur := fileStat{size: info.Size(), modTime: info.ModTime()}
|
||||||
|
if prev, ok := p.stats[path]; !ok || prev != cur {
|
||||||
|
changed = true
|
||||||
|
p.stats[path] = cur
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Group ties one Fetcher to one or more views. A Load call fetches once and,
|
// Group ties one Fetcher to one or more views. A Load call fetches once and,
|
||||||
// on the first call or when the source reports a change, reloads every view
|
// on the first call or when the source reports a change, reloads every view
|
||||||
// and atomically swaps its current value.
|
// and atomically swaps its current value.
|
||||||
|
|||||||
@ -2,8 +2,10 @@ package dataset_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/therootcompany/golib/sync/dataset"
|
"github.com/therootcompany/golib/sync/dataset"
|
||||||
)
|
)
|
||||||
@ -127,6 +129,49 @@ func TestGroup_LoaderError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPollFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := dir + "/a.txt"
|
||||||
|
b := dir + "/b.txt"
|
||||||
|
if err := os.WriteFile(a, []byte("1"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(b, []byte("2"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
p := dataset.PollFiles(a, b)
|
||||||
|
|
||||||
|
if u, err := p.Fetch(); err != nil || !u {
|
||||||
|
t.Fatalf("first Fetch: updated=%v err=%v, want true/nil", u, err)
|
||||||
|
}
|
||||||
|
if u, err := p.Fetch(); err != nil || u {
|
||||||
|
t.Fatalf("unchanged Fetch: updated=%v err=%v, want false/nil", u, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bump mtime + change contents on b.
|
||||||
|
future := time.Now().Add(2 * time.Second)
|
||||||
|
if err := os.WriteFile(b, []byte("22"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chtimes(b, future, future); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u, err := p.Fetch(); err != nil || !u {
|
||||||
|
t.Errorf("after change: updated=%v err=%v, want true/nil", u, err)
|
||||||
|
}
|
||||||
|
if u, err := p.Fetch(); err != nil || u {
|
||||||
|
t.Errorf("steady Fetch: updated=%v err=%v, want false/nil", u, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPollFiles_MissingFile(t *testing.T) {
|
||||||
|
p := dataset.PollFiles(t.TempDir() + "/nope.txt")
|
||||||
|
if _, err := p.Fetch(); err == nil {
|
||||||
|
t.Error("expected error for missing file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFetcherFunc(t *testing.T) {
|
func TestFetcherFunc(t *testing.T) {
|
||||||
var called bool
|
var called bool
|
||||||
f := dataset.FetcherFunc(func() (bool, error) {
|
f := dataset.FetcherFunc(func() (bool, error) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user