diff --git a/cmd/check-ip/main.go b/cmd/check-ip/main.go index 5d74180..ddb497b 100644 --- a/cmd/check-ip/main.go +++ b/cmd/check-ip/main.go @@ -292,7 +292,10 @@ func openBlocklists(cfg Config) ( func newBlocklistFetcher(cfg Config) (fetcher dataset.Fetcher, inPaths, outPaths []string, err error) { switch { case cfg.Inbound != "" || cfg.Outbound != "": - return dataset.NopFetcher{}, splitCSV(cfg.Inbound), splitCSV(cfg.Outbound), nil + inPaths := splitCSV(cfg.Inbound) + outPaths := splitCSV(cfg.Outbound) + all := append(append([]string(nil), inPaths...), outPaths...) + return dataset.PollFiles(all...), inPaths, outPaths, nil case cfg.GitURL != "": dir, err := cacheDir(cfg.DataDir) diff --git a/sync/dataset/dataset.go b/sync/dataset/dataset.go index 6d9a633..f35693d 100644 --- a/sync/dataset/dataset.go +++ b/sync/dataset/dataset.go @@ -16,6 +16,8 @@ package dataset import ( "context" + "os" + "sync" "sync/atomic" "time" ) @@ -32,12 +34,52 @@ type FetcherFunc func() (bool, error) func (f FetcherFunc) Fetch() (bool, error) { return f() } -// NopFetcher always reports no update. Use for groups backed by local files -// that don't need a refresh cycle. +// NopFetcher always reports no update. Use for groups whose source never +// changes (test fixtures, embedded data). type NopFetcher struct{} func (NopFetcher) Fetch() (bool, error) { return false, nil } +// PollFiles returns a Fetcher that stat's the given paths and reports +// "updated" whenever any file's size or modtime has changed since the last +// call. The first call always reports updated=true. +// +// Use for Group's whose source is local files that may be edited out of band +// (e.g. a user-provided --inbound list) — pair with Group.Tick to pick up +// changes automatically. +func PollFiles(paths ...string) Fetcher { + return &filePoller{paths: paths, stats: make(map[string]fileStat, len(paths))} +} + +type fileStat struct { + size int64 + modTime time.Time +} + +type filePoller struct { + mu sync.Mutex + paths []string + stats map[string]fileStat +} + +func (p *filePoller) Fetch() (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + changed := false + for _, path := range p.paths { + info, err := os.Stat(path) + if err != nil { + return false, err + } + cur := fileStat{size: info.Size(), modTime: info.ModTime()} + if prev, ok := p.stats[path]; !ok || prev != cur { + changed = true + p.stats[path] = cur + } + } + return changed, nil +} + // Group ties one Fetcher to one or more views. A Load call fetches once and, // on the first call or when the source reports a change, reloads every view // and atomically swaps its current value. diff --git a/sync/dataset/dataset_test.go b/sync/dataset/dataset_test.go index f5d12b1..3cf64dd 100644 --- a/sync/dataset/dataset_test.go +++ b/sync/dataset/dataset_test.go @@ -2,8 +2,10 @@ package dataset_test import ( "errors" + "os" "sync/atomic" "testing" + "time" "github.com/therootcompany/golib/sync/dataset" ) @@ -127,6 +129,49 @@ func TestGroup_LoaderError(t *testing.T) { } } +func TestPollFiles(t *testing.T) { + dir := t.TempDir() + a := dir + "/a.txt" + b := dir + "/b.txt" + if err := os.WriteFile(a, []byte("1"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(b, []byte("2"), 0o644); err != nil { + t.Fatal(err) + } + + p := dataset.PollFiles(a, b) + + if u, err := p.Fetch(); err != nil || !u { + t.Fatalf("first Fetch: updated=%v err=%v, want true/nil", u, err) + } + if u, err := p.Fetch(); err != nil || u { + t.Fatalf("unchanged Fetch: updated=%v err=%v, want false/nil", u, err) + } + + // Bump mtime + change contents on b. + future := time.Now().Add(2 * time.Second) + if err := os.WriteFile(b, []byte("22"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(b, future, future); err != nil { + t.Fatal(err) + } + if u, err := p.Fetch(); err != nil || !u { + t.Errorf("after change: updated=%v err=%v, want true/nil", u, err) + } + if u, err := p.Fetch(); err != nil || u { + t.Errorf("steady Fetch: updated=%v err=%v, want false/nil", u, err) + } +} + +func TestPollFiles_MissingFile(t *testing.T) { + p := dataset.PollFiles(t.TempDir() + "/nope.txt") + if _, err := p.Fetch(); err == nil { + t.Error("expected error for missing file") + } +} + func TestFetcherFunc(t *testing.T) { var called bool f := dataset.FetcherFunc(func() (bool, error) {