diff --git a/net/gitdataset/gitdataset.go b/net/gitdataset/gitdataset.go new file mode 100644 index 0000000..fad3cae --- /dev/null +++ b/net/gitdataset/gitdataset.go @@ -0,0 +1,92 @@ +package gitdataset + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "time" + + "github.com/therootcompany/golib/net/gitshallow" +) + +// TODO maybe a GitRepo should contain GitDatasets such that loading +// multiple datasets from the same GitRepo won't cause issues with file locking? + +type GitDataset[T any] struct { + LoadFile func(path string) (*T, error) + atomic.Pointer[T] + gitRepo string + shallowRepo *gitshallow.ShallowRepo + path string +} + +func New[T any](gitURL, path string, loadFile func(path string) (*T, error)) *GitDataset[T] { + gitRepo := filepath.Dir(path) + gitDepth := 1 + gitBranch := "" + shallowRepo := gitshallow.New(gitURL, gitRepo, gitDepth, gitBranch) + + b := &GitDataset[T]{ + Pointer: atomic.Pointer[T]{}, + LoadFile: loadFile, + gitRepo: gitRepo, + shallowRepo: shallowRepo, + path: path, + } + b.Store(new(T)) + return b +} + +func (b *GitDataset[T]) Init(skipGC bool) (updated bool, err error) { + gitDir := filepath.Join(b.gitRepo, ".git") + if _, err := os.Stat(gitDir); err != nil { + if _, err := b.shallowRepo.Clone(); err != nil { + return false, err + } + } + + force := true + return b.reload(skipGC, force) +} + +func (b *GitDataset[T]) Run(ctx context.Context) { + ticker := time.NewTicker(47 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if ok, err := b.reload(false, false); err != nil { + fmt.Fprintf(os.Stderr, "error: git data: %v\n", err) + } else if ok { + fmt.Fprintf(os.Stderr, "git data: loaded repo\n") + } else { + fmt.Fprintf(os.Stderr, "git data: already up-to-date\n") + } + case <-ctx.Done(): + return + } + } +} + +func (b *GitDataset[T]) reload(skipGC, force bool) (updated bool, err error) { + laxGC := skipGC + lazyPrune := skipGC + updated, err = b.shallowRepo.Sync(laxGC, lazyPrune) + if err != nil { + return false, fmt.Errorf("git sync: %w", err) + } + if !updated && !force { + return false, nil + } + + nextDataset, err := b.LoadFile(b.path) + if err != nil { + return false, err + } + + _ = b.Swap(nextDataset) + return true, nil +} diff --git a/net/ipcohort/cmd/check-ip-blacklist/blacklist.go b/net/ipcohort/cmd/check-ip-blacklist/blacklist.go deleted file mode 100644 index 4b4b6d4..0000000 --- a/net/ipcohort/cmd/check-ip-blacklist/blacklist.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "os" - "path/filepath" - "time" - - "github.com/therootcompany/golib/net/gitshallow" - "github.com/therootcompany/golib/net/ipcohort" -) - -type Blacklist struct { - *ipcohort.Cohort - gitRepo string - shallowRepo *gitshallow.ShallowRepo - path string -} - -func NewBlacklist(gitURL, path string) *Blacklist { - gitRepo := filepath.Dir(path) - gitDepth := 1 - gitBranch := "" - shallowRepo := gitshallow.New(gitURL, gitRepo, gitDepth, gitBranch) - - return &Blacklist{ - Cohort: ipcohort.New(), - gitRepo: gitRepo, - shallowRepo: shallowRepo, - path: path, - } -} - -func (b *Blacklist) Init(skipGC bool) (int, error) { - gitDir := filepath.Join(b.gitRepo, ".git") - if _, err := os.Stat(gitDir); err != nil { - if _, err := b.shallowRepo.Clone(); err != nil { - log.Fatalf("Failed to load blacklist: %v", err) - fmt.Printf("%q is not a git repo, skipping sync\n", b.gitRepo) - return b.Size(), nil - } - } - - force := true - return b.reload(skipGC, force) -} - -func (r Blacklist) Run(ctx context.Context) { - ticker := time.NewTicker(47 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if n, err := r.reload(false, false); err != nil { - fmt.Fprintf(os.Stderr, "error: ip cohort: %v\n", err) - } else { - fmt.Fprintf(os.Stderr, "ip cohort: loaded %d blacklist entries\n", n) - } - case <-ctx.Done(): - return - } - } -} - -func (b Blacklist) reload(skipGC, force bool) (int, error) { - laxGC := skipGC - lazyPrune := skipGC - updated, err := b.shallowRepo.Sync(laxGC, lazyPrune) - if err != nil { - return 0, fmt.Errorf("git sync: %w", err) - } - if !updated && !force { - return 0, nil - } - - needsSort := false - nextCohort, err := ipcohort.LoadFile(b.path, needsSort) - if err != nil { - return 0, fmt.Errorf("ip cohort: %w", err) - } - - b.Swap(nextCohort) - return b.Size(), nil -}