feat(net/gitshallow): incremental shallow git-repo mirroring

New package for keeping a local mirror of a remote git repo via shallow
fetch+reset (not pull --ff-only, which loses force-pushed history). Adds
MaxAge gate via FETCH_HEAD mtime to skip redundant network calls under
concurrent callers, and a git-shallow-sync CLI. Includes integration
tests plus a force-push regression test.

net/gitshallow/cmd/git-shallow-sync/main.go

@@ -0,0 +1,92 @@
+// git-shallow-sync syncs a shallow git clone at the given local path,
+// cloning on first run and fetching + hard-resetting on subsequent runs.
+//
+// Usage:
+//
+//	git-shallow-sync <repository-url> <local-path>
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+const version = "dev"
+
+type Config struct {
+	Depth  int
+	Branch string
+}
+
+func main() {
+	cfg := Config{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	fs.IntVar(&cfg.Depth, "depth", 1, "clone/fetch depth (-1 for full history)")
+	fs.StringVar(&cfg.Branch, "branch", "", "branch to track (empty: remote default)")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s [flags] <repository-url> <local-path>\n", os.Args[0])
+		fs.PrintDefaults()
+	}
+
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "-V", "-version", "--version", "version":
+			fmt.Fprintf(os.Stdout, "git-shallow-sync %s\n", version)
+			os.Exit(0)
+		case "help", "-help", "--help":
+			fmt.Fprintf(os.Stdout, "git-shallow-sync %s\n\n", version)
+			fs.SetOutput(os.Stdout)
+			fs.Usage()
+			os.Exit(0)
+		}
+	}
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			os.Exit(0)
+		}
+		os.Exit(1)
+	}
+
+	args := fs.Args()
+	if len(args) != 2 {
+		fs.Usage()
+		os.Exit(1)
+	}
+	url := args[0]
+	path := args[1]
+
+	if len(path) > 0 && path[0] == '~' {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error: resolve home: %v\n", err)
+			os.Exit(1)
+		}
+		path = filepath.Join(home, path[1:])
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid path: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Fprintf(os.Stderr, "Syncing %s -> %s... ", url, absPath)
+	t := time.Now()
+	repo := gitshallow.New(url, absPath, cfg.Depth, cfg.Branch)
+	updated, err := repo.Sync()
+	if err != nil {
+		fmt.Fprintln(os.Stderr)
+		fmt.Fprintf(os.Stderr, "error: sync: %v\n", err)
+		os.Exit(1)
+	}
+	state := "already up to date"
+	if updated {
+		state = "updated"
+	}
+	fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state)
+}

net/gitshallow/forcepush_test.go

@@ -0,0 +1,102 @@
+package gitshallow_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+// TestRepo_ForcePushRecovery verifies that Fetch transparently picks up an
+// upstream history rewrite. The previous pull-based flow could fail with
+// "refusing to merge unrelated histories"; the current fetch+reset must
+// succeed and install the new HEAD.
+func TestRepo_ForcePushRecovery(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not available")
+	}
+
+	root := t.TempDir()
+	upstream := filepath.Join(root, "upstream.git") // bare
+	scratch := filepath.Join(root, "scratch")        // working copy that rewrites history
+	clone := filepath.Join(root, "clone")            // gitshallow clone under test
+
+	mustGit := func(t *testing.T, dir string, args ...string) {
+		t.Helper()
+		cmd := exec.Command("git", args...)
+		cmd.Dir = dir
+		cmd.Env = append(os.Environ(),
+			"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
+			"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
+		)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("git %v (in %s): %v\n%s", args, dir, err, out)
+		}
+	}
+
+	// Bare upstream.
+	if err := os.MkdirAll(upstream, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, upstream, "init", "--bare", "--initial-branch=main")
+
+	// Scratch working copy with one commit, push to upstream.
+	if err := os.MkdirAll(scratch, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "init", "--initial-branch=main")
+	mustGit(t, scratch, "remote", "add", "origin", upstream)
+	if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v1\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "add", "data.txt")
+	mustGit(t, scratch, "commit", "-m", "v1")
+	mustGit(t, scratch, "push", "-u", "origin", "main")
+
+	// gitshallow clones, reads v1.
+	repo := gitshallow.New(upstream, clone, 1, "main")
+	if updated, err := repo.Fetch(); err != nil {
+		t.Fatalf("first Fetch: %v", err)
+	} else if !updated {
+		t.Fatal("first Fetch: expected updated=true")
+	}
+	got, err := os.ReadFile(filepath.Join(clone, "data.txt"))
+	if err != nil {
+		t.Fatalf("read v1: %v", err)
+	}
+	if string(got) != "v1\n" {
+		t.Fatalf("v1: got %q want %q", got, "v1\n")
+	}
+
+	// Rewrite upstream history with an unrelated commit and force-push.
+	mustGit(t, scratch, "checkout", "--orphan", "fresh")
+	mustGit(t, scratch, "rm", "-rf", ".")
+	if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v2\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "add", "data.txt")
+	mustGit(t, scratch, "commit", "-m", "v2")
+	mustGit(t, scratch, "branch", "-M", "main")
+	mustGit(t, scratch, "push", "--force", "origin", "main")
+
+	// gitshallow must recover — fetch+reset, not pull --ff-only.
+	// Use a fresh instance: the same *Repo has a 1s debounce that would
+	// skip the follow-up fetch in this test's wall-clock window.
+	repo2 := gitshallow.New(upstream, clone, 1, "main")
+	updated, err := repo2.Fetch()
+	if err != nil {
+		t.Fatalf("post-force-push Fetch: %v", err)
+	}
+	if !updated {
+		t.Error("post-force-push Fetch: expected updated=true")
+	}
+	got, err = os.ReadFile(filepath.Join(clone, "data.txt"))
+	if err != nil {
+		t.Fatalf("read v2: %v", err)
+	}
+	if string(got) != "v2\n" {
+		t.Fatalf("v2: got %q want %q", got, "v2\n")
+	}
+}

net/gitshallow/gitshallow.go

@@ -0,0 +1,291 @@
+package gitshallow
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Repo manages a shallow git clone used as a periodically-updated data source.
+type Repo struct {
+	URL    string
+	Path   string
+	Depth  int    // 0 defaults to 1, -1 for all
+	Branch string // Optional: specific branch to clone/pull
+
+	// MaxAge skips the git fetch when .git/FETCH_HEAD is younger than this
+	// duration. Persists across process restarts (unlike the in-memory
+	// lastSynced debounce) — so repeated short-lived CLI invocations don't
+	// hammer the remote. 0 disables.
+	MaxAge time.Duration
+
+	// GCInterval controls explicit aggressive GC after pulls.
+	//   0 (default) — no explicit gc; git runs gc.auto on its own schedule
+	//   1           — aggressive gc after every pull
+	//   N           — aggressive gc after every Nth pull
+	GCInterval int
+
+	mu         sync.Mutex
+	pullCount  int
+	lastSynced time.Time
+}
+
+// New creates a new Repo instance.
+func New(url, path string, depth int, branch string) *Repo {
+	return &Repo{
+		URL:    url,
+		Path:   path,
+		Depth:  depth,
+		Branch: strings.TrimSpace(branch),
+	}
+}
+
+// effectiveDepth returns the depth to use for clone/pull.
+// 0 means unset — defaults to 1. -1 means full history.
+func (r *Repo) effectiveDepth() int {
+	if r.Depth == 0 {
+		return 1
+	}
+	return r.Depth
+}
+
+// Init clones the repo if missing, then syncs once.
+// Returns whether anything new was fetched.
+func (r *Repo) Init() (bool, error) {
+	gitDir := filepath.Join(r.Path, ".git")
+	if _, err := os.Stat(gitDir); err != nil {
+		if _, err := r.Clone(); err != nil {
+			return false, err
+		}
+	}
+	return r.syncGit()
+}
+
+// Clone performs a shallow clone (--depth N --single-branch --no-tags).
+func (r *Repo) Clone() (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.clone()
+}
+
+func (r *Repo) clone() (bool, error) {
+	if r.exists() {
+		return false, nil
+	}
+	if r.URL == "" {
+		return false, fmt.Errorf("repository URL is required")
+	}
+	if r.Path == "" {
+		return false, fmt.Errorf("local path is required")
+	}
+	if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
+		return false, err
+	}
+
+	args := []string{"clone", "--no-tags"}
+	if depth := r.effectiveDepth(); depth >= 0 {
+		args = append(args, "--depth", fmt.Sprintf("%d", depth))
+	}
+	args = append(args, "--single-branch")
+	if r.Branch != "" {
+		args = append(args, "--branch", r.Branch)
+	}
+	args = append(args, r.URL, filepath.Base(r.Path))
+
+	_, err := r.runGit(args...)
+	return true, err
+}
+
+// exists checks if the directory contains a .git folder.
+func (r *Repo) exists() bool {
+	_, err := os.Stat(filepath.Join(r.Path, ".git"))
+	return err == nil
+}
+
+// runGit executes a git command in the repo directory (or parent for clone).
+func (r *Repo) runGit(args ...string) (string, error) {
+	cmd := exec.Command("git", args...)
+	if _, err := os.Stat(r.Path); err == nil && r.exists() {
+		cmd.Dir = r.Path
+	} else {
+		cmd.Dir = filepath.Dir(r.Path)
+	}
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("git %s failed: %v\n%s", strings.Join(args, " "), err, output)
+	}
+	return strings.TrimSpace(string(output)), nil
+}
+
+// Pull fetches from origin and hard-resets the working tree to the remote
+// branch, reporting whether HEAD changed. This is a read-only mirror — we
+// never try to merge, so force-pushes upstream are handled transparently.
+func (r *Repo) Pull() (updated bool, err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.pull()
+}
+
+func (r *Repo) pull() (updated bool, err error) {
+	if !r.exists() {
+		return false, fmt.Errorf("repository does not exist at %s", r.Path)
+	}
+
+	oldHead, _ := r.runGit("rev-parse", "HEAD")
+
+	branch := r.Branch
+	if branch == "" {
+		out, err := r.runGit("symbolic-ref", "--short", "refs/remotes/origin/HEAD")
+		if err != nil {
+			return false, err
+		}
+		_, branch, _ = strings.Cut(out, "/")
+	}
+
+	fetchArgs := []string{"fetch", "--no-tags"}
+	if depth := r.effectiveDepth(); depth >= 0 {
+		fetchArgs = append(fetchArgs, "--depth", fmt.Sprintf("%d", depth))
+	}
+	fetchArgs = append(fetchArgs, "origin", branch)
+	if _, err := r.runGit(fetchArgs...); err != nil {
+		return false, err
+	}
+
+	if _, err := r.runGit("reset", "--hard", "origin/"+branch); err != nil {
+		return false, err
+	}
+
+	newHead, err := r.runGit("rev-parse", "HEAD")
+	if err != nil {
+		return false, err
+	}
+	return oldHead != newHead, nil
+}
+
+// GC runs git gc --aggressive --prune=now.
+func (r *Repo) GC() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.gc()
+}
+
+func (r *Repo) gc() error {
+	if !r.exists() {
+		return fmt.Errorf("repository does not exist at %s", r.Path)
+	}
+	_, err := r.runGit("gc", "--aggressive", "--prune=now")
+	return err
+}
+
+// Sync clones if missing, pulls, and conditionally runs GC based on GCEvery.
+// Returns whether HEAD changed.
+func (r *Repo) Sync() (bool, error) {
+	return r.syncGit()
+}
+
+// Fetch syncs the repo and reports whether HEAD changed.
+func (r *Repo) Fetch() (bool, error) {
+	return r.syncGit()
+}
+
+// FilePath returns the absolute path to relPath within this repo.
+func (r *Repo) FilePath(rel string) string {
+	return filepath.Join(r.Path, rel)
+}
+
+// File returns a handle to relPath within this repo.
+// The handle's Path and Open methods give access to the file; its Fetch method
+// syncs the repo and reports whether this specific file changed (by mtime).
+func (r *Repo) File(relPath string) *File {
+	return &File{repo: r, rel: relPath}
+}
+
+// File is a handle to a single file inside a Repo.
+// Fetch syncs the repo (deduped across all File handles sharing the same
+// Repo) and reports whether this file changed.
+type File struct {
+	repo    *Repo
+	rel     string
+	mu      sync.Mutex
+	lastMod time.Time
+}
+
+// Path returns the absolute path to the file.
+func (f *File) Path() string {
+	return filepath.Join(f.repo.Path, f.rel)
+}
+
+// Open returns an open *os.File for reading. The caller must Close it.
+func (f *File) Open() (*os.File, error) {
+	return os.Open(f.Path())
+}
+
+// Fetch syncs the repo and reports whether this file changed since last call.
+// Safe to call concurrently.
+func (f *File) Fetch() (bool, error) {
+	if _, err := f.repo.syncGit(); err != nil {
+		return false, err
+	}
+	info, err := os.Stat(f.Path())
+	if err != nil {
+		return false, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if info.ModTime().Equal(f.lastMod) {
+		return false, nil
+	}
+	f.lastMod = info.ModTime()
+	return true, nil
+}
+
+func (r *Repo) syncGit() (updated bool, err error) {
+	// MaxAge: file-mtime gate (FETCH_HEAD is rewritten on every successful
+	// fetch, so its mtime is "last time we talked to the remote"). Checked
+	// outside the lock — just a stat.
+	if r.MaxAge > 0 {
+		if info, err := os.Stat(filepath.Join(r.Path, ".git", "FETCH_HEAD")); err == nil {
+			if time.Since(info.ModTime()) < r.MaxAge {
+				return false, nil
+			}
+		}
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// If another caller just finished a sync while we were waiting for the
+	// lock, skip the pull — the repo is already current.
+	if !r.lastSynced.IsZero() && time.Since(r.lastSynced) < time.Second {
+		return false, nil
+	}
+
+	if cloned, err := r.clone(); err != nil {
+		return false, err
+	} else if cloned {
+		r.lastSynced = time.Now()
+		return true, nil
+	}
+
+	updated, err = r.pull()
+	if err != nil {
+		return false, err
+	}
+	r.lastSynced = time.Now()
+	if !updated {
+		return false, nil
+	}
+
+	if r.GCInterval > 0 {
+		r.pullCount++
+		if r.pullCount%r.GCInterval == 0 {
+			return true, r.gc()
+		}
+	}
+
+	return true, nil
+}

net/gitshallow/gitshallow_integration_test.go

@@ -0,0 +1,101 @@
+//go:build integration
+
+package gitshallow_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+const testRepoURL = "https://github.com/bitwire-it/ipblocklist"
+
+func testdataDir(t *testing.T) string {
+	t.Helper()
+	dir, _ := filepath.Abs(".")
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return filepath.Join(dir, "testdata")
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			t.Fatal("could not find module root (go.mod)")
+		}
+		dir = parent
+	}
+}
+
+func repoDir(t *testing.T) string {
+	return filepath.Join(testdataDir(t), "gitshallow_ipblocklist")
+}
+
+func TestRepo_Clone(t *testing.T) {
+	dir := repoDir(t)
+	os.RemoveAll(dir)
+
+	repo := gitshallow.New(testRepoURL, dir, 1, "")
+	updated, err := repo.Fetch()
+	if err != nil {
+		t.Fatalf("Fetch (clone): %v", err)
+	}
+	if !updated {
+		t.Error("fresh clone: expected updated=true")
+	}
+
+	for _, rel := range []string{
+		"tables/inbound/single_ips.txt",
+		"tables/inbound/networks.txt",
+		"tables/outbound/single_ips.txt",
+		"tables/outbound/networks.txt",
+	} {
+		info, err := os.Stat(filepath.Join(dir, rel))
+		if err != nil {
+			t.Errorf("expected file missing: %s", rel)
+		} else {
+			t.Logf("%s: %d bytes", rel, info.Size())
+		}
+	}
+}
+
+func TestRepo_Pull_SameInstance(t *testing.T) {
+	dir := repoDir(t)
+
+	repo := gitshallow.New(testRepoURL, dir, 1, "")
+	// Ensure cloned.
+	if _, err := repo.Fetch(); err != nil {
+		t.Fatalf("initial Fetch: %v", err)
+	}
+
+	// Second pull on the same instance — already at HEAD, should not advance.
+	updated, err := repo.Fetch()
+	if err != nil {
+		t.Fatalf("second Fetch: %v", err)
+	}
+	if updated {
+		t.Error("same-instance second pull: expected updated=false (already at HEAD)")
+	}
+	t.Log("same-instance pull correctly reported no update")
+}
+
+func TestRepo_Pull_FreshInstance(t *testing.T) {
+	dir := repoDir(t)
+
+	// Ensure cloned via a first instance.
+	first := gitshallow.New(testRepoURL, dir, 1, "")
+	if _, err := first.Fetch(); err != nil {
+		t.Fatalf("initial Fetch: %v", err)
+	}
+
+	// Fresh instance with no in-memory state — git HEAD on disk drives the check.
+	fresh := gitshallow.New(testRepoURL, dir, 1, "")
+	updated, err := fresh.Fetch()
+	if err != nil {
+		t.Fatalf("fresh-instance Fetch: %v", err)
+	}
+	if updated {
+		t.Error("fresh-instance pull: expected updated=false (HEAD unchanged on disk)")
+	}
+	t.Log("fresh-instance pull correctly reported no update")
+}

net/gitshallow/go.mod

@@ -0,0 +1,3 @@
+module github.com/therootcompany/golib/net/gitshallow
+
+go 1.26.0