feat(net/gitshallow): incremental shallow git-repo mirroring

Periodically-updated shallow clone for repos used as a data source.
One Fetch verb (clone-or-pull-or-skip) drives both the Fetcher
interface and per-File handles. MaxAge gates on FETCH_HEAD mtime so
short-lived CLI invocations don't hammer the remote across process
restarts; an in-memory 1s debounce coalesces concurrent Fetches.
Hardens git invocations against argument injection: validates
branch/ref shape, rejects URLs starting with '-', and uses '--'
separators before positional args. Userinfo in URL is redacted from
error output (both args echo and git's own messages).

.gitignore

@@ -1,4 +1,5 @@
 credentials.tsv
+testdata/
 
 .env
 *.env

net/gitshallow/cmd/git-shallow-sync/main.go

@@ -0,0 +1,115 @@
+// git-shallow-sync syncs a shallow git clone at the given local path,
+// cloning on first run and fetching + hard-resetting on subsequent runs.
+//
+// Usage:
+//
+//	git-shallow-sync <repository-url> <local-path>
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+// Replaced by goreleaser / ldflags at build time.
+var (
+	name         = "git-shallow-sync"
+	version      = "0.0.0-dev"
+	commit       = "0000000"
+	date         = "0001-01-01"
+	licenseYear  = "2021-present"
+	licenseOwner = "AJ ONeal <aj@therootcompany.com>"
+	licenseType  = "MPL-2.0"
+)
+
+type Config struct {
+	Depth  int
+	Branch string
+}
+
+func printVersion(w io.Writer) {
+	_, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, version, commit[:7], date)
+	_, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner)
+	_, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType)
+}
+
+func main() {
+	cfg := Config{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	fs.IntVar(&cfg.Depth, "depth", 1, "clone/fetch depth (-1 for full history)")
+	fs.StringVar(&cfg.Branch, "branch", "", "branch to track (empty: remote default)")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s [flags] <repository-url> <local-path>\n", os.Args[0])
+		fs.PrintDefaults()
+	}
+
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "-V", "-version", "--version", "version":
+			printVersion(os.Stdout)
+			os.Exit(0)
+		case "help", "-help", "--help":
+			printVersion(os.Stdout)
+			fmt.Fprintln(os.Stdout, "")
+			fs.SetOutput(os.Stdout)
+			fs.Usage()
+			os.Exit(0)
+		}
+	}
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			os.Exit(0)
+		}
+		os.Exit(1)
+	}
+
+	args := fs.Args()
+	if len(args) != 2 {
+		fs.Usage()
+		os.Exit(1)
+	}
+	url := args[0]
+	path := args[1]
+
+	if len(path) > 0 && path[0] == '~' {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error: resolve home: %v\n", err)
+			os.Exit(1)
+		}
+		path = filepath.Join(home, path[1:])
+	}
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid path: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	fmt.Fprintf(os.Stderr, "Syncing %s -> %s... ", url, absPath)
+	t := time.Now()
+	repo := gitshallow.New(url, absPath, cfg.Depth, cfg.Branch)
+	updated, err := repo.Fetch(ctx)
+	if err != nil {
+		fmt.Fprintln(os.Stderr)
+		fmt.Fprintf(os.Stderr, "error: sync: %v\n", err)
+		os.Exit(1)
+	}
+	state := "already up to date"
+	if updated {
+		state = "updated"
+	}
+	fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state)
+}

net/gitshallow/forcepush_test.go

@@ -0,0 +1,102 @@
+package gitshallow_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+// TestRepo_ForcePushRecovery verifies that Fetch transparently picks up an
+// upstream history rewrite. The previous pull-based flow could fail with
+// "refusing to merge unrelated histories"; the current fetch+reset must
+// succeed and install the new HEAD.
+func TestRepo_ForcePushRecovery(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not available")
+	}
+
+	root := t.TempDir()
+	upstream := filepath.Join(root, "upstream.git") // bare
+	scratch := filepath.Join(root, "scratch")       // working copy that rewrites history
+	clone := filepath.Join(root, "clone")           // gitshallow clone under test
+
+	mustGit := func(t *testing.T, dir string, args ...string) {
+		t.Helper()
+		cmd := exec.Command("git", args...)
+		cmd.Dir = dir
+		cmd.Env = append(os.Environ(),
+			"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
+			"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
+		)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("git %v (in %s): %v\n%s", args, dir, err, out)
+		}
+	}
+
+	// Bare upstream.
+	if err := os.MkdirAll(upstream, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, upstream, "init", "--bare", "--initial-branch=main")
+
+	// Scratch working copy with one commit, push to upstream.
+	if err := os.MkdirAll(scratch, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "init", "--initial-branch=main")
+	mustGit(t, scratch, "remote", "add", "origin", upstream)
+	if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v1\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "add", "data.txt")
+	mustGit(t, scratch, "commit", "-m", "v1")
+	mustGit(t, scratch, "push", "-u", "origin", "main")
+
+	// gitshallow clones, reads v1.
+	repo := gitshallow.New(upstream, clone, 1, "main")
+	if updated, err := repo.Fetch(t.Context()); err != nil {
+		t.Fatalf("first Fetch: %v", err)
+	} else if !updated {
+		t.Fatal("first Fetch: expected updated=true")
+	}
+	got, err := os.ReadFile(filepath.Join(clone, "data.txt"))
+	if err != nil {
+		t.Fatalf("read v1: %v", err)
+	}
+	if string(got) != "v1\n" {
+		t.Fatalf("v1: got %q want %q", got, "v1\n")
+	}
+
+	// Rewrite upstream history with an unrelated commit and force-push.
+	mustGit(t, scratch, "checkout", "--orphan", "fresh")
+	mustGit(t, scratch, "rm", "-rf", ".")
+	if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v2\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	mustGit(t, scratch, "add", "data.txt")
+	mustGit(t, scratch, "commit", "-m", "v2")
+	mustGit(t, scratch, "branch", "-M", "main")
+	mustGit(t, scratch, "push", "--force", "origin", "main")
+
+	// gitshallow must recover — fetch+reset, not pull --ff-only.
+	// Use a fresh instance: the same *Repo has a 1s debounce that would
+	// skip the follow-up fetch in this test's wall-clock window.
+	repo2 := gitshallow.New(upstream, clone, 1, "main")
+	updated, err := repo2.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("post-force-push Fetch: %v", err)
+	}
+	if !updated {
+		t.Error("post-force-push Fetch: expected updated=true")
+	}
+	got, err = os.ReadFile(filepath.Join(clone, "data.txt"))
+	if err != nil {
+		t.Fatalf("read v2: %v", err)
+	}
+	if string(got) != "v2\n" {
+		t.Fatalf("v2: got %q want %q", got, "v2\n")
+	}
+}

net/gitshallow/gitshallow.go

@@ -0,0 +1,313 @@
+package gitshallow
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+)
+
+// Repo manages a shallow git clone used as a periodically-updated data source.
+type Repo struct {
+	URL    string
+	Path   string
+	Depth  int    // 0 defaults to 1, -1 for all
+	Branch string // Optional: specific branch to clone/pull
+
+	// MaxAge skips the git fetch when .git/FETCH_HEAD is younger than this
+	// duration. Persists across process restarts (unlike the in-process
+	// singleflight dedup) — so repeated short-lived CLI invocations don't
+	// hammer the remote. 0 defaults to 1 minute; -1 disables.
+	MaxAge time.Duration
+
+	// GCInterval controls explicit aggressive GC after pulls.
+	//   0 (default) — no explicit gc; git runs gc.auto on its own schedule
+	//   1           — aggressive gc after every pull
+	//   N           — aggressive gc after every Nth pull
+	GCInterval int
+
+	sf        singleflight.Group
+	mu        sync.Mutex // guards pullCount
+	pullCount int
+}
+
+// New creates a new Repo instance.
+func New(url, path string, depth int, branch string) *Repo {
+	return &Repo{
+		URL:    url,
+		Path:   path,
+		Depth:  depth,
+		Branch: strings.TrimSpace(branch),
+	}
+}
+
+// validRef matches branch/ref names that are safe to pass to git as a
+// positional argument: no leading dash (which git would treat as a flag),
+// no whitespace, no shell metacharacters.
+var validRef = regexp.MustCompile(`^[A-Za-z0-9._/-]+$`)
+
+// validateArgs rejects branch values git would interpret as options.
+// URL is passed to git after a `--` flag terminator, so git won't
+// reinterpret a leading `-` as an option.
+func (r *Repo) validateArgs() error {
+	if _, err := url.Parse(r.URL); err != nil {
+		return fmt.Errorf("invalid URL %q: %w", r.URL, err)
+	}
+	if r.Branch != "" && !validRef.MatchString(r.Branch) {
+		return fmt.Errorf("branch %q contains invalid characters", r.Branch)
+	}
+	return nil
+}
+
+// effectiveDepth returns the depth to use for clone/pull.
+// 0 means unset — defaults to 1. -1 means full history.
+func (r *Repo) effectiveDepth() int {
+	if r.Depth == 0 {
+		return 1
+	}
+	return r.Depth
+}
+
+// effectiveMaxAge returns the MaxAge to apply.
+// 0 means unset — defaults to 1 minute. -1 disables the gate.
+func (r *Repo) effectiveMaxAge() time.Duration {
+	switch {
+	case r.MaxAge == 0:
+		return time.Minute
+	case r.MaxAge < 0:
+		return 0
+	default:
+		return r.MaxAge
+	}
+}
+
+func (r *Repo) clone(ctx context.Context) (bool, error) {
+	if r.exists() {
+		return false, nil
+	}
+	if r.URL == "" {
+		return false, fmt.Errorf("repository URL is required")
+	}
+	if r.Path == "" {
+		return false, fmt.Errorf("local path is required")
+	}
+	if err := r.validateArgs(); err != nil {
+		return false, err
+	}
+	if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
+		return false, err
+	}
+
+	args := []string{"clone", "--no-tags"}
+	if depth := r.effectiveDepth(); depth >= 0 {
+		args = append(args, "--depth", fmt.Sprintf("%d", depth))
+	}
+	args = append(args, "--single-branch")
+	if r.Branch != "" {
+		args = append(args, "--branch", r.Branch)
+	}
+	// `--` separates flags from positional URL/path so a URL beginning
+	// with `-` cannot be reinterpreted by git as an option.
+	args = append(args, "--", r.URL, filepath.Base(r.Path))
+
+	_, err := r.runGit(ctx, args...)
+	return true, err
+}
+
+// exists checks if the directory contains a .git folder.
+func (r *Repo) exists() bool {
+	_, err := os.Stat(filepath.Join(r.Path, ".git"))
+	return err == nil
+}
+
+// runGit executes a git command in the repo directory (or parent for clone).
+// ctx cancels the child git process via SIGKILL.
+func (r *Repo) runGit(ctx context.Context, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, "git", args...)
+	if _, err := os.Stat(r.Path); err == nil && r.exists() {
+		cmd.Dir = r.Path
+	} else {
+		cmd.Dir = filepath.Dir(r.Path)
+	}
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		// Redact userinfo from both the args echo (we may have just passed
+		// r.URL) and from git's own output (git embeds the URL in messages
+		// like "fatal: unable to access 'https://user:pass@host/'"). If a
+		// caller put credentials in the URL — even though the docs steer
+		// them to Authorization headers — they don't end up in error logs.
+		safeArgs := redactUserinfo(strings.Join(args, " "), r.URL)
+		safeOut := redactUserinfo(string(output), r.URL)
+		return "", fmt.Errorf("git %s failed: %w\n%s", safeArgs, err, safeOut)
+	}
+	return strings.TrimSpace(string(output)), nil
+}
+
+// redactUserinfo returns s with any "user:pass@" or "user@" segment from
+// rawURL replaced with "redacted@". No-op if rawURL has no userinfo.
+func redactUserinfo(s, rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.User == nil {
+		return s
+	}
+	return strings.ReplaceAll(s, u.User.String()+"@", "redacted@")
+}
+
+func (r *Repo) pull(ctx context.Context) (updated bool, err error) {
+	if !r.exists() {
+		return false, fmt.Errorf("repository does not exist at %s", r.Path)
+	}
+	if err := r.validateArgs(); err != nil {
+		return false, err
+	}
+
+	oldHead, _ := r.runGit(ctx, "rev-parse", "HEAD")
+
+	branch := r.Branch
+	if branch == "" {
+		out, err := r.runGit(ctx, "symbolic-ref", "--short", "refs/remotes/origin/HEAD")
+		if err != nil {
+			return false, err
+		}
+		_, branch, _ = strings.Cut(out, "/")
+		if !validRef.MatchString(branch) {
+			return false, fmt.Errorf("remote default branch %q contains invalid characters", branch)
+		}
+	}
+
+	fetchArgs := []string{"fetch", "--no-tags"}
+	if depth := r.effectiveDepth(); depth >= 0 {
+		fetchArgs = append(fetchArgs, "--depth", fmt.Sprintf("%d", depth))
+	}
+	// `--` separates flags from positional remote/branch.
+	fetchArgs = append(fetchArgs, "--", "origin", branch)
+	if _, err := r.runGit(ctx, fetchArgs...); err != nil {
+		return false, err
+	}
+
+	if _, err := r.runGit(ctx, "reset", "--hard", "origin/"+branch); err != nil {
+		return false, err
+	}
+
+	newHead, err := r.runGit(ctx, "rev-parse", "HEAD")
+	if err != nil {
+		return false, err
+	}
+	return oldHead != newHead, nil
+}
+
+func (r *Repo) gc(ctx context.Context) error {
+	if !r.exists() {
+		return fmt.Errorf("repository does not exist at %s", r.Path)
+	}
+	_, err := r.runGit(ctx, "gc", "--aggressive", "--prune=now")
+	return err
+}
+
+// Fetch clones the repo if missing, pulls otherwise, and conditionally runs
+// GC based on GCInterval. Returns whether HEAD changed. Implements Fetcher.
+// Safe to call concurrently — concurrent callers share a single in-flight
+// fetch (via singleflight) and all receive the same result. To force a
+// pull regardless of MaxAge, set MaxAge=-1 before calling.
+func (r *Repo) Fetch(ctx context.Context) (updated bool, err error) {
+	// MaxAge: file-mtime gate (FETCH_HEAD is rewritten on every successful
+	// fetch, so its mtime is "last time we talked to the remote").
+	if maxAge := r.effectiveMaxAge(); maxAge > 0 {
+		if info, err := os.Stat(filepath.Join(r.Path, ".git", "FETCH_HEAD")); err == nil {
+			if time.Since(info.ModTime()) < maxAge {
+				return false, nil
+			}
+		}
+	}
+
+	v, err, _ := r.sf.Do("fetch", func() (any, error) {
+		return r.fetch(ctx)
+	})
+	if err != nil {
+		return false, err
+	}
+	return v.(bool), nil
+}
+
+func (r *Repo) fetch(ctx context.Context) (bool, error) {
+	if cloned, err := r.clone(ctx); err != nil {
+		return false, err
+	} else if cloned {
+		return true, nil
+	}
+
+	updated, err := r.pull(ctx)
+	if err != nil {
+		return false, err
+	}
+	if !updated {
+		return false, nil
+	}
+
+	if r.GCInterval > 0 {
+		r.mu.Lock()
+		r.pullCount++
+		shouldGC := r.pullCount%r.GCInterval == 0
+		r.mu.Unlock()
+		if shouldGC {
+			return true, r.gc(ctx)
+		}
+	}
+
+	return true, nil
+}
+
+// FilePath returns the absolute path to relPath within this repo.
+func (r *Repo) FilePath(rel string) string {
+	return filepath.Join(r.Path, rel)
+}
+
+// File returns a handle to relPath within this repo. The handle's Path and
+// Open methods give access to the file; its Fetch method syncs the repo and
+// reports whether this specific file changed (by mtime).
+func (r *Repo) File(relPath string) *RepoFile {
+	return &RepoFile{repo: r, rel: relPath}
+}
+
+// RepoFile is a handle to a single file inside a Repo. Fetch syncs the repo
+// (deduped across all RepoFile handles sharing the same Repo) and reports
+// whether this file changed. The Repo prefix avoids collision with os.File,
+// which is the type returned by Open.
+type RepoFile struct {
+	repo    *Repo
+	rel     string
+	mu      sync.Mutex
+	lastMod time.Time
+}
+
+// Path returns the absolute path to the file.
+func (f *RepoFile) Path() string {
+	return filepath.Join(f.repo.Path, f.rel)
+}
+
+// Fetch syncs the repo and reports whether this file changed since last call.
+// Safe to call concurrently. Implements Fetcher.
+func (f *RepoFile) Fetch(ctx context.Context) (bool, error) {
+	if _, err := f.repo.Fetch(ctx); err != nil {
+		return false, err
+	}
+	info, err := os.Stat(f.Path())
+	if err != nil {
+		return false, err
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if info.ModTime().Equal(f.lastMod) {
+		return false, nil
+	}
+	f.lastMod = info.ModTime()
+	return true, nil
+}

net/gitshallow/gitshallow_integration_test.go

@@ -0,0 +1,101 @@
+//go:build integration
+
+package gitshallow_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/therootcompany/golib/net/gitshallow"
+)
+
+const testRepoURL = "https://github.com/bitwire-it/ipblocklist"
+
+func testdataDir(t *testing.T) string {
+	t.Helper()
+	dir, _ := filepath.Abs(".")
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return filepath.Join(dir, "testdata")
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			t.Fatal("could not find module root (go.mod)")
+		}
+		dir = parent
+	}
+}
+
+func repoDir(t *testing.T) string {
+	return filepath.Join(testdataDir(t), "gitshallow_ipblocklist")
+}
+
+func TestRepo_Clone(t *testing.T) {
+	dir := repoDir(t)
+	os.RemoveAll(dir)
+
+	repo := gitshallow.New(testRepoURL, dir, 1, "")
+	updated, err := repo.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("Fetch (clone): %v", err)
+	}
+	if !updated {
+		t.Error("fresh clone: expected updated=true")
+	}
+
+	for _, rel := range []string{
+		"tables/inbound/single_ips.txt",
+		"tables/inbound/networks.txt",
+		"tables/outbound/single_ips.txt",
+		"tables/outbound/networks.txt",
+	} {
+		info, err := os.Stat(filepath.Join(dir, rel))
+		if err != nil {
+			t.Errorf("expected file missing: %s", rel)
+		} else {
+			t.Logf("%s: %d bytes", rel, info.Size())
+		}
+	}
+}
+
+func TestRepo_Pull_SameInstance(t *testing.T) {
+	dir := repoDir(t)
+
+	repo := gitshallow.New(testRepoURL, dir, 1, "")
+	// Ensure cloned.
+	if _, err := repo.Fetch(t.Context()); err != nil {
+		t.Fatalf("initial Fetch: %v", err)
+	}
+
+	// Second pull on the same instance — already at HEAD, should not advance.
+	updated, err := repo.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("second Fetch: %v", err)
+	}
+	if updated {
+		t.Error("same-instance second pull: expected updated=false (already at HEAD)")
+	}
+	t.Log("same-instance pull correctly reported no update")
+}
+
+func TestRepo_Pull_FreshInstance(t *testing.T) {
+	dir := repoDir(t)
+
+	// Ensure cloned via a first instance.
+	first := gitshallow.New(testRepoURL, dir, 1, "")
+	if _, err := first.Fetch(t.Context()); err != nil {
+		t.Fatalf("initial Fetch: %v", err)
+	}
+
+	// Fresh instance with no in-memory state — git HEAD on disk drives the check.
+	fresh := gitshallow.New(testRepoURL, dir, 1, "")
+	updated, err := fresh.Fetch(t.Context())
+	if err != nil {
+		t.Fatalf("fresh-instance Fetch: %v", err)
+	}
+	if updated {
+		t.Error("fresh-instance pull: expected updated=false (HEAD unchanged on disk)")
+	}
+	t.Log("fresh-instance pull correctly reported no update")
+}

net/gitshallow/go.mod

@@ -0,0 +1,5 @@
+module github.com/therootcompany/golib/net/gitshallow
+
+go 1.26.0
+
+require golang.org/x/sync v0.20.0

net/gitshallow/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=