From 0659004ddfb6c5acfb02249896a4c16bf5512c7a Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 30 Apr 2026 16:48:39 -0600 Subject: [PATCH] feat(net/gitshallow): incremental shallow git-repo mirroring New package for keeping a local mirror of a remote git repo via shallow fetch+reset (not pull --ff-only, which loses force-pushed history). Adds MaxAge gate via FETCH_HEAD mtime to skip redundant network calls under concurrent callers, and a git-shallow-sync CLI. Includes integration tests plus a force-push regression test. --- net/gitshallow/cmd/git-shallow-sync/main.go | 92 ++++++ net/gitshallow/forcepush_test.go | 102 ++++++ net/gitshallow/gitshallow.go | 291 ++++++++++++++++++ net/gitshallow/gitshallow_integration_test.go | 101 ++++++ net/gitshallow/go.mod | 3 + 5 files changed, 589 insertions(+) create mode 100644 net/gitshallow/cmd/git-shallow-sync/main.go create mode 100644 net/gitshallow/forcepush_test.go create mode 100644 net/gitshallow/gitshallow.go create mode 100644 net/gitshallow/gitshallow_integration_test.go create mode 100644 net/gitshallow/go.mod diff --git a/net/gitshallow/cmd/git-shallow-sync/main.go b/net/gitshallow/cmd/git-shallow-sync/main.go new file mode 100644 index 0000000..e1b0c81 --- /dev/null +++ b/net/gitshallow/cmd/git-shallow-sync/main.go @@ -0,0 +1,92 @@ +// git-shallow-sync syncs a shallow git clone at the given local path, +// cloning on first run and fetching + hard-resetting on subsequent runs. +// +// Usage: +// +// git-shallow-sync +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/therootcompany/golib/net/gitshallow" +) + +const version = "dev" + +type Config struct { + Depth int + Branch string +} + +func main() { + cfg := Config{} + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.IntVar(&cfg.Depth, "depth", 1, "clone/fetch depth (-1 for full history)") + fs.StringVar(&cfg.Branch, "branch", "", "branch to track (empty: remote default)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) + fs.PrintDefaults() + } + + if len(os.Args) > 1 { + switch os.Args[1] { + case "-V", "-version", "--version", "version": + fmt.Fprintf(os.Stdout, "git-shallow-sync %s\n", version) + os.Exit(0) + case "help", "-help", "--help": + fmt.Fprintf(os.Stdout, "git-shallow-sync %s\n\n", version) + fs.SetOutput(os.Stdout) + fs.Usage() + os.Exit(0) + } + } + if err := fs.Parse(os.Args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + os.Exit(1) + } + + args := fs.Args() + if len(args) != 2 { + fs.Usage() + os.Exit(1) + } + url := args[0] + path := args[1] + + if len(path) > 0 && path[0] == '~' { + home, err := os.UserHomeDir() + if err != nil { + fmt.Fprintf(os.Stderr, "error: resolve home: %v\n", err) + os.Exit(1) + } + path = filepath.Join(home, path[1:]) + } + absPath, err := filepath.Abs(path) + if err != nil { + fmt.Fprintf(os.Stderr, "error: invalid path: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "Syncing %s -> %s... ", url, absPath) + t := time.Now() + repo := gitshallow.New(url, absPath, cfg.Depth, cfg.Branch) + updated, err := repo.Sync() + if err != nil { + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "error: sync: %v\n", err) + os.Exit(1) + } + state := "already up to date" + if updated { + state = "updated" + } + fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state) +} diff --git a/net/gitshallow/forcepush_test.go b/net/gitshallow/forcepush_test.go new file mode 100644 index 0000000..d41938b --- /dev/null +++ b/net/gitshallow/forcepush_test.go @@ -0,0 +1,102 @@ +package gitshallow_test + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/gitshallow" +) + +// TestRepo_ForcePushRecovery verifies that Fetch transparently picks up an +// upstream history rewrite. The previous pull-based flow could fail with +// "refusing to merge unrelated histories"; the current fetch+reset must +// succeed and install the new HEAD. +func TestRepo_ForcePushRecovery(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root := t.TempDir() + upstream := filepath.Join(root, "upstream.git") // bare + scratch := filepath.Join(root, "scratch") // working copy that rewrites history + clone := filepath.Join(root, "clone") // gitshallow clone under test + + mustGit := func(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v (in %s): %v\n%s", args, dir, err, out) + } + } + + // Bare upstream. + if err := os.MkdirAll(upstream, 0o755); err != nil { + t.Fatal(err) + } + mustGit(t, upstream, "init", "--bare", "--initial-branch=main") + + // Scratch working copy with one commit, push to upstream. + if err := os.MkdirAll(scratch, 0o755); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "init", "--initial-branch=main") + mustGit(t, scratch, "remote", "add", "origin", upstream) + if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v1\n"), 0o644); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "add", "data.txt") + mustGit(t, scratch, "commit", "-m", "v1") + mustGit(t, scratch, "push", "-u", "origin", "main") + + // gitshallow clones, reads v1. + repo := gitshallow.New(upstream, clone, 1, "main") + if updated, err := repo.Fetch(); err != nil { + t.Fatalf("first Fetch: %v", err) + } else if !updated { + t.Fatal("first Fetch: expected updated=true") + } + got, err := os.ReadFile(filepath.Join(clone, "data.txt")) + if err != nil { + t.Fatalf("read v1: %v", err) + } + if string(got) != "v1\n" { + t.Fatalf("v1: got %q want %q", got, "v1\n") + } + + // Rewrite upstream history with an unrelated commit and force-push. + mustGit(t, scratch, "checkout", "--orphan", "fresh") + mustGit(t, scratch, "rm", "-rf", ".") + if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v2\n"), 0o644); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "add", "data.txt") + mustGit(t, scratch, "commit", "-m", "v2") + mustGit(t, scratch, "branch", "-M", "main") + mustGit(t, scratch, "push", "--force", "origin", "main") + + // gitshallow must recover — fetch+reset, not pull --ff-only. + // Use a fresh instance: the same *Repo has a 1s debounce that would + // skip the follow-up fetch in this test's wall-clock window. + repo2 := gitshallow.New(upstream, clone, 1, "main") + updated, err := repo2.Fetch() + if err != nil { + t.Fatalf("post-force-push Fetch: %v", err) + } + if !updated { + t.Error("post-force-push Fetch: expected updated=true") + } + got, err = os.ReadFile(filepath.Join(clone, "data.txt")) + if err != nil { + t.Fatalf("read v2: %v", err) + } + if string(got) != "v2\n" { + t.Fatalf("v2: got %q want %q", got, "v2\n") + } +} diff --git a/net/gitshallow/gitshallow.go b/net/gitshallow/gitshallow.go new file mode 100644 index 0000000..54b33ad --- /dev/null +++ b/net/gitshallow/gitshallow.go @@ -0,0 +1,291 @@ +package gitshallow + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// Repo manages a shallow git clone used as a periodically-updated data source. +type Repo struct { + URL string + Path string + Depth int // 0 defaults to 1, -1 for all + Branch string // Optional: specific branch to clone/pull + + // MaxAge skips the git fetch when .git/FETCH_HEAD is younger than this + // duration. Persists across process restarts (unlike the in-memory + // lastSynced debounce) — so repeated short-lived CLI invocations don't + // hammer the remote. 0 disables. + MaxAge time.Duration + + // GCInterval controls explicit aggressive GC after pulls. + // 0 (default) — no explicit gc; git runs gc.auto on its own schedule + // 1 — aggressive gc after every pull + // N — aggressive gc after every Nth pull + GCInterval int + + mu sync.Mutex + pullCount int + lastSynced time.Time +} + +// New creates a new Repo instance. +func New(url, path string, depth int, branch string) *Repo { + return &Repo{ + URL: url, + Path: path, + Depth: depth, + Branch: strings.TrimSpace(branch), + } +} + +// effectiveDepth returns the depth to use for clone/pull. +// 0 means unset — defaults to 1. -1 means full history. +func (r *Repo) effectiveDepth() int { + if r.Depth == 0 { + return 1 + } + return r.Depth +} + +// Init clones the repo if missing, then syncs once. +// Returns whether anything new was fetched. +func (r *Repo) Init() (bool, error) { + gitDir := filepath.Join(r.Path, ".git") + if _, err := os.Stat(gitDir); err != nil { + if _, err := r.Clone(); err != nil { + return false, err + } + } + return r.syncGit() +} + +// Clone performs a shallow clone (--depth N --single-branch --no-tags). +func (r *Repo) Clone() (bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.clone() +} + +func (r *Repo) clone() (bool, error) { + if r.exists() { + return false, nil + } + if r.URL == "" { + return false, fmt.Errorf("repository URL is required") + } + if r.Path == "" { + return false, fmt.Errorf("local path is required") + } + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return false, err + } + + args := []string{"clone", "--no-tags"} + if depth := r.effectiveDepth(); depth >= 0 { + args = append(args, "--depth", fmt.Sprintf("%d", depth)) + } + args = append(args, "--single-branch") + if r.Branch != "" { + args = append(args, "--branch", r.Branch) + } + args = append(args, r.URL, filepath.Base(r.Path)) + + _, err := r.runGit(args...) + return true, err +} + +// exists checks if the directory contains a .git folder. +func (r *Repo) exists() bool { + _, err := os.Stat(filepath.Join(r.Path, ".git")) + return err == nil +} + +// runGit executes a git command in the repo directory (or parent for clone). +func (r *Repo) runGit(args ...string) (string, error) { + cmd := exec.Command("git", args...) + if _, err := os.Stat(r.Path); err == nil && r.exists() { + cmd.Dir = r.Path + } else { + cmd.Dir = filepath.Dir(r.Path) + } + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("git %s failed: %v\n%s", strings.Join(args, " "), err, output) + } + return strings.TrimSpace(string(output)), nil +} + +// Pull fetches from origin and hard-resets the working tree to the remote +// branch, reporting whether HEAD changed. This is a read-only mirror — we +// never try to merge, so force-pushes upstream are handled transparently. +func (r *Repo) Pull() (updated bool, err error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.pull() +} + +func (r *Repo) pull() (updated bool, err error) { + if !r.exists() { + return false, fmt.Errorf("repository does not exist at %s", r.Path) + } + + oldHead, _ := r.runGit("rev-parse", "HEAD") + + branch := r.Branch + if branch == "" { + out, err := r.runGit("symbolic-ref", "--short", "refs/remotes/origin/HEAD") + if err != nil { + return false, err + } + _, branch, _ = strings.Cut(out, "/") + } + + fetchArgs := []string{"fetch", "--no-tags"} + if depth := r.effectiveDepth(); depth >= 0 { + fetchArgs = append(fetchArgs, "--depth", fmt.Sprintf("%d", depth)) + } + fetchArgs = append(fetchArgs, "origin", branch) + if _, err := r.runGit(fetchArgs...); err != nil { + return false, err + } + + if _, err := r.runGit("reset", "--hard", "origin/"+branch); err != nil { + return false, err + } + + newHead, err := r.runGit("rev-parse", "HEAD") + if err != nil { + return false, err + } + return oldHead != newHead, nil +} + +// GC runs git gc --aggressive --prune=now. +func (r *Repo) GC() error { + r.mu.Lock() + defer r.mu.Unlock() + return r.gc() +} + +func (r *Repo) gc() error { + if !r.exists() { + return fmt.Errorf("repository does not exist at %s", r.Path) + } + _, err := r.runGit("gc", "--aggressive", "--prune=now") + return err +} + +// Sync clones if missing, pulls, and conditionally runs GC based on GCEvery. +// Returns whether HEAD changed. +func (r *Repo) Sync() (bool, error) { + return r.syncGit() +} + +// Fetch syncs the repo and reports whether HEAD changed. +func (r *Repo) Fetch() (bool, error) { + return r.syncGit() +} + +// FilePath returns the absolute path to relPath within this repo. +func (r *Repo) FilePath(rel string) string { + return filepath.Join(r.Path, rel) +} + +// File returns a handle to relPath within this repo. +// The handle's Path and Open methods give access to the file; its Fetch method +// syncs the repo and reports whether this specific file changed (by mtime). +func (r *Repo) File(relPath string) *File { + return &File{repo: r, rel: relPath} +} + +// File is a handle to a single file inside a Repo. +// Fetch syncs the repo (deduped across all File handles sharing the same +// Repo) and reports whether this file changed. +type File struct { + repo *Repo + rel string + mu sync.Mutex + lastMod time.Time +} + +// Path returns the absolute path to the file. +func (f *File) Path() string { + return filepath.Join(f.repo.Path, f.rel) +} + +// Open returns an open *os.File for reading. The caller must Close it. +func (f *File) Open() (*os.File, error) { + return os.Open(f.Path()) +} + +// Fetch syncs the repo and reports whether this file changed since last call. +// Safe to call concurrently. +func (f *File) Fetch() (bool, error) { + if _, err := f.repo.syncGit(); err != nil { + return false, err + } + info, err := os.Stat(f.Path()) + if err != nil { + return false, err + } + f.mu.Lock() + defer f.mu.Unlock() + if info.ModTime().Equal(f.lastMod) { + return false, nil + } + f.lastMod = info.ModTime() + return true, nil +} + +func (r *Repo) syncGit() (updated bool, err error) { + // MaxAge: file-mtime gate (FETCH_HEAD is rewritten on every successful + // fetch, so its mtime is "last time we talked to the remote"). Checked + // outside the lock — just a stat. + if r.MaxAge > 0 { + if info, err := os.Stat(filepath.Join(r.Path, ".git", "FETCH_HEAD")); err == nil { + if time.Since(info.ModTime()) < r.MaxAge { + return false, nil + } + } + } + + r.mu.Lock() + defer r.mu.Unlock() + + // If another caller just finished a sync while we were waiting for the + // lock, skip the pull — the repo is already current. + if !r.lastSynced.IsZero() && time.Since(r.lastSynced) < time.Second { + return false, nil + } + + if cloned, err := r.clone(); err != nil { + return false, err + } else if cloned { + r.lastSynced = time.Now() + return true, nil + } + + updated, err = r.pull() + if err != nil { + return false, err + } + r.lastSynced = time.Now() + if !updated { + return false, nil + } + + if r.GCInterval > 0 { + r.pullCount++ + if r.pullCount%r.GCInterval == 0 { + return true, r.gc() + } + } + + return true, nil +} diff --git a/net/gitshallow/gitshallow_integration_test.go b/net/gitshallow/gitshallow_integration_test.go new file mode 100644 index 0000000..a7717db --- /dev/null +++ b/net/gitshallow/gitshallow_integration_test.go @@ -0,0 +1,101 @@ +//go:build integration + +package gitshallow_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/gitshallow" +) + +const testRepoURL = "https://github.com/bitwire-it/ipblocklist" + +func testdataDir(t *testing.T) string { + t.Helper() + dir, _ := filepath.Abs(".") + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata") + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find module root (go.mod)") + } + dir = parent + } +} + +func repoDir(t *testing.T) string { + return filepath.Join(testdataDir(t), "gitshallow_ipblocklist") +} + +func TestRepo_Clone(t *testing.T) { + dir := repoDir(t) + os.RemoveAll(dir) + + repo := gitshallow.New(testRepoURL, dir, 1, "") + updated, err := repo.Fetch() + if err != nil { + t.Fatalf("Fetch (clone): %v", err) + } + if !updated { + t.Error("fresh clone: expected updated=true") + } + + for _, rel := range []string{ + "tables/inbound/single_ips.txt", + "tables/inbound/networks.txt", + "tables/outbound/single_ips.txt", + "tables/outbound/networks.txt", + } { + info, err := os.Stat(filepath.Join(dir, rel)) + if err != nil { + t.Errorf("expected file missing: %s", rel) + } else { + t.Logf("%s: %d bytes", rel, info.Size()) + } + } +} + +func TestRepo_Pull_SameInstance(t *testing.T) { + dir := repoDir(t) + + repo := gitshallow.New(testRepoURL, dir, 1, "") + // Ensure cloned. + if _, err := repo.Fetch(); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + + // Second pull on the same instance — already at HEAD, should not advance. + updated, err := repo.Fetch() + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if updated { + t.Error("same-instance second pull: expected updated=false (already at HEAD)") + } + t.Log("same-instance pull correctly reported no update") +} + +func TestRepo_Pull_FreshInstance(t *testing.T) { + dir := repoDir(t) + + // Ensure cloned via a first instance. + first := gitshallow.New(testRepoURL, dir, 1, "") + if _, err := first.Fetch(); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + + // Fresh instance with no in-memory state — git HEAD on disk drives the check. + fresh := gitshallow.New(testRepoURL, dir, 1, "") + updated, err := fresh.Fetch() + if err != nil { + t.Fatalf("fresh-instance Fetch: %v", err) + } + if updated { + t.Error("fresh-instance pull: expected updated=false (HEAD unchanged on disk)") + } + t.Log("fresh-instance pull correctly reported no update") +} diff --git a/net/gitshallow/go.mod b/net/gitshallow/go.mod new file mode 100644 index 0000000..01121e1 --- /dev/null +++ b/net/gitshallow/go.mod @@ -0,0 +1,3 @@ +module github.com/therootcompany/golib/net/gitshallow + +go 1.26.0