diff --git a/.gitignore b/.gitignore index 345a707..7f6d837 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ credentials.tsv +testdata/ .env *.env diff --git a/net/gitshallow/cmd/git-shallow-sync/main.go b/net/gitshallow/cmd/git-shallow-sync/main.go new file mode 100644 index 0000000..938a2a8 --- /dev/null +++ b/net/gitshallow/cmd/git-shallow-sync/main.go @@ -0,0 +1,115 @@ +// git-shallow-sync syncs a shallow git clone at the given local path, +// cloning on first run and fetching + hard-resetting on subsequent runs. +// +// Usage: +// +// git-shallow-sync +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/therootcompany/golib/net/gitshallow" +) + +// Replaced by goreleaser / ldflags at build time. +var ( + name = "git-shallow-sync" + version = "0.0.0-dev" + commit = "0000000" + date = "0001-01-01" + licenseYear = "2021-present" + licenseOwner = "AJ ONeal " + licenseType = "MPL-2.0" +) + +type Config struct { + Depth int + Branch string +} + +func printVersion(w io.Writer) { + _, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, version, commit[:7], date) + _, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner) + _, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType) +} + +func main() { + cfg := Config{} + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.IntVar(&cfg.Depth, "depth", 1, "clone/fetch depth (-1 for full history)") + fs.StringVar(&cfg.Branch, "branch", "", "branch to track (empty: remote default)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", os.Args[0]) + fs.PrintDefaults() + } + + if len(os.Args) > 1 { + switch os.Args[1] { + case "-V", "-version", "--version", "version": + printVersion(os.Stdout) + os.Exit(0) + case "help", "-help", "--help": + printVersion(os.Stdout) + fmt.Fprintln(os.Stdout, "") + fs.SetOutput(os.Stdout) + fs.Usage() + os.Exit(0) + } + } + if err := fs.Parse(os.Args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + os.Exit(1) + } + + args := fs.Args() + if len(args) != 2 { + fs.Usage() + os.Exit(1) + } + url := args[0] + path := args[1] + + if len(path) > 0 && path[0] == '~' { + home, err := os.UserHomeDir() + if err != nil { + fmt.Fprintf(os.Stderr, "error: resolve home: %v\n", err) + os.Exit(1) + } + path = filepath.Join(home, path[1:]) + } + absPath, err := filepath.Abs(path) + if err != nil { + fmt.Fprintf(os.Stderr, "error: invalid path: %v\n", err) + os.Exit(1) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + fmt.Fprintf(os.Stderr, "Syncing %s -> %s... ", url, absPath) + t := time.Now() + repo := gitshallow.New(url, absPath, cfg.Depth, cfg.Branch) + updated, err := repo.Fetch(ctx) + if err != nil { + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "error: sync: %v\n", err) + os.Exit(1) + } + state := "already up to date" + if updated { + state = "updated" + } + fmt.Fprintf(os.Stderr, "%s (%s)\n", time.Since(t).Round(time.Millisecond), state) +} diff --git a/net/gitshallow/forcepush_test.go b/net/gitshallow/forcepush_test.go new file mode 100644 index 0000000..a389d21 --- /dev/null +++ b/net/gitshallow/forcepush_test.go @@ -0,0 +1,102 @@ +package gitshallow_test + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/gitshallow" +) + +// TestRepo_ForcePushRecovery verifies that Fetch transparently picks up an +// upstream history rewrite. The previous pull-based flow could fail with +// "refusing to merge unrelated histories"; the current fetch+reset must +// succeed and install the new HEAD. +func TestRepo_ForcePushRecovery(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root := t.TempDir() + upstream := filepath.Join(root, "upstream.git") // bare + scratch := filepath.Join(root, "scratch") // working copy that rewrites history + clone := filepath.Join(root, "clone") // gitshallow clone under test + + mustGit := func(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v (in %s): %v\n%s", args, dir, err, out) + } + } + + // Bare upstream. + if err := os.MkdirAll(upstream, 0o755); err != nil { + t.Fatal(err) + } + mustGit(t, upstream, "init", "--bare", "--initial-branch=main") + + // Scratch working copy with one commit, push to upstream. + if err := os.MkdirAll(scratch, 0o755); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "init", "--initial-branch=main") + mustGit(t, scratch, "remote", "add", "origin", upstream) + if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v1\n"), 0o644); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "add", "data.txt") + mustGit(t, scratch, "commit", "-m", "v1") + mustGit(t, scratch, "push", "-u", "origin", "main") + + // gitshallow clones, reads v1. + repo := gitshallow.New(upstream, clone, 1, "main") + if updated, err := repo.Fetch(t.Context()); err != nil { + t.Fatalf("first Fetch: %v", err) + } else if !updated { + t.Fatal("first Fetch: expected updated=true") + } + got, err := os.ReadFile(filepath.Join(clone, "data.txt")) + if err != nil { + t.Fatalf("read v1: %v", err) + } + if string(got) != "v1\n" { + t.Fatalf("v1: got %q want %q", got, "v1\n") + } + + // Rewrite upstream history with an unrelated commit and force-push. + mustGit(t, scratch, "checkout", "--orphan", "fresh") + mustGit(t, scratch, "rm", "-rf", ".") + if err := os.WriteFile(filepath.Join(scratch, "data.txt"), []byte("v2\n"), 0o644); err != nil { + t.Fatal(err) + } + mustGit(t, scratch, "add", "data.txt") + mustGit(t, scratch, "commit", "-m", "v2") + mustGit(t, scratch, "branch", "-M", "main") + mustGit(t, scratch, "push", "--force", "origin", "main") + + // gitshallow must recover — fetch+reset, not pull --ff-only. + // Use a fresh instance: the same *Repo has a 1s debounce that would + // skip the follow-up fetch in this test's wall-clock window. + repo2 := gitshallow.New(upstream, clone, 1, "main") + updated, err := repo2.Fetch(t.Context()) + if err != nil { + t.Fatalf("post-force-push Fetch: %v", err) + } + if !updated { + t.Error("post-force-push Fetch: expected updated=true") + } + got, err = os.ReadFile(filepath.Join(clone, "data.txt")) + if err != nil { + t.Fatalf("read v2: %v", err) + } + if string(got) != "v2\n" { + t.Fatalf("v2: got %q want %q", got, "v2\n") + } +} diff --git a/net/gitshallow/gitshallow.go b/net/gitshallow/gitshallow.go new file mode 100644 index 0000000..29fed9c --- /dev/null +++ b/net/gitshallow/gitshallow.go @@ -0,0 +1,313 @@ +package gitshallow + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// Repo manages a shallow git clone used as a periodically-updated data source. +type Repo struct { + URL string + Path string + Depth int // 0 defaults to 1, -1 for all + Branch string // Optional: specific branch to clone/pull + + // MaxAge skips the git fetch when .git/FETCH_HEAD is younger than this + // duration. Persists across process restarts (unlike the in-process + // singleflight dedup) — so repeated short-lived CLI invocations don't + // hammer the remote. 0 defaults to 1 minute; -1 disables. + MaxAge time.Duration + + // GCInterval controls explicit aggressive GC after pulls. + // 0 (default) — no explicit gc; git runs gc.auto on its own schedule + // 1 — aggressive gc after every pull + // N — aggressive gc after every Nth pull + GCInterval int + + sf singleflight.Group + mu sync.Mutex // guards pullCount + pullCount int +} + +// New creates a new Repo instance. +func New(url, path string, depth int, branch string) *Repo { + return &Repo{ + URL: url, + Path: path, + Depth: depth, + Branch: strings.TrimSpace(branch), + } +} + +// validRef matches branch/ref names that are safe to pass to git as a +// positional argument: no leading dash (which git would treat as a flag), +// no whitespace, no shell metacharacters. +var validRef = regexp.MustCompile(`^[A-Za-z0-9._/-]+$`) + +// validateArgs rejects branch values git would interpret as options. +// URL is passed to git after a `--` flag terminator, so git won't +// reinterpret a leading `-` as an option. +func (r *Repo) validateArgs() error { + if _, err := url.Parse(r.URL); err != nil { + return fmt.Errorf("invalid URL %q: %w", r.URL, err) + } + if r.Branch != "" && !validRef.MatchString(r.Branch) { + return fmt.Errorf("branch %q contains invalid characters", r.Branch) + } + return nil +} + +// effectiveDepth returns the depth to use for clone/pull. +// 0 means unset — defaults to 1. -1 means full history. +func (r *Repo) effectiveDepth() int { + if r.Depth == 0 { + return 1 + } + return r.Depth +} + +// effectiveMaxAge returns the MaxAge to apply. +// 0 means unset — defaults to 1 minute. -1 disables the gate. +func (r *Repo) effectiveMaxAge() time.Duration { + switch { + case r.MaxAge == 0: + return time.Minute + case r.MaxAge < 0: + return 0 + default: + return r.MaxAge + } +} + +func (r *Repo) clone(ctx context.Context) (bool, error) { + if r.exists() { + return false, nil + } + if r.URL == "" { + return false, fmt.Errorf("repository URL is required") + } + if r.Path == "" { + return false, fmt.Errorf("local path is required") + } + if err := r.validateArgs(); err != nil { + return false, err + } + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return false, err + } + + args := []string{"clone", "--no-tags"} + if depth := r.effectiveDepth(); depth >= 0 { + args = append(args, "--depth", fmt.Sprintf("%d", depth)) + } + args = append(args, "--single-branch") + if r.Branch != "" { + args = append(args, "--branch", r.Branch) + } + // `--` separates flags from positional URL/path so a URL beginning + // with `-` cannot be reinterpreted by git as an option. + args = append(args, "--", r.URL, filepath.Base(r.Path)) + + _, err := r.runGit(ctx, args...) + return true, err +} + +// exists checks if the directory contains a .git folder. +func (r *Repo) exists() bool { + _, err := os.Stat(filepath.Join(r.Path, ".git")) + return err == nil +} + +// runGit executes a git command in the repo directory (or parent for clone). +// ctx cancels the child git process via SIGKILL. +func (r *Repo) runGit(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "git", args...) + if _, err := os.Stat(r.Path); err == nil && r.exists() { + cmd.Dir = r.Path + } else { + cmd.Dir = filepath.Dir(r.Path) + } + output, err := cmd.CombinedOutput() + if err != nil { + // Redact userinfo from both the args echo (we may have just passed + // r.URL) and from git's own output (git embeds the URL in messages + // like "fatal: unable to access 'https://user:pass@host/'"). If a + // caller put credentials in the URL — even though the docs steer + // them to Authorization headers — they don't end up in error logs. + safeArgs := redactUserinfo(strings.Join(args, " "), r.URL) + safeOut := redactUserinfo(string(output), r.URL) + return "", fmt.Errorf("git %s failed: %w\n%s", safeArgs, err, safeOut) + } + return strings.TrimSpace(string(output)), nil +} + +// redactUserinfo returns s with any "user:pass@" or "user@" segment from +// rawURL replaced with "redacted@". No-op if rawURL has no userinfo. +func redactUserinfo(s, rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil || u.User == nil { + return s + } + return strings.ReplaceAll(s, u.User.String()+"@", "redacted@") +} + +func (r *Repo) pull(ctx context.Context) (updated bool, err error) { + if !r.exists() { + return false, fmt.Errorf("repository does not exist at %s", r.Path) + } + if err := r.validateArgs(); err != nil { + return false, err + } + + oldHead, _ := r.runGit(ctx, "rev-parse", "HEAD") + + branch := r.Branch + if branch == "" { + out, err := r.runGit(ctx, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") + if err != nil { + return false, err + } + _, branch, _ = strings.Cut(out, "/") + if !validRef.MatchString(branch) { + return false, fmt.Errorf("remote default branch %q contains invalid characters", branch) + } + } + + fetchArgs := []string{"fetch", "--no-tags"} + if depth := r.effectiveDepth(); depth >= 0 { + fetchArgs = append(fetchArgs, "--depth", fmt.Sprintf("%d", depth)) + } + // `--` separates flags from positional remote/branch. + fetchArgs = append(fetchArgs, "--", "origin", branch) + if _, err := r.runGit(ctx, fetchArgs...); err != nil { + return false, err + } + + if _, err := r.runGit(ctx, "reset", "--hard", "origin/"+branch); err != nil { + return false, err + } + + newHead, err := r.runGit(ctx, "rev-parse", "HEAD") + if err != nil { + return false, err + } + return oldHead != newHead, nil +} + +func (r *Repo) gc(ctx context.Context) error { + if !r.exists() { + return fmt.Errorf("repository does not exist at %s", r.Path) + } + _, err := r.runGit(ctx, "gc", "--aggressive", "--prune=now") + return err +} + +// Fetch clones the repo if missing, pulls otherwise, and conditionally runs +// GC based on GCInterval. Returns whether HEAD changed. Implements Fetcher. +// Safe to call concurrently — concurrent callers share a single in-flight +// fetch (via singleflight) and all receive the same result. To force a +// pull regardless of MaxAge, set MaxAge=-1 before calling. +func (r *Repo) Fetch(ctx context.Context) (updated bool, err error) { + // MaxAge: file-mtime gate (FETCH_HEAD is rewritten on every successful + // fetch, so its mtime is "last time we talked to the remote"). + if maxAge := r.effectiveMaxAge(); maxAge > 0 { + if info, err := os.Stat(filepath.Join(r.Path, ".git", "FETCH_HEAD")); err == nil { + if time.Since(info.ModTime()) < maxAge { + return false, nil + } + } + } + + v, err, _ := r.sf.Do("fetch", func() (any, error) { + return r.fetch(ctx) + }) + if err != nil { + return false, err + } + return v.(bool), nil +} + +func (r *Repo) fetch(ctx context.Context) (bool, error) { + if cloned, err := r.clone(ctx); err != nil { + return false, err + } else if cloned { + return true, nil + } + + updated, err := r.pull(ctx) + if err != nil { + return false, err + } + if !updated { + return false, nil + } + + if r.GCInterval > 0 { + r.mu.Lock() + r.pullCount++ + shouldGC := r.pullCount%r.GCInterval == 0 + r.mu.Unlock() + if shouldGC { + return true, r.gc(ctx) + } + } + + return true, nil +} + +// FilePath returns the absolute path to relPath within this repo. +func (r *Repo) FilePath(rel string) string { + return filepath.Join(r.Path, rel) +} + +// File returns a handle to relPath within this repo. The handle's Path and +// Open methods give access to the file; its Fetch method syncs the repo and +// reports whether this specific file changed (by mtime). +func (r *Repo) File(relPath string) *RepoFile { + return &RepoFile{repo: r, rel: relPath} +} + +// RepoFile is a handle to a single file inside a Repo. Fetch syncs the repo +// (deduped across all RepoFile handles sharing the same Repo) and reports +// whether this file changed. The Repo prefix avoids collision with os.File, +// which is the type returned by Open. +type RepoFile struct { + repo *Repo + rel string + mu sync.Mutex + lastMod time.Time +} + +// Path returns the absolute path to the file. +func (f *RepoFile) Path() string { + return filepath.Join(f.repo.Path, f.rel) +} + +// Fetch syncs the repo and reports whether this file changed since last call. +// Safe to call concurrently. Implements Fetcher. +func (f *RepoFile) Fetch(ctx context.Context) (bool, error) { + if _, err := f.repo.Fetch(ctx); err != nil { + return false, err + } + info, err := os.Stat(f.Path()) + if err != nil { + return false, err + } + f.mu.Lock() + defer f.mu.Unlock() + if info.ModTime().Equal(f.lastMod) { + return false, nil + } + f.lastMod = info.ModTime() + return true, nil +} diff --git a/net/gitshallow/gitshallow_integration_test.go b/net/gitshallow/gitshallow_integration_test.go new file mode 100644 index 0000000..0364db2 --- /dev/null +++ b/net/gitshallow/gitshallow_integration_test.go @@ -0,0 +1,101 @@ +//go:build integration + +package gitshallow_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/therootcompany/golib/net/gitshallow" +) + +const testRepoURL = "https://github.com/bitwire-it/ipblocklist" + +func testdataDir(t *testing.T) string { + t.Helper() + dir, _ := filepath.Abs(".") + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata") + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find module root (go.mod)") + } + dir = parent + } +} + +func repoDir(t *testing.T) string { + return filepath.Join(testdataDir(t), "gitshallow_ipblocklist") +} + +func TestRepo_Clone(t *testing.T) { + dir := repoDir(t) + os.RemoveAll(dir) + + repo := gitshallow.New(testRepoURL, dir, 1, "") + updated, err := repo.Fetch(t.Context()) + if err != nil { + t.Fatalf("Fetch (clone): %v", err) + } + if !updated { + t.Error("fresh clone: expected updated=true") + } + + for _, rel := range []string{ + "tables/inbound/single_ips.txt", + "tables/inbound/networks.txt", + "tables/outbound/single_ips.txt", + "tables/outbound/networks.txt", + } { + info, err := os.Stat(filepath.Join(dir, rel)) + if err != nil { + t.Errorf("expected file missing: %s", rel) + } else { + t.Logf("%s: %d bytes", rel, info.Size()) + } + } +} + +func TestRepo_Pull_SameInstance(t *testing.T) { + dir := repoDir(t) + + repo := gitshallow.New(testRepoURL, dir, 1, "") + // Ensure cloned. + if _, err := repo.Fetch(t.Context()); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + + // Second pull on the same instance — already at HEAD, should not advance. + updated, err := repo.Fetch(t.Context()) + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if updated { + t.Error("same-instance second pull: expected updated=false (already at HEAD)") + } + t.Log("same-instance pull correctly reported no update") +} + +func TestRepo_Pull_FreshInstance(t *testing.T) { + dir := repoDir(t) + + // Ensure cloned via a first instance. + first := gitshallow.New(testRepoURL, dir, 1, "") + if _, err := first.Fetch(t.Context()); err != nil { + t.Fatalf("initial Fetch: %v", err) + } + + // Fresh instance with no in-memory state — git HEAD on disk drives the check. + fresh := gitshallow.New(testRepoURL, dir, 1, "") + updated, err := fresh.Fetch(t.Context()) + if err != nil { + t.Fatalf("fresh-instance Fetch: %v", err) + } + if updated { + t.Error("fresh-instance pull: expected updated=false (HEAD unchanged on disk)") + } + t.Log("fresh-instance pull correctly reported no update") +} diff --git a/net/gitshallow/go.mod b/net/gitshallow/go.mod new file mode 100644 index 0000000..27508db --- /dev/null +++ b/net/gitshallow/go.mod @@ -0,0 +1,5 @@ +module github.com/therootcompany/golib/net/gitshallow + +go 1.26.0 + +require golang.org/x/sync v0.20.0 diff --git a/net/gitshallow/go.sum b/net/gitshallow/go.sum new file mode 100644 index 0000000..733d716 --- /dev/null +++ b/net/gitshallow/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=