Add input parameter filter for git commands

This adds a new input parameter filter to assist in working with large
git projects. The param filter sets the git filter-spec argument when
fetching repositories. Only one argument can be provided, the combined
form can be used to set multiple filters. When using filter combined
with a subdir the checkout will now also be sparse to further reduce the
number of workspace files.

CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## [Unreleased]
 
-- No changes yet.
+- Add input parameter `fitler` for use with git inputs. This sets the filter
+  argument for the git fetch command.
 
 ## [v1.49.0] - 2025-01-07
 

private/buf/buffetch/internal/git_ref.go

@@ -42,6 +42,7 @@ type gitRef struct {
 	depth             uint32
 	recurseSubmodules bool
 	subDirPath        string
+	filter            string
 }
 
 func newGitRef(
@@ -51,6 +52,7 @@ func newGitRef(
 	depth uint32,
 	recurseSubmodules bool,
 	subDirPath string,
+	filter string,
 ) (*gitRef, error) {
 	gitScheme, path, err := getGitSchemeAndPath(format, path)
 	if err != nil {
@@ -74,6 +76,7 @@ func newGitRef(
 		recurseSubmodules,
 		depth,
 		subDirPath,
+		filter,
 	), nil
 }
 
@@ -85,6 +88,7 @@ func newDirectGitRef(
 	recurseSubmodules bool,
 	depth uint32,
 	subDirPath string,
+	filter string,
 ) *gitRef {
 	return &gitRef{
 		format:            format,
@@ -94,6 +98,7 @@ func newDirectGitRef(
 		depth:             depth,
 		recurseSubmodules: recurseSubmodules,
 		subDirPath:        subDirPath,
+		filter:            filter,
 	}
 }
 
@@ -125,6 +130,10 @@ func (r *gitRef) SubDirPath() string {
 	return r.subDirPath
 }
 
+func (r *gitRef) Filter() string {
+	return r.filter
+}
+
 func (*gitRef) ref()       {}
 func (*gitRef) bucketRef() {}
 func (*gitRef) gitRef()    {}

private/buf/buffetch/internal/internal.go

@@ -207,6 +207,8 @@ type GitRef interface {
 	RecurseSubmodules() bool
 	// Will be empty instead of "." for root directory
 	SubDirPath() string
+	// Filter spec to use, see the --filter option in git rev-list.
+	Filter() string
 	gitRef()
 }
 
@@ -217,8 +219,9 @@ func NewGitRef(
 	depth uint32,
 	recurseSubmodules bool,
 	subDirPath string,
+	filter string,
 ) (GitRef, error) {
-	return newGitRef("", path, gitName, depth, recurseSubmodules, subDirPath)
+	return newGitRef("", path, gitName, depth, recurseSubmodules, subDirPath, filter)
 }
 
 // ModuleRef is a module reference.
@@ -353,6 +356,7 @@ func NewDirectParsedGitRef(
 	recurseSubmodules bool,
 	depth uint32,
 	subDirPath string,
+	filter string,
 ) ParsedGitRef {
 	return newDirectGitRef(
 		format,
@@ -362,6 +366,7 @@ func NewDirectParsedGitRef(
 		recurseSubmodules,
 		depth,
 		subDirPath,
+		filter,
 	)
 }
 
@@ -557,6 +562,9 @@ type RawRef struct {
 	// requested GitRef will be included when cloning the requested branch
 	// (or the repo's default branch if GitBranch is empty).
 	GitDepth uint32
+	// Only set for git formats.
+	// The filter spec to use, see the --filter option in git rev-list.
+	GitFilter string
 	// Only set for archive formats.
 	ArchiveStripComponents uint32
 	// Only set for proto file ref format.

private/buf/buffetch/internal/reader.go

@@ -370,6 +370,8 @@ func (r *reader) getGitBucket(
 		git.CloneToBucketOptions{
 			Name:              gitRef.GitName(),
 			RecurseSubmodules: gitRef.RecurseSubmodules(),
+			SubDir:            gitRef.SubDirPath(),
+			Filter:            gitRef.Filter(),
 		},
 	); err != nil {
 		return nil, nil, fmt.Errorf("could not clone %s: %v", gitURL, err)

private/buf/buffetch/internal/ref_parser.go

@@ -142,6 +142,8 @@ func (a *refParser) getRawRef(
 			rawRef.GitCommitOrTag = value
 		case "ref":
 			rawRef.GitRef = value
+		case "filter":
+			rawRef.GitFilter = value
 		case "depth":
 			depth, err := parseGitDepth(value)
 			if err != nil {
@@ -519,6 +521,7 @@ func getGitRef(
 		rawRef.GitDepth,
 		rawRef.GitRecurseSubmodules,
 		rawRef.SubDirPath,
+		rawRef.GitFilter,
 	)
 }
 

private/pkg/execext/execext.go

@@ -16,6 +16,7 @@ package execext
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os/exec"
 	"slices"
@@ -34,6 +35,7 @@ func Run(ctx context.Context, name string, options ...RunOption) error {
 	for _, option := range options {
 		option.applyRun(runStartOptions)
 	}
+	fmt.Println("RUNNING", name, runStartOptions.args)
 	cmd := exec.CommandContext(ctx, name, runStartOptions.args...)
 	runStartOptions.applyCmd(cmd)
 	return cmd.Run()

private/pkg/git/cloner.go

@@ -125,6 +125,21 @@ func (c *cloner) CloneToBucket(
 			return err
 		}
 	}
+
+	// Build the args for the fetch command.
+	fetchArgs := []string{}
+	fetchArgs = append(fetchArgs, gitConfigAuthArgs...)
+	fetchArgs = append(
+		fetchArgs,
+		"fetch",
+		"--depth", depthArg,
+		// Required on branches matching the current branch of git init.
+		"--update-head-ok",
+	)
+	if options.Filter != "" {
+		fetchArgs = append(fetchArgs, "--filter", options.Filter)
+	}
+
 	// First, try to fetch the fetchRef directly. If the ref is not found, we
 	// will try to fetch the fallback ref with a depth to allow resolving partial
 	// refs locally. If the fetch fails, we will return an error.
@@ -134,14 +149,7 @@ func (c *cloner) CloneToBucket(
 	if err := execext.Run(
 		ctx,
 		"git",
-		execext.WithArgs(append(
-			gitConfigAuthArgs,
-			"fetch",
-			"--depth", depthArg,
-			"--update-head-ok", // Required on branches matching the current branch of git init.
-			"origin",
-			fetchRef,
-		)...),
+		execext.WithArgs(append(fetchArgs, "origin", fetchRef)...),
 		execext.WithEnv(app.Environ(envContainer)),
 		execext.WithStderr(buffer),
 		execext.WithDir(baseDir.Path()),
@@ -156,14 +164,24 @@ func (c *cloner) CloneToBucket(
 		if err := execext.Run(
 			ctx,
 			"git",
-			execext.WithArgs(append(
-				gitConfigAuthArgs,
-				"fetch",
-				"--depth", depthArg,
-				"--update-head-ok", // Required on branches matching the current branch of git init.
-				"origin",
-				fallbackRef,
-			)...),
+			execext.WithArgs(append(fetchArgs, "origin", fallbackRef)...),
+			execext.WithEnv(app.Environ(envContainer)),
+			execext.WithStderr(buffer),
+			execext.WithDir(baseDir.Path()),
+		); err != nil {
+			return newGitCommandError(err, buffer)
+		}
+	}
+
+	// As a further optimization, if a filter is applied with a subdir, we run
+	// a sparse checkout to reduce the size of the working directory.
+	buffer.Reset()
+	if options.Filter != "" && options.SubDir != "" {
+		// Set the subdir for sparse checkout.
+		if err := execext.Run(
+			ctx,
+			"git",
+			execext.WithArgs("sparse-checkout", "set", options.SubDir),
 			execext.WithEnv(app.Environ(envContainer)),
 			execext.WithStderr(buffer),
 			execext.WithDir(baseDir.Path()),

private/pkg/git/git.go

@@ -104,6 +104,8 @@ type CloneToBucketOptions struct {
 	Matcher           storage.Matcher
 	Name              Name
 	RecurseSubmodules bool
+	SubDir            string
+	Filter            string
 }
 
 // NewCloner returns a new Cloner.