[WIP] implement common file path filter LRU cache

Building on the work in PR git-lfs#6047, we implement a common LRU cache
for the Filter structure in our "filepathfilter" package, which is
used by all commands that accept command-line or configuration
options to filter their actions by file paths.

Specifically, any of our commands which accept --include or --exclude
options (or their -I and -X short forms), or read the "lfs.fetchInclude"
and "lfs.fetchExclude" configuration options, should benefit from
the use of a cache to return previously-determined results when
matching file paths against the file path filter patterns.

We set the default cache size at 10,000 entries, and provide a new
"lfs.pathFilterCacheSize" configuration option which may be used to
resize the cache as desired.  The special values "none" and "0" disable
the cache, and the value "unlimited" allows the cache to grow without
bound.

Note that to pass the cache size setting to the "filepathfilter"
package we rename its "option" type to "Option" so it will be
exported and available from our "commands" package.

We use the Least-Recently Used (LRU) cache implemention from the
"golang/groupcache" Go module, which used a map plus a doubly-linked
list to track the most- and least-recently used entries.  We add a
mutex around our accesses to the cache in case any of our commands
use the Allows() method of the Filter structure from concurrent
goroutines.

Author: chrisd8088

commands/command_migrate.go

@@ -63,10 +63,6 @@ var (
 	// migrateFixup is the flag indicating whether or not to infer the
 	// included and excluded filepath patterns.
 	migrateFixup bool
-
-	// migrateCacheFilePathFilterResult is the flag indicating that a file path
-	// filter result may be cached
-	migrateCacheFilePathFilterResults bool
 )
 
 // migrate takes the given command and arguments, *gitobj.ObjectDatabase, as well
@@ -405,7 +401,6 @@ func init() {
 	importCmd.Flags().BoolVar(&migrateNoRewrite, "no-rewrite", false, "Add new history without rewriting previous")
 	importCmd.Flags().StringVarP(&migrateCommitMessage, "message", "m", "", "With --no-rewrite, an optional commit message")
 	importCmd.Flags().BoolVar(&migrateFixup, "fixup", false, "Infer filepaths based on .gitattributes")
-	importCmd.Flags().BoolVar(&migrateCacheFilePathFilterResults, "cache-file-path-filter-results", false, "Cache file path filter results")
 
 	exportCmd := NewCommand("export", migrateExportCommand)
 	exportCmd.Flags().BoolVar(&migrateVerbose, "verbose", false, "Verbose logging")

commands/commands.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +28,10 @@ import (
 // Populate man pages
 //go:generate go run ../docs/man/mangen.go
 
+const (
+	defaultFilepathFilterCacheSize = 10000
+)
+
 var (
 	ErrorBuffer  = &bytes.Buffer{}
 	ErrorWriter  = newMultiWriter(os.Stderr, ErrorBuffer)
@@ -134,7 +139,7 @@ func buildFilepathFilter(config *config.Configuration, includeArg, excludeArg *s
 
 func buildFilepathFilterWithPatternType(config *config.Configuration, includeArg, excludeArg *string, useFetchOptions bool, patternType filepathfilter.PatternType) *filepathfilter.Filter {
 	inc, exc := determineIncludeExcludePaths(config, includeArg, excludeArg, useFetchOptions)
-	return filepathfilter.New(inc, exc, patternType, filepathfilter.UseCache(migrateCacheFilePathFilterResults))
+	return filepathfilter.New(inc, exc, patternType, determineFilepathFilterCache(config))
 }
 
 func downloadTransfer(p *lfs.WrappedPointer) (name, path, oid string, size int64, missing bool, err error) {
@@ -523,6 +528,29 @@ func determineIncludeExcludePaths(config *config.Configuration, includeArg, excl
 	return
 }
 
+func determineFilepathFilterCache(config *config.Configuration) filepathfilter.Option {
+	cacheSize := defaultFilepathFilterCacheSize
+
+	if configSize, ok := config.Git.Get("lfs.pathfiltercachesize"); ok {
+		switch configSize {
+		case "none":
+			return filepathfilter.DisableCache()
+		case "unlimited":
+			cacheSize = 0
+		default:
+			if s, err := strconv.Atoi(configSize); err == nil {
+				if s == 0 {
+					return filepathfilter.DisableCache()
+				} else if s > 0 {
+					cacheSize = s
+				}
+			}
+		}
+	}
+
+	return filepathfilter.EnableCache(cacheSize)
+}
+
 func buildProgressMeter(dryRun bool, d tq.Direction) *tq.Meter {
 	m := tq.NewMeter(cfg)
 	m.Logger = m.LoggerFromEnv(cfg.Os)

docs/man/git-lfs-config.adoc

@@ -106,6 +106,14 @@ interpreted as the password.
 +
 Enables in-memory SSH and Git Credential caching for a single 'git lfs'
 command. Default: enabled.
+* `lfs.pathFilterCacheSize`
++
+Sets the size of the in-memory cache of results from matching file paths
+against the filter defined by the `lfs.fetchInclude` and `lfs.fetchExclude`
+options, or for commands which accept them, by the `--include` and `--exclude`
+command-line options or their `-I` and `-X` equivalents.
+Set to `0` or `none` to disable the cache, or to `unlimited` to allow
+the cache to grow without bound.  Default: 10,000 unique file paths.
 * `lfs.storage`
 +
 Allow override LFS storage directory. Non-absolute path is relativized

docs/man/git-lfs-migrate.adoc

@@ -149,10 +149,6 @@ export::
 `--include-ref` or `--exclude-ref`.
 `--everything`::
   See <<_include_and_exclude_references>>.
-`--cache-file-path-filter-results`::
-  Makes Git LFS cache the result of include or exclude file path pattern matching
-  while importing Git objects to LFS. This may speed up migration of a large
-  repository.
 +
 Note: Git refs are "case-sensitive" on all platforms in "packed from"
 (see `git-pack-refs(1)`). On "case-insensitive" file systems, e.g. NTFS

filepathfilter/filepathfilter.go

@@ -2,9 +2,11 @@ package filepathfilter
 
 import (
 	"strings"
+	"sync"
 
 	"github.com/git-lfs/git-lfs/v3/tr"
 	"github.com/git-lfs/wildmatch/v2"
+	"github.com/golang/groupcache/lru"
 	"github.com/rubyist/tracerx"
 )
 
@@ -19,7 +21,8 @@ type Filter struct {
 	include      []Pattern
 	exclude      []Pattern
 	defaultValue bool
-	cache        map[string]bool
+	cache        *lru.Cache
+	cacheLock    sync.Mutex
 }
 
 type PatternType bool
@@ -39,38 +42,48 @@ func (p PatternType) String() string {
 type options struct {
 	defaultValue bool
 	useCache     bool
+	cacheSize    int
 }
 
-type option func(*options)
+type Option func(*options)
 
 // DefaultValue is an option representing the default value of a filepathfilter
 // if no patterns match.  If this option is not provided, the default is true.
-func DefaultValue(val bool) option {
+func DefaultValue(val bool) Option {
 	return func(args *options) {
 		args.defaultValue = val
 	}
 }
 
-func UseCache(val bool) option {
+func EnableCache(size int) Option {
 	return func(args *options) {
-		args.useCache = val
+		args.useCache = true
+		args.cacheSize = size
 	}
 }
 
-func NewFromPatterns(include, exclude []Pattern, setters ...option) *Filter {
+func DisableCache() Option {
+	return func(args *options) {
+		args.useCache = false
+	}
+}
+
+func NewFromPatterns(include, exclude []Pattern, setters ...Option) *Filter {
 	args := &options{defaultValue: true, useCache: false}
 	for _, setter := range setters {
 		setter(args)
 	}
-	var cache map[string]bool
-	cache = nil
-	if args.useCache {
-		cache = make(map[string]bool)
+
+	f := &Filter{include: include, exclude: exclude, defaultValue: args.defaultValue}
+
+	if args.useCache && args.cacheSize >= 0 {
+		f.cache = lru.New(args.cacheSize)
 	}
-	return &Filter{include: include, exclude: exclude, defaultValue: args.defaultValue, cache: cache}
+
+	return f
 }
 
-func New(include, exclude []string, ptype PatternType, setters ...option) *Filter {
+func New(include, exclude []string, ptype PatternType, setters ...Option) *Filter {
 	return NewFromPatterns(
 		convertToWildmatch(include, ptype),
 		convertToWildmatch(exclude, ptype), setters...)
@@ -95,7 +108,7 @@ func wildmatchToString(ps ...Pattern) []string {
 	return s
 }
 
-func (f *Filter) allowsUncached(filename string) bool {
+func (f *Filter) allows(filename string) bool {
 	var included bool
 	for _, inc := range f.include {
 		if included = inc.Match(filename); included {
@@ -135,17 +148,23 @@ func (f *Filter) Allows(filename string) bool {
 		return true
 	}
 
-	if f.cache == nil {
-		return f.allowsUncached(filename)
+	if f.cache != nil {
+		f.cacheLock.Lock()
+		res, ok := f.cache.Get(filename)
+		f.cacheLock.Unlock()
+		if ok {
+			return res.(bool)
+		}
 	}
 
-	cachedResult, cacheHit := f.cache[filename]
-	if cacheHit {
-		return cachedResult
+	res := f.allows(filename)
+
+	if f.cache != nil {
+		f.cacheLock.Lock()
+		f.cache.Add(filename, res)
+		f.cacheLock.Unlock()
 	}
 
-	res := f.allowsUncached(filename)
-	f.cache[filename] = res
 	return res
 }
 

git/githistory/rewriter_test.go

@@ -228,7 +228,7 @@ func TestRewriterIgnoresPathsThatDontMatchFilterWithResultCaching(t *testing.T)
 	include := []string{"*.txt"}
 	exclude := []string{"subdir/*.txt"}
 
-	filter := filepathfilter.New(include, exclude, filepathfilter.GitIgnore, filepathfilter.UseCache(true))
+	filter := filepathfilter.New(include, exclude, filepathfilter.GitIgnore, filepathfilter.UseCache(10))
 
 	db := DatabaseFromFixture(t, "non-repeated-subtrees.git")
 	r := NewRewriter(db, WithFilter(filter))

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/git-lfs/go-netrc v0.0.0-20250218165306-ba0029b43d11
 	github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825
 	github.com/git-lfs/wildmatch/v2 v2.0.1
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8
 	github.com/jmhodges/clock v1.2.0
 	github.com/leonelquinteros/gotext v1.5.0
 	github.com/mattn/go-isatty v0.0.4

go.sum

@@ -16,6 +16,8 @@ github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825 h1:riQhgheTL7tMF4d
 github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825/go.mod h1:fenKRzpXDjNpsIBhuhUzvjCKlDjKam0boRAenTE0Q6A=
 github.com/git-lfs/wildmatch/v2 v2.0.1 h1:Ds+aobrV5bK0wStILUOn9irllPyf9qrFETbKzwzoER8=
 github.com/git-lfs/wildmatch/v2 v2.0.1/go.mod h1:EVqonpk9mXbREP3N8UkwoWdrF249uHpCUo5CPXY81gw=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=