git: improve sparse file support

When we invoke `git ls-files` to try to find all LFS files, we don't
honour sparse file paths or exclusions.  While we should never actually
traverse excluded files, using the `--exclude-standard` option can
avoid loading some data with filtered clones, which may result in less
data being downloaded.

In addition, we can honour sparse checkouts, since this code path is
only used to handle the working tree and we know that the only files we
need to consider are those Git actually put in the working tree.  The
`--sparse` option is new in 2.35, but we already require 2.42 above, so
we can use it unconditionally.

Co-authored-by: Chris Darroch <[email protected]>

Author: bk2204

git/git.go

@@ -329,7 +329,9 @@ func LsFilesLFS() (*subprocess.BufferedCmd, error) {
 	return gitNoLFSBuffered(
 		"ls-files",
 		"--cached",
+		"--exclude-standard",
 		"--full-name",
+		"--sparse",
 		"-z",
 		"--format=%(objectmode) %(objecttype) %(objectname) %(objectsize)\t%(path)",
 		":(top,attr:filter=lfs)",

t/t-checkout.sh

@@ -280,3 +280,68 @@ begin_test "checkout: GIT_WORK_TREE"
   [ "$contents" = "$(cat "$reponame/file1.dat")" ]
 )
 end_test
+
+begin_test "checkout with partial clone and sparse checkout and index"
+(
+  set -e
+
+  # Only test with Git version 2.42.0 as it introduced support for the
+  # "objecttype" format option to the "git ls-files" command, which our
+  # code requires.
+  ensure_git_version_isnt "$VERSION_LOWER" "2.42.0"
+
+  reponame="checkout-partial-clone"
+  setup_remote_repo "$reponame"
+
+  clone_repo "$reponame" "$reponame"
+
+  git lfs track "*.dat"
+
+  contents1="a"
+  contents1_oid=$(calc_oid "$contents1")
+  contents2="b"
+  contents2_oid=$(calc_oid "$contents2")
+  contents3="c"
+  contents3_oid=$(calc_oid "$contents3")
+
+  mkdir in out
+  printf "%s" "$contents1" > a.dat
+  printf "%s" "$contents2" > in/b.dat
+  printf "%s" "$contents3" > out/c.dat
+  git add .
+  git commit -m "add files"
+
+  git push origin main
+
+  assert_server_object "$reponame" "$contents1_oid"
+  assert_server_object "$reponame" "$contents2_oid"
+  assert_server_object "$reponame" "$contents3_oid"
+
+  # Create a partial clone with a cone-mode sparse checkout of one directory
+  # and a sparse index, which is important because otherwise the "git ls-files"
+  # command ignores the --sparse option and lists all LFS files.
+  cd ..
+  git clone --filter=tree:0 --depth=1 --no-checkout \
+    "$GITSERVER/$reponame" partial
+
+  cd partial
+  git sparse-checkout init --cone --sparse-index
+  git sparse-checkout set in
+  GIT_LFS_SKIP_SMUDGE=1 git checkout main
+  git lfs fetch origin main
+
+  # This was downloaded by `git lfs fetch`.
+  delete_local_object "$contents3_oid"
+
+  assert_local_object "$contents1_oid" 1
+  assert_local_object "$contents2_oid" 1
+  refute_local_object "$contents3_oid" 1
+
+  git lfs checkout
+  test -f "out/c.dat" && exit 1
+
+  assert_local_object "$contents1_oid" 1
+  assert_local_object "$contents2_oid" 1
+  refute_local_object "$contents3_oid" 1
+)
+end_test

t/t-pull.sh

@@ -392,3 +392,67 @@ begin_test "pull with empty file doesn't modify mtime"
   diff -u foo.mtime foo.mtime2
 )
 end_test
+
+begin_test "pull with partial clone and sparse checkout"
+(
+  set -e
+
+  # Only test with Git version 2.42.0 as it introduced support for the
+  # "objecttype" format option to the "git ls-files" command, which our
+  # code requires.
+  ensure_git_version_isnt "$VERSION_LOWER" "2.42.0"
+
+  reponame="pull-sparse"
+  setup_remote_repo "$reponame"
+
+  clone_repo "$reponame" "$reponame"
+
+  git lfs track "*.dat"
+
+  contents1="a"
+  contents1_oid=$(calc_oid "$contents1")
+  contents2="b"
+  contents2_oid=$(calc_oid "$contents2")
+  contents3="c"
+  contents3_oid=$(calc_oid "$contents3")
+
+  mkdir in out
+  printf "%s" "$contents1" > a.dat
+  printf "%s" "$contents2" > in/b.dat
+  printf "%s" "$contents3" > out/c.dat
+  git add .
+  git commit -m "add files"
+
+  git push origin main
+
+  assert_server_object "$reponame" "$contents1_oid"
+  assert_server_object "$reponame" "$contents2_oid"
+  assert_server_object "$reponame" "$contents3_oid"
+
+  # Create a partial clone with a cone-mode sparse checkout of one directory
+  # and a sparse index, which is important because otherwise the "git ls-files" 
+  # command ignores the --sparse option and lists all Git LFS files.
+  cd ..
+  git clone --filter=tree:0 --depth=1 --no-checkout \
+    "$GITSERVER/$reponame" "${reponame}-partial"
+
+  cd "${reponame}-partial"
+  git sparse-checkout init --cone --sparse-index
+  git sparse-checkout set in
+  git checkout main
+
+  [ -d "in" ]
+  [ ! -e "out" ]
+
+  assert_local_object "$contents1_oid" 1
+  assert_local_object "$contents2_oid" 1
+  refute_local_object "$contents3_oid"
+
+  git lfs pull 2>&1 | tee pull.log
+  grep -q "Downloading LFS objects" pull.log && exit 1
+
+  # Git LFS objects associated with files outside of the sparse cone
+  # should not have been pulled.
+  refute_local_object "$contents3_oid"
+)
+end_test