WIP

Author: chrisd8088

commands/command_fetch.go

@@ -19,6 +19,7 @@ var (
 	fetchRecentArg bool
 	fetchAllArg    bool
 	fetchPruneArg  bool
+	fetchDryRunArg bool
 )
 
 func getIncludeExcludeArgs(cmd *cobra.Command) (include, exclude *string) {
@@ -106,7 +107,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 		verifyUnreachable := fetchPruneCfg.PruneVerifyUnreachableAlways
 
 		// assume false for non available options in fetch
-		prune(fetchPruneCfg, verify, verifyUnreachable, false, false, false)
+		prune(fetchPruneCfg, verify, verifyUnreachable, false, fetchDryRunArg, fetchDryRunArg)
 	}
 
 	if !success {
@@ -147,7 +148,7 @@ func fetchRef(ref string, filter *filepathfilter.Filter) bool {
 	if err != nil {
 		Panic(err, tr.Tr.Get("Could not scan for Git LFS files"))
 	}
-	return fetchAndReportToChan(pointers, filter, nil)
+	return fetch(pointers)
 }
 
 func pointersToFetchForRefs(refs []string) ([]*lfs.WrappedPointer, error) {
@@ -189,7 +190,7 @@ func fetchRefs(refs []string) bool {
 	if err != nil {
 		Panic(err, tr.Tr.Get("Could not scan for Git LFS files"))
 	}
-	return fetchAndReportToChan(pointers, nil, nil)
+	return fetch(pointers)
 }
 
 // Fetch all previous versions of objects from since to ref (not including final state at ref)
@@ -212,7 +213,7 @@ func fetchPreviousVersions(ref string, since time.Time, filter *filepathfilter.F
 		ExitWithError(err)
 	}
 
-	return fetchAndReportToChan(pointers, filter, nil)
+	return fetch(pointers)
 }
 
 // Fetch recent objects based on config
@@ -282,7 +283,7 @@ func fetchRecent(fetchconf lfs.FetchPruneConfig, alreadyFetchedRefs []*git.Ref,
 func fetchAll() bool {
 	pointers := scanAll()
 	Print("fetch: %s", tr.Tr.Get("Fetching all references..."))
-	return fetchAndReportToChan(pointers, nil, nil)
+	return fetch(pointers)
 }
 
 func scanAll() []*lfs.WrappedPointer {
@@ -323,44 +324,23 @@ func scanAll() []*lfs.WrappedPointer {
 	return pointers
 }
 
-// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
+func printTransfers(out <-chan *tq.Transfer) {
+	for p := range out {
+		Print("%s %s => %s", tr.Tr.Get("fetch"), p.Oid, p.Name)
+	}
+}
+
+// Fetch
 // Returns true if all completed with no errors, false if errors were written to stderr/log
-func fetchAndReportToChan(allpointers []*lfs.WrappedPointer, filter *filepathfilter.Filter, out chan<- *lfs.WrappedPointer) bool {
-	ready, pointers, meter := readyAndMissingPointers(allpointers, filter)
+func fetch(allpointers []*lfs.WrappedPointer) bool {
+	pointers, meter := missingPointers(allpointers)
 	q := newDownloadQueue(
 		getTransferManifestOperationRemote("download", cfg.Remote()),
-		cfg.Remote(), tq.WithProgress(meter),
+		cfg.Remote(), tq.WithProgress(meter), tq.DryRun(fetchDryRunArg),
 	)
 
-	if out != nil {
-		// If we already have it, or it won't be fetched
-		// report it to chan immediately to support pull/checkout
-		for _, p := range ready {
-			out <- p
-		}
-
-		dlwatch := q.Watch()
-
-		go func() {
-			// fetch only reports single OID, but OID *might* be referenced by multiple
-			// WrappedPointers if same content is at multiple paths, so map oid->slice
-			oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
-			for _, pointer := range pointers {
-				plist := oidToPointers[pointer.Oid]
-				oidToPointers[pointer.Oid] = append(plist, pointer)
-			}
-
-			for t := range dlwatch {
-				plist, ok := oidToPointers[t.Oid]
-				if !ok {
-					continue
-				}
-				for _, p := range plist {
-					out <- p
-				}
-			}
-			close(out)
-		}()
+	if fetchDryRunArg {
+		go printTransfers(q.Watch())
 	}
 
 	for _, p := range pointers {
@@ -381,37 +361,27 @@ func fetchAndReportToChan(allpointers []*lfs.WrappedPointer, filter *filepathfil
 	return ok
 }
 
-func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, *tq.Meter) {
+func missingPointers(allpointers []*lfs.WrappedPointer) ([]*lfs.WrappedPointer, *tq.Meter) {
 	logger := tasklog.NewLogger(os.Stdout,
 		tasklog.ForceProgress(cfg.ForceProgress()),
 	)
-	meter := buildProgressMeter(false, tq.Download)
+	meter := buildProgressMeter(fetchDryRunArg, tq.Download)
 	logger.Enqueue(meter)
 
-	seen := make(map[string]bool, len(allpointers))
 	missing := make([]*lfs.WrappedPointer, 0, len(allpointers))
-	ready := make([]*lfs.WrappedPointer, 0, len(allpointers))
 
 	for _, p := range allpointers {
-		// no need to download the same object multiple times
-		if seen[p.Oid] {
-			continue
-		}
-
-		seen[p.Oid] = true
-
 		// no need to download objects that exist locally already
 		lfs.LinkOrCopyFromReference(cfg, p.Oid, p.Size)
 		if cfg.LFSObjectExists(p.Oid, p.Size) {
-			ready = append(ready, p)
 			continue
 		}
 
 		missing = append(missing, p)
 		meter.Add(p.Size)
 	}
 
-	return ready, missing, meter
+	return missing, meter
 }
 
 func init() {
@@ -421,5 +391,6 @@ func init() {
 		cmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits")
 		cmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced")
 		cmd.Flags().BoolVarP(&fetchPruneArg, "prune", "p", false, "After fetching, prune old data")
+		cmd.Flags().BoolVarP(&fetchDryRunArg, "dry-run", "d", false, "Do not fetch, only show what would be fetched")
 	})
 }

t/t-fetch.sh

@@ -63,11 +63,19 @@ begin_test "fetch"
   cd clone
   rm -rf .git/lfs/objects
 
+  git lfs fetch --dry-run 2>&1 | tee fetch.log
+  grep "fetch $contents_oid => a\.dat" fetch.log
+  refute_local_object "$contents_oid"
+
   git lfs fetch
   assert_local_object "$contents_oid" 1
 
   git lfs fsck 2>&1 | tee fsck.log
   grep "Git LFS fsck OK" fsck.log
+
+  git lfs fetch --dry-run 2>&1 | tee fetch.log
+  grep "fetch .* => a\.dat" fetch.log && exit 1
+  true
 )
 end_test
 
@@ -128,12 +136,22 @@ begin_test "fetch with remote and branches"
 
   rm -rf .git/lfs/objects
 
+  git lfs fetch origin main newbranch --dry-run | tee fetch.log
+  grep "fetch $contents_oid => a\.dat" fetch.log
+  grep "fetch $b_oid => b\.dat" fetch.log
+  refute_local_object "$contents_oid"
+  refute_local_object "$b_oid"
+
   git lfs fetch origin main newbranch
   assert_local_object "$contents_oid" 1
   assert_local_object "$b_oid" 1
 
   git lfs fsck 2>&1 | tee fsck.log
   grep "Git LFS fsck OK" fsck.log
+
+  git lfs fetch origin main newbranch --dry-run | tee fetch.log
+  grep "fetch .* => [ab]\.dat" fetch.log && exit 1
+  true
 )
 end_test