Skip to content

Commit dc465f7

Browse files
zzl0meta-codesync[bot]
authored andcommitted
subtree: add --git-shallow-clone flag to subtree import
Summary: Add a --git-shallow-clone flag to `sl subtree import` that fetches only the requested commit using `git fetch --depth=1`. This is faster for importing from large repositories since it avoids downloading full history. The shallow-cloned Git repo is placed in a temp directory and deleted after import, because a shallow clone lacks the commit history needed for blame and log operations. ## Performance ### 1. facebook/sapling Before ``` $ time sl subtree import --url https://github.com/facebook/sapling -r 5401649 --to-path fb_sl01 -m "import fb sl" ... Receiving objects: 100% (1019726/1019726), 268.11 MiB | 48.80 MiB/s, done. Resolving deltas: 100% (694194/694194), done. ... ________________________________________________________ Executed in 30.64 secs fish external usr time 42.47 secs 0.00 micros 42.47 secs sys time 17.92 secs 445.00 micros 17.92 secs ``` After ``` $ time sl subtree import --url https://github.com/facebook/sapling -r 5401649 --to-path fb_sl02 -m "import fb sl" --git-shallow-clone ... Receiving objects: 100% (12613/12613), 29.62 MiB | 19.82 MiB/s, done. Resolving deltas: 100% (2705/2705), done. ... ________________________________________________________ Executed in 17.35 secs fish external usr time 6.73 secs 408.00 micros 6.73 secs sys time 2.96 secs 27.00 micros 2.96 secs ``` ### 2. python/cpython Before ``` $ time sl subtree import --url https://github.com/python/cpython.git -r 36f15ba5cd15607fb4e4908ddbfb462c44626d6b --to-path cpython -m "import cpython" ... ________________________________________________________ Executed in 35.94 secs fish external usr time 81.70 secs 405.00 micros 81.70 secs sys time 13.57 secs 44.00 micros 13.57 secs ``` After ``` $ time sl subtree import --url https://github.com/python/cpython.git -r 36f15ba5cd15607fb4e4908ddbfb462c44626d6b --to-path cpython2 -m "import cpython" --git-shallow-clone ________________________________________________________ Executed in 15.44 secs fish external usr time 5.16 secs 40.71 millis 5.11 secs sys time 1.77 secs 4.97 millis 1.77 secs ``` Reviewed By: janezhang10 Differential Revision: D99630689 fbshipit-source-id: 5b912cd90884dd9b581d1f2ea4b41b189d107d5e
1 parent b52a402 commit dc465f7

File tree

4 files changed

+76
-6
lines changed

4 files changed

+76
-6
lines changed

eden/scm/sapling/commands/subtree.py

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@
33
# This software may be used and distributed according to the terms of the
44
# GNU General Public License version 2.
55

6+
import contextlib
67
import json
78
import os
9+
import shutil
10+
import tempfile
811
from collections import defaultdict
912
from typing import List
1013

@@ -158,6 +161,12 @@ def subtree_copy(ui, repo, *args, **opts):
158161
_("REV"),
159162
),
160163
("f", "force", None, _("overwrite existing path")),
164+
(
165+
"",
166+
"git-shallow-clone",
167+
None,
168+
_("use git shallow clone (--depth=1) (EXPERIMENTAL)"),
169+
),
161170
]
162171
+ subtree_path_opts
163172
+ commitopts
@@ -177,6 +186,11 @@ def subtree_import(ui, repo, *args, **opts):
177186
Optionally, a source path within the external repository can be specified to
178187
import only a subdirectory.
179188
189+
Use ``--git-shallow-clone`` to fetch only the requested commit without
190+
full history (``git fetch --depth=1``). This is faster for large
191+
repositories but future blame or log on the imported path may be slower
192+
because they need to download the commit history.
193+
180194
Examples:
181195
182196
- Import entire repository::
@@ -825,6 +839,26 @@ def commitfunc(ui, repo, message, match, opts):
825839
cmdutil.commit(ui, repo, commitfunc, [], opts)
826840

827841

842+
@contextlib.contextmanager
843+
def _shallow_clone_git_repo(ui, url, from_rev):
844+
"""Clone a git repo with --depth=1 into a temp directory.
845+
846+
The cloned repo is not cached and is deleted after the context exits,
847+
because a shallow clone does not support blame or log operations.
848+
"""
849+
from .. import git
850+
851+
git_repo_dir = tempfile.mkdtemp(prefix="sl-subtree-")
852+
# disable partial clone when shallow clone is enabled
853+
overrides = {("git", "depth"): 1, ("git", "filter"): None}
854+
ui.status(_("creating git repo at %s\n") % git_repo_dir)
855+
try:
856+
with ui.configoverride(overrides, "subtree-import"):
857+
yield git.clone(ui, url, git_repo_dir, update=from_rev)
858+
finally:
859+
shutil.rmtree(git_repo_dir, ignore_errors=True)
860+
861+
828862
def _do_import(ui, repo, *args, **opts):
829863
cmdutil.bailifchanged(repo)
830864

@@ -850,13 +884,18 @@ def _do_import(ui, repo, *args, **opts):
850884

851885
abort_or_remove_paths(ui, repo, to_paths, "import", opts)
852886

853-
git_repo = get_or_clone_git_repo(ui, giturl, from_rev)
854-
from_ctx = git_repo[from_rev]
855-
subtreeutil.validate_path_exist(ui, from_ctx, from_paths, abort_on_missing=True)
856-
857-
copy_files(ui, git_repo, repo, from_ctx, from_paths, to_paths, "import")
887+
if opts.get("git_shallow_clone"):
888+
repo_ctx_manager = _shallow_clone_git_repo(ui, giturl, from_rev)
889+
else:
890+
repo_ctx_manager = contextlib.nullcontext(
891+
get_or_clone_git_repo(ui, giturl, from_rev)
892+
)
858893

859-
from_commit = from_ctx.hex()
894+
with repo_ctx_manager as git_repo:
895+
from_ctx = git_repo[from_rev]
896+
from_commit = from_ctx.hex()
897+
subtreeutil.validate_path_exist(ui, from_ctx, from_paths, abort_on_missing=True)
898+
copy_files(ui, git_repo, repo, from_ctx, from_paths, to_paths, "import")
860899
# use the original `url` in the metadata, as the `giturl` may lost information
861900
# e.g.: "git+" prefix
862901
extra = subtreeutil.gen_import_info(ui, url, from_commit, from_paths, to_paths)

eden/scm/sapling/git.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -765,6 +765,9 @@ def pullrefspecs(repo, url, refspecs):
765765
# Nothing to pull
766766
return 0
767767
args = ["fetch", "--no-tags", "--prune"]
768+
depth = repo.ui.configint("git", "depth")
769+
if depth:
770+
args.append(f"--depth={depth}")
768771
if repo.ui.configbool("git", "shallow"):
769772
filter_config = repo.ui.config("git", "filter")
770773
if filter_config:

eden/scm/tests/test-subtree-import-blame.t

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,10 @@ Test blame support phabricator diff number
6060
: 1
6161
D1234567: 2
6262
: 3
63+
64+
Test subtree import with --git-shallow-clone, blame should work
65+
66+
$ sl subtree import -q --url $GIT_URL --rev main --to-path bar2 -m "import gitrepo to bar2" --git-shallow-clone
67+
$ sl blame bar2/alpha
68+
b6c31add3e60~: 1
69+
6a5b13188f04~: 2

eden/scm/tests/test-subtree-import-log.t

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,3 +201,24 @@ Test log.follow-xrepo config
201201
│ user: test
202202
~ date: Thu Jan 01 00:00:00 1970 +0000
203203
summary: import gitrepo to bar
204+
205+
Test subtree import with --git-shallow-clone, log should work
206+
207+
$ sl subtree import -q --url $GIT_URL --rev main --to-path bar2 -m "import gitrepo to bar2" --git-shallow-clone
208+
$ sl log bar2/alpha
209+
commit: * (glob)
210+
user: test
211+
date: Thu Jan 01 00:00:00 1970 +0000
212+
summary: import gitrepo to bar2
213+
214+
commit: 6a5b13188f04~
215+
bookmark: remote/main
216+
hoistedname: main
217+
user: test <test@example.org>
218+
date: Mon Jan 01 00:00:10 2007 +0000
219+
summary: update alpha\nhttps://phabricator.test.com/D1234567
220+
221+
commit: b6c31add3e60~
222+
user: test <test@example.org>
223+
date: Mon Jan 01 00:00:10 2007 +0000
224+
summary: alpha

0 commit comments

Comments
 (0)