Skip to content

Commit 9f00ced

Browse files
authored
perf:optimize smart_open for S3-to-S3 transfers using s3+protocol (#531)
1 parent 25f251b commit 9f00ced

2 files changed

Lines changed: 16 additions & 4 deletions

File tree

megfile/smart.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,16 @@ def _default_copy_func(
307307
pass
308308

309309

310+
def _get_copy_func(src_protocol, dst_protocol):
311+
if src_protocol.startswith("s3+") and src_protocol == dst_protocol:
312+
src_protocol = dst_protocol = "s3"
313+
314+
try:
315+
return _copy_funcs[src_protocol][dst_protocol]
316+
except KeyError:
317+
return _default_copy_func
318+
319+
310320
def smart_copy(
311321
src_path: PathLike,
312322
dst_path: PathLike,
@@ -351,10 +361,8 @@ def smart_copy(
351361
src_protocol, _ = SmartPath._extract_protocol(src_path)
352362
dst_protocol, _ = SmartPath._extract_protocol(dst_path)
353363

354-
try:
355-
copy_func = _copy_funcs[src_protocol][dst_protocol]
356-
except KeyError:
357-
copy_func = _default_copy_func
364+
copy_func = _get_copy_func(src_protocol, dst_protocol)
365+
358366
try:
359367
copy_func(
360368
src_path,

tests/test_smart.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1232,3 +1232,7 @@ def test_smart_concat(s3_empty_client, fs):
12321232
assert smart.smart_load_content("c.txt") == b"ab"
12331233

12341234
smart.smart_concat([], "c.txt")
1235+
1236+
1237+
def test__get_copy_func():
1238+
assert smart._get_copy_func("s3+a", "s3+a") == smart._copy_funcs["s3"]["s3"]

0 commit comments

Comments
 (0)