Skip to content

Commit b603b0e

Browse files
authored
test: cover UPath and DVC integrations (#68)
1 parent 46f318b commit b603b0e

5 files changed

Lines changed: 117 additions & 0 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ venv/
55
*.pyc
66
.pdm-python
77
.idea
8+
.downstreams/
89

910
opendalfs.egg-info/
1011
dist/

pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,14 @@ bench = [
3030
test = [
3131
"boto3>=1.37.3",
3232
"dask[dataframe]==2026.8.0",
33+
"dvc-objects==5.2.0",
3334
"pandas==3.0.5",
3435
"pyarrow>=22.0.0",
3536
"pytest>=8.4.0",
3637
"pytest-asyncio>=1.0.0",
3738
"pytest-cov>=6.1.1",
3839
"s3fs>=2025.5.1",
40+
"universal-pathlib==0.3.10",
3941
]
4042
dev = [
4143
{ include-group = "bench" },
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""dvc-objects compatibility cases migrated for issue #51.
2+
3+
Source: dvc-objects 5.2.0,
4+
``tests/fs/test_generic.py::test_copy`` and
5+
``tests/fs/test_localfs.py::test_walk``. The operations pass an opendalfs
6+
instance through dvc-objects' existing ``MemoryFileSystem(fs=...)`` entry point.
7+
"""
8+
9+
from dvc_objects.fs.memory import MemoryFileSystem
10+
11+
from opendalfs import OpendalFileSystem
12+
13+
14+
def test_dvc_objects_find_walk_put_and_get(tmp_path):
15+
fs = MemoryFileSystem(
16+
fs=OpendalFileSystem(
17+
scheme="memory",
18+
asynchronous=False,
19+
skip_instance_cache=True,
20+
)
21+
)
22+
sources = {
23+
tmp_path / "sources" / "one.txt": b"one",
24+
tmp_path / "sources" / "nested" / "two.txt": b"two",
25+
}
26+
for path, content in sources.items():
27+
path.parent.mkdir(parents=True, exist_ok=True)
28+
path.write_bytes(content)
29+
30+
remote_paths = ["dataset/one.txt", "dataset/nested/two.txt"]
31+
fs.put([str(path) for path in sources], remote_paths)
32+
33+
assert set(fs.find("dataset")) == set(remote_paths)
34+
assert [
35+
(root, set(dirs), set(files)) for root, dirs, files in fs.walk("dataset")
36+
] == [
37+
("dataset", {"nested"}, {"one.txt"}),
38+
("dataset/nested", set(), {"two.txt"}),
39+
]
40+
41+
downloads = [tmp_path / "downloads" / path for path in ("one.txt", "two.txt")]
42+
fs.get(remote_paths, [str(path) for path in downloads])
43+
44+
assert [path.read_bytes() for path in downloads] == [b"one", b"two"]
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
"""universal_pathlib compatibility cases migrated for issue #51.
2+
3+
Source: universal-pathlib 0.3.10,
4+
``docs/usage.md`` (the public ``protocol=`` constructor) and
5+
``upath/tests/cases.py`` (``make_test_files``, ``test_iterdir``,
6+
``test_write_text`` and ``test_write_bytes``).
7+
"""
8+
9+
from upath import UPath
10+
11+
from opendalfs import register_opendal_service
12+
13+
14+
def test_upath_protocol_read_write_and_listing():
15+
protocol = register_opendal_service("memory")
16+
root = UPath("universal-pathlib", protocol=protocol)
17+
18+
folder = root / "folder1"
19+
folder.mkdir(parents=True)
20+
(folder / "file1.txt").write_text("hello world")
21+
(folder / "file2.txt").write_bytes(b"hello bytes")
22+
23+
assert (folder / "file1.txt").read_text() == "hello world"
24+
assert (folder / "file2.txt").read_bytes() == b"hello bytes"
25+
assert {path.name for path in folder.iterdir()} == {
26+
"file1.txt",
27+
"file2.txt",
28+
}

uv.lock

Lines changed: 42 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)