Skip to content

Commit e218034

Browse files
committed
fix: tolerate stray directories in list_partitions (improvements)
1 parent 19aadfc commit e218034

1 file changed

Lines changed: 17 additions & 5 deletions

File tree

  • zcollection/partitioning

zcollection/partitioning/abc.py

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import abc
1313
import collections
1414
from collections.abc import Callable, Generator, Iterator, Sequence
15+
import operator
1516
import re
1617
from re import Match
1718

@@ -403,6 +404,10 @@ def join(partition_scheme: tuple[tuple[str, int], ...], sep: str) -> str:
403404
"""
404405
return sep.join(f'{k}={v}' for k, v in partition_scheme)
405406

407+
import operator
408+
409+
...
410+
406411
def list_partitions(
407412
self,
408413
fs: fsspec.AbstractFileSystem,
@@ -413,14 +418,21 @@ def list_partitions(
413418
Args:
414419
fs: The filesystem to be used.
415420
path: The path to the directory containing the partitions.
421+
416422
Yields:
417423
The partitions.
418424
"""
419-
partitions = []
420-
for p in list_partitions(fs, path, depth=len(self) - 1):
425+
parsed_partitions = []
426+
427+
for partition in list_partitions(fs, path, depth=len(self) - 1):
421428
try:
422-
self.parse(p)
429+
parsed_partition = self.parse(partition)
423430
except ValueError:
424431
continue
425-
partitions.append(p)
426-
yield from sorted(partitions, key=self.parse)
432+
433+
parsed_partitions.append((parsed_partition, partition))
434+
435+
parsed_partitions.sort(key=operator.itemgetter(0))
436+
437+
for _, partition in parsed_partitions:
438+
yield partition

0 commit comments

Comments
 (0)