Update documentation

Author: rchevrier

docs/advanced.md

@@ -48,6 +48,7 @@ Remote file system listing can be quite long. Implementations are usually
 shipped with layouts for an improved listing speed. See the
 {ref}`Layout <layout>` introduction if listing performance becomes an issue.
 
+(disable-layouts)=
 
 ## Disable layouts
 

docs/getting_started.md

@@ -48,6 +48,7 @@ given criterias
 
 ```{code-cell}
 from fcollections.implementations import NetcdfFilesDatabaseSwotLRL2
+
 fc = NetcdfFilesDatabaseSwotLRL2(path)
 fc.list_files(cycle_number=1)
 ```
@@ -83,6 +84,8 @@ ds = fc.query(selected_variables=['ssha'])
 list(ds.variables)
 ```
 
+### Filter types
+
 Each implementation has its own filters. By order of availability, the user
 should consult:
 
@@ -95,23 +98,42 @@ should consult:
 fc.query?
 ```
 
+### Filter values
+
+Possible values for a given filter can be displayed
+
+```{code-cell}
+fc.filter_values('version')
+```
+
+Only filters whose information are contained in the intermediated folders can be
+scanned in a quick way, other will trigger a full scan. As such, to ensure
+optimal performance, this method should be called with the layouts enabled, with
+files organized with folders (see the [advanced section](#disable-layouts)), and
+on filters whose information is encoded in the folders.
+
 ## Access metadata
 
 The database can display information about the variables and attributes
 contained in the files' collection using the ``variables_info`` method
 
 ```{code-cell}
-fc.variables_info(subset='Expert')
+# Use the enumeration name for filtering a specific subset
+fc.variables_info()
 ```
 
 It will offer a simple collapsible tree view with multiple levels of nesting
 depending on the data you manipulate
 
-In order to return consistent metadata, the method ensures that only one
-homogeneous subset is selected. In case you handle unmixable data (for example
-Expert and Unsmoothed datasets), you must give proper filters on the subset
-partitioning keys ``fc.unmixer.partition_keys``. If these filters are missing,
-an error with the possible choices will be raised.
+## Subsets
+
+### Errors on mixed subsets
+
+In order to return consistent results, most methods must work on an homogeneous
+subset of data. In case multiple subsets are mixed (for example Expert and
+Unsmoothed datasets), proper filters matching the partitioning keys must be
+given. If these filters are missing, an error with the possible choices will be
+raised.
 
 ```{code-cell}
 :tags: [raises-exception]
@@ -123,7 +145,32 @@ ds.to_netcdf(f'{path}/SWOT_L2_LR_SSH_Unsmoothed_001_012_20240101T030000_20240101
 fc.variables_info()
 ```
 
+### Compatibility matrix
+
+The following table summarizes which methods can work on mixed data. Most
+methods need homogeneous data and will require filtering the subset.
+
+| Method             | Works on mixed data ? |
+|--------------------|-----------------------|
+| ``list_files``     | Yes                   |
+| ``variables_info`` | No                    |
+| ``filter_values``  | No                    |
+| ``query``          | No                    |
+| ``map``            | No                    |
+
+### Listing subsets
+
+Subsets that are on the file system can be listed using the
+{meth}`subsets <fcollections.core.FilesDatabase.subsets>` property.
+
+```{code-cell}
+fc.subsets
+```
+
+One of the returned choices must be selected and used as a filter to work on an
+homogeneous dataset.
+
 ```{code-cell}
-# Use the enumeration name for filtering
+# Use the enumeration name for filtering a specific subset
 fc.variables_info(subset='Expert')
 ```

src/fcollections/core/_listing.py

@@ -264,6 +264,10 @@ def children(self) -> tp.Iterator[INode]:
             The child nodes, either files or folders
         """
 
+    @abc.abstractmethod
+    def clear(self):
+        """Clear child nodes."""
+
 
 class FileNode(INode):
     """File node of a file system tree."""
@@ -281,6 +285,9 @@ def children(self) -> tp.Iterator[INode]:
         """
         return []
 
+    def clear(self):
+        pass
+
 
 class DirNode(INode):
     """Directory node of a file system tree.
@@ -326,6 +333,9 @@ def children(self) -> tp.Iterable[INode]:
             self._children = list(self._compute_children())
         return self._children
 
+    def clear(self):
+        self._children = None
+
     def _compute_children(self) -> tp.Iterator[INode]:
         # return list of FileNode or DirNode instances
         # Block of code extracted from fsspec and simplified (no topbottom
@@ -798,6 +808,7 @@ def walk(node: INode, visitor: IVisitor) -> tp.Iterator[tp.Any]:
         return
 
     for child in node.children():
+        logger.debug("child %s", child.name)
         yield from walk(child, visitor.advance(result))
 
 
@@ -929,6 +940,7 @@ def discover(
             # to modify the Layout interface
             layout.set_filters(**filters)
 
+        self.root_node.clear()
         if enable_layouts:
             logger.debug("Using layouts to speed up listing")
             visitor = LayoutVisitor(self.layouts, stat_fields)

src/fcollections/implementations/_l2_lr_ssh.py

@@ -31,7 +31,7 @@
 
 
 SWOT_L2_PATTERN = re.compile(
-    r"SWOT_(?P<level>.*)_LR_SSH_(?P<subset>.*)_(?P<cycle_number>\d{3})_(?P<pass_number>\d{3})_"
+    r"SWOT_L2_LR_SSH_(?P<subset>.*)_(?P<cycle_number>\d{3})_(?P<pass_number>\d{3})_"
     r"(?P<time>\d{8}T\d{6}_\d{8}T\d{6})_(?P<version>P[I|G][A-Z]\d{1}_\d{2}).nc"
 )
 
@@ -519,15 +519,12 @@ def __init__(self):
                 FileNameFieldPeriod(
                     "time", "%Y%m%dT%H%M%S", "_", description=DESCRIPTIONS["time"]
                 ),
-                FileNameFieldEnum(
-                    "level", ProductLevel, description=DESCRIPTIONS["level"]
-                ),
                 FileNameFieldEnum(
                     "subset", ProductSubset, description=DESCRIPTIONS["subset"]
                 ),
                 L2VersionField("version"),
             ],
-            generation_string="SWOT_{level!f}_LR_SSH_{subset!f}_{cycle_number:>03d}_{pass_number:>03d}_{time!f}_{version!f}.nc",
+            generation_string="SWOT_L2_LR_SSH_{subset!f}_{cycle_number:>03d}_{pass_number:>03d}_{time!f}_{version!f}.nc",
         )
 
 
@@ -544,17 +541,17 @@ def __init__(self):
 AVISO_L2_LR_SSH_LAYOUT = Layout(
     [
         FileNameConvention(
-            re.compile(r"(?P<version>P[I|G][A-Z]\d{1})"),
+            re.compile(r"^(?P<version>P[I|G][A-Z]\d{1})$"),
             [_ADAPTED_L2_FIELD],
             "{version!f}",
         ),
         FileNameConvention(
-            re.compile(r"(?P<subset>.*)"),
+            re.compile(r"^(?P<subset>.*)$"),
             [FileNameConventionSwotL2().get_field("subset")],
             "{subset!f}",
         ),
         FileNameConvention(
-            re.compile(r"cycle_(?P<cycle_number>\d{3})"),
+            re.compile(r"^cycle_(?P<cycle_number>\d{3})$"),
             [FileNameConventionSwotL2().get_field("cycle_number")],
             "cycle_{cycle_number:0>3d}",
         ),
@@ -572,7 +569,7 @@ class BasicNetcdfFilesDatabaseSwotLRL2(FilesDatabase, PeriodMixin):
     sort_keys = "time"
 
     # These keys determines an homogeneous subset
-    unmixer = SubsetsUnmixer(partition_keys=["level", "subset"])
+    unmixer = SubsetsUnmixer(partition_keys=["subset"])
     # We expect multiple versions in an homogeneous subset. Only one half orbit
     # record is tolerated so we deduplicate the multiple version with an
     # autopick

tests/core/test_listing.py

@@ -49,6 +49,14 @@ def test_file_node_no_children():
     assert len(node.children()) == 0
 
 
+def test_dir_node_clear():
+    node = DirNode("foo", {"name": "foo"}, MemoryFileSystem(), 0)
+    node.children()
+    assert node._children is not None
+    node.clear()
+    assert node._children is None
+
+
 @pytest.fixture(scope="session")
 def filepaths() -> list[str]:
     # The files that we will use to test the listing and filtering using the

tests/implementations/collections/test_l2_lr_ssh.py

@@ -678,3 +678,29 @@ def test_list_swot_lr_l2_layout(
         assert set(map(tuple, actual.to_numpy())) == set(
             map(tuple, expected.to_numpy())
         )
+
+
+def test_subsets(l2_lr_ssh_dir_empty_files_layout: Path):
+
+    expected = [
+        {"subset": ProductSubset.Basic},
+        {"subset": ProductSubset.Unsmoothed},
+        {"subset": ProductSubset.Expert},
+    ]
+
+    db = NetcdfFilesDatabaseSwotLRL2(l2_lr_ssh_dir_empty_files_layout)
+    assert len(db.subsets) == len(expected)
+    assert all([x in expected for x in db.subsets])
+
+
+def test_subsets_flat(l2_lr_ssh_dir_empty_files: Path):
+
+    expected = [
+        {"subset": ProductSubset.Basic},
+        {"subset": ProductSubset.Unsmoothed},
+        {"subset": ProductSubset.Expert},
+    ]
+
+    db = NetcdfFilesDatabaseSwotLRL2(l2_lr_ssh_dir_empty_files)
+    assert len(db.subsets) == len(expected)
+    assert all([x in expected for x in db.subsets])

tests/implementations/test_docstrings.py

@@ -43,7 +43,6 @@ def sst_db(tmpdir, scope="module"):
                 "cycle_number",
                 "pass_number",
                 "time",
-                "level",
                 "subset",
             ],
         ),
@@ -86,7 +85,6 @@ def test_list_files_docstring(db, keywords, request):
                 "cycle_number",
                 "pass_number",
                 "time",
-                "level",
                 "subset",
                 "right_swath",
             ],
@@ -135,7 +133,7 @@ def test_query_docstring(db, keywords, request):
 @pytest.mark.parametrize(
     "db, keywords",
     [
-        ("swot_lr_l2_db", ["metadata describing the variables", "level", "subset"]),
+        ("swot_lr_l2_db", ["metadata describing the variables", "subset"]),
         ("swot_lr_l3_db", ["metadata describing the variables"]),
         ("nadir_db", ["metadata describing the variables", "resolution", "sensor"]),
         ("sst_db", ["metadata describing the variables"]),