refactor: adapt code to work with pandas 3

Author: robin-cls

.github/workflows/doc.yaml

@@ -52,7 +52,7 @@ jobs:
             beautifulsoup4
             shapely
             geopandas
-            pyinterp
+            pyinterp<2026.2.0
             dask
       - name: Install package (for autodoc)
         shell: bash -l {0}

.github/workflows/tests.yaml

@@ -40,7 +40,7 @@ jobs:
             pyftpdlib
             shapely
             geopandas
-            pyinterp
+            pyinterp<2026.2.0
             dask
       - name: Install python dependencies
         shell: bash -el {0}
@@ -51,7 +51,7 @@ jobs:
       - name: Run unit tests
         shell: bash -el {0}
         run: |
-          pytest tests --cov=src --cov-report=xml:coverage-ut.xml --cov-report=term
+          pytest tests --cov=src/fcollections --cov-report=xml:coverage-ut.xml --cov-report=term
           mv .coverage .coverage-ut
       - name: Upload unit test coverage artifacts
         uses: actions/upload-artifact@v4
@@ -94,7 +94,7 @@ jobs:
             pyftpdlib
             shapely
             geopandas
-            pyinterp
+            pyinterp<2026.2.0
             dask
       - name: Install python dependencies
         shell: bash -el {0}
@@ -104,7 +104,7 @@ jobs:
       - name: Run unit tests without geo
         shell: bash -el {0}
         run: |
-          pytest tests/implementations/collections --ignore tests/implementations/collections/test_area_selector.py --without-geo-packages --cov=src --cov-report=xml:coverage-ut-nogeo.xml --cov-report=term
+          pytest tests/implementations/collections --ignore tests/implementations/collections/test_area_selector.py --without-geo-packages --cov=src/fcollections --cov-report=xml:coverage-ut-nogeo.xml --cov-report=term
           mv .coverage .coverage-ut-nogeo
       - name: Upload unit test no-geo coverage artifacts
         uses: actions/upload-artifact@v4
@@ -129,7 +129,7 @@ jobs:
           pip install -e ".[testing]"
       - name: Run integration tests
         run: |
-          pytest integration_tests --cov=src --cov-report=xml:coverage-it.xml --cov-report=term
+          pytest integration_tests --cov=src/fcollections --cov-report=xml:coverage-it.xml --cov-report=term
           mv .coverage .coverage-it
       - name: Upload integration test coverage artifacts
         uses: actions/upload-artifact@v4

pyproject.toml

@@ -10,14 +10,15 @@ maintainers = [
     {name = "Robin Chevrier", email = "rchevrier@groupcls.com"},
     {name = "Anne-Sophie Tonneau", email = "atonneau@groupcls.com"}
 ]
-requires-python = ">= 3.8"
+requires-python = ">= 3.11"
 readme = "README.md"
 dependencies = [
     "docstring_parser",
     "fsspec",
     "jinja2",
     "netCDF4",
     "numpy",
+    "pandas>=3",
     "requests",
     "typer",
     "xarray[io]"]
@@ -48,7 +49,7 @@ Repository='https://github.com/robin-cls/fcollections'
 
 [project.optional-dependencies]
 testing = ["pytest", "pytest-cov", "sympy", "beautifulsoup4", "pyftpdlib", "pyasynchat"]
-geo = ["shapely", "geopandas", "pyinterp", "dask", "numba"]
+geo = ["shapely", "geopandas", "pyinterp<2026.2.0", "dask", "numba"]
 doc = ["cartopy", "matplotlib", "sphinx", "sphinx-copybutton", "sphinx-design",
         "sphinx-book-theme", "myst-nb"]
 

src/fcollections/core/_filesdb.py

@@ -690,7 +690,18 @@ def __call__(self, df: pda.DataFrame) -> pda.DataFrame:
             )
         except AttributeError:
             # We have a tuple
-            subsets = df.groupby(list(self.partition_keys))
+            grouping_keys = list(self.partition_keys)
+            subsets = df.groupby(
+                # In pandas 2, a list with one single element gave a scalar keys
+                # for the groups. In pandas 3, a future warning is raised,
+                # asking to give either a single key or a list with more than
+                # one key
+                grouping_keys if len(grouping_keys) > 1 else grouping_keys[0],
+                # Pandas 3 tries to sort the groups, which can raise an error if
+                # a column cannot be ordered. We don't need this sort so we
+                # disable it
+                sort=False,
+            )
 
         # Pick one subset using panda duplicate handling
         subset_names = [
@@ -719,11 +730,14 @@ def __call__(self, df: pda.DataFrame) -> pda.DataFrame:
                     for key in manual_pick
                     if len(df_subsets[key].unique()) > 1
                 }
-                raise ValueError(
-                    f"Subsets could not be unmixed, the following keys are duplicated and should be fixed manually: {ambiguity}"
+                msg = (
+                    "Subsets could not be unmixed, the following keys are "
+                    f"duplicated and should be fixed manually: {ambiguity}"
                 )
+                raise ValueError(msg)
 
         group_name = tuple(df_subsets.to_records(index=False)[-1])
+        group_name = group_name if len(group_name) > 1 else group_name[0]
         logger.debug("Subset selected %s", group_name)
         return subsets.get_group(group_name)
 

tests/core/test_filesdb.py

@@ -315,8 +315,8 @@ def test_metadata_wrong_filters(tmp_path: Path):
 def test_list_files(db_with_files: FilesDatabaseTest):
     expected = pda.DataFrame(
         [
-            (np.datetime64("2025-01-01"), 1, "/flat/a_file_001_20250101.nc"),
-            (np.datetime64("2025-01-01"), 2, "/flat/a_file_002_20250101.nc"),
+            (np.datetime64("2025-01-01", "us"), 1, "/flat/a_file_001_20250101.nc"),
+            (np.datetime64("2025-01-01", "us"), 2, "/flat/a_file_002_20250101.nc"),
         ],
         columns=["time", "a_number", "filename"],
     )
@@ -325,7 +325,7 @@ def test_list_files(db_with_files: FilesDatabaseTest):
 
 def test_list_files_filter(db_with_files: FilesDatabaseTest):
     expected = pda.DataFrame(
-        [(np.datetime64("2025-01-01"), 2, "/flat/a_file_002_20250101.nc")],
+        [(np.datetime64("2025-01-01", "us"), 2, "/flat/a_file_002_20250101.nc")],
         columns=["time", "a_number", "filename"],
     )
     assert expected.equals(db_with_files.list_files(a_number=2))
@@ -352,8 +352,8 @@ def test_list_files_predicate(
 ):
     expected = pda.DataFrame(
         [
-            (np.datetime64("2025-01-01"), 2, "/predicate/a_file_002_20250101.nc"),
-            (np.datetime64("2025-01-01"), 4, "/predicate/a_file_004_20250101.nc"),
+            (np.datetime64("2025-01-01", "us"), 2, "/predicate/a_file_002_20250101.nc"),
+            (np.datetime64("2025-01-01", "us"), 4, "/predicate/a_file_004_20250101.nc"),
         ],
         columns=["time", "a_number", "filename"],
     )