add tests for reading of legacy files (closes #92)

.github/workflows/release.yaml

@@ -6,8 +6,14 @@ on:
       - "v?[0-9]+.[0-9]+.[0-9]+**"
   workflow_dispatch:
 
-# Use "trusted publishing", see https://docs.pypi.org/trusted-publishers/
+permissions:
+  contents: read
+
+env:
+  ref_name: ${{ github.ref_name }}
+
 jobs:
+  # Use "trusted publishing", see https://docs.pypi.org/trusted-publishers/
   release:
     name: Upload release to PyPI
     runs-on: ubuntu-latest
@@ -30,3 +36,89 @@ jobs:
         run: uv build
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
+
+  is-prerelease:
+    name: Check if this is a pre-release
+    runs-on: ubuntu-slim
+    outputs:
+      is_prerelease: ${{ steps.check_prerelease.outputs.is_prerelease }}
+    steps:
+      - name: Check if this is a pre-release
+        id: check_prerelease
+        run: |
+          PRE=$(echo "$ref_name" | sed -E 's/^v[0-9]+\.[0-9]+\.[0-9]+(.+?)$/\1/' | wc -w)
+          if [ $PRE = 0 ] ; then
+              PRE=false
+          else
+              PRE=true
+          fi
+          echo "is_prerelease=$PRE" | tee $GITHUB_OUTPUT
+
+  generate-test-data:
+    name: Generate test files for backwards compatibility testing
+    needs: is-prerelease
+    if: ${{ needs.is-prerelease.outputs.is_prerelease == 'false' && github.event_name == 'push' && github.event.created && github.ref_type == 'tag' }}
+    runs-on: ubuntu-slim
+    env:
+      data_path: ./tests/data/archives/${{ github.ref_name }}
+      default_branch: ${{ github.event.repository.default_branch }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          filter: blob:none
+          fetch-depth: 0
+          persist-credentials: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: false
+      - name: Get environment
+        # get the test environment with the oldest Python
+        id: get-env
+        run: |
+          ENV=$(uvx hatch env show --json | jq -r '[to_entries[] | select(.key | startswith("hatch-test") and (contains("pre") | not))]
+            | sort_by(.value.python)[0].key')
+          echo "hatch_env=${ENV}" | tee $GITHUB_ENV
+      - name: create hatch environment
+        run: uvx hatch env create "$hatch_env"
+      - name: generate output directory
+        run: mkdir -p "$data_path"
+      - name: generate files
+        working-directory: ${{ env.data_path }}
+        run: |
+          uvx hatch run "$hatch_env":../../../../src/mudata/tests/reference.py
+          uvx hatch run "$hatch_env":sh -c "uv pip freeze | uv pip compile - -o pylock.toml"
+      - name: configure git
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          BRANCH=$(git branch --show-current)
+          if [ ! "$BRANCH" ] ; then
+              BRANCH=$(git branch -a --contains "$ref_name" | tail -n 1 | sed 's/^\s*remotes\/origin\///')
+          fi
+          echo branch=${BRANCH} | tee $GITHUB_ENV
+      - name: commit and push
+        if: ${{ env.branch != '' }}
+        working-directory: ${{ env.data_path }}
+        run: |
+          git checkout "$branch"
+          git add .
+          git commit -m "add backwards compat test files for $ref_name"
+          git push
+          if [ "$branch" != "$default_branch" ] ; then
+              git checkout "$default_branch"
+              git pull
+              git cherry-pick -x "$branch"
+              git push
+          fi
+      - name: commit and push
+        if: ${{ env.branch == '' }}
+        working-directory: ${{ env.data_path }}
+        run: |
+          git checkout "$default_branch"
+          git pull
+          git add .
+          git commit -m "add backwards compat test files for $ref_name"
+          git push

pyproject.toml

@@ -48,7 +48,15 @@ dev = [
   "pre-commit",
   "twine>=4.0.2",
 ]
-test = [ "coverage>=7.10", "jsonschema", "mudata[io]", "packaging", "pytest", "referencing" ]
+test = [
+  "coverage>=7.10",
+  "fast-array-utils>=1.2.3", # for anndata.tests.helpers
+  "jsonschema",
+  "mudata[io]",
+  "packaging",
+  "pytest",
+  "referencing"
+]
 doc = [
   "anndata>=0.13",
   "docutils>=0.8,!=0.18.*,!=0.19.*",
@@ -139,6 +147,10 @@ filterwarnings = [
   "ignore:Writing zarr v2 data will no longer be the default",
   "ignore::scipy.sparse.SparseEfficiencyWarning",
 ]
+markers = [
+  "array_api: required to import anndata.tests.helpers.assert_equal",
+  "gpu: required to import anndata.tests.helpers.assert_equal",
+]
 strict = true
 testpaths = [ "tests" ]
 

src/mudata/_core/utils.py

@@ -123,7 +123,7 @@ def try_convert_series_to_numpy_dtype(col: pd.Series) -> pd.Series:
                 col = col.astype(bool)
             case pd.core.arrays.integer.IntegerDtype(type=dtype) | pd.core.arrays.floating.FloatingDtype(type=dtype):
                 col = col.astype(dtype)
-            case pd.StringDtype():
+            case pd.StringDtype() if pd.__version__ < "3":
                 col = col.astype(object)
     return col
 

src/mudata/tests/reference.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+import zarr
+from anndata import AnnData
+
+from mudata import MuData, write_h5mu, write_zarr
+
+
+def generate_reference(axis: Literal[0, 1]) -> MuData:
+    """Generate a reference MuData object to test backwards compatibility."""
+    mod1 = AnnData(
+        np.arange(0, 200, 0.1, dtype=np.float32).reshape(-1, 20), obs=pd.DataFrame(index=[str(i) for i in range(100)])
+    )
+    mod2 = AnnData(
+        np.arange(101, 3101, 1, dtype=np.float32).reshape(-1, 30),
+        obs=pd.DataFrame(index=[str(i) for i in range(99, -1, -1)]),
+    )
+    mod1.var["assert-bool"] = True
+    mod2.var["assert-bool"] = False
+    mod1.var["assert-boolean-1"] = True
+    mod2.var["assert-boolean-2"] = False
+
+    mod1.raw = mod1[:, :10].copy()
+    mods = {"mod2": mod2, "mod1": mod1}
+
+    attr = "obs" if axis == 0 else "var"
+    oattr = "var" if axis == 0 else "obs"
+    for modname, mod in mods.items():
+        mod.var["feature_type"] = modname
+        setattr(mod, f"{attr}_names", [f"{attr}_{i}" for i in range(mod.shape[axis])])
+        setattr(mod, f"{oattr}_names", [f"{modname}_{oattr}_{i}" for i in range(mod.shape[1 - axis])])
+    mod1.obs.index.name = "fizz"
+    mod2.var.index.name = "buzz"
+    mdata = MuData(mods, axis=axis)
+    mdata.obs["arange"] = np.arange(mdata.n_obs)
+    return mdata
+
+
+if __name__ == "__main__":
+    for axis in (0, 1):
+        mdata = generate_reference(axis)
+        write_h5mu(f"mdata_axis{axis}.h5mu", mdata, compression="gzip", compression_opts=9)
+        write_zarr(zarr.storage.ZipStore(f"mdata_axis{axis}.zarr.zip", mode="w"), mdata)

tests/data/archives/v0.1.2/pylock.toml

@@ -0,0 +1,112 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile - -o pylock.toml
+lock-version = "1.0"
+created-by = "uv"
+requires-python = ">=3.10.19"
+
+[[packages]]
+name = "anndata"
+version = "0.7.8"
+sdist = { url = "https://files.pythonhosted.org/packages/01/51/bad4a0a259b4b2e350b5b68a53e698b5150d38ffad18f541ab4bc3940df5/anndata-0.7.8.tar.gz", upload-time = 2021-11-09T20:55:20Z, size = 80217, hashes = { sha256 = "1efd7eb40839e0325bb066238280228a980d7dde6410793dbff2835f44a2d3ef" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/76/dc/cec2e749efbb16e77675865518b517ab3f62d00573e36671e4240aa481b7/anndata-0.7.8-py3-none-any.whl", upload-time = 2021-11-09T20:55:19Z, size = 91056, hashes = { sha256 = "cc098d46662230f91e421c707590337c3e16459c494f28d80b8ff5baae54e539" } }]
+
+[[packages]]
+name = "asciitree"
+version = "0.3.3"
+sdist = { url = "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz", upload-time = 2016-09-05T19:10:42Z, size = 3951, hashes = { sha256 = "4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e" } }
+
+[[packages]]
+name = "fasteners"
+version = "0.17.3"
+sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/148f44998c1bdb064a508e7cbcf9e50b34572b3d36fcc378a5d61b7dc8c5/fasteners-0.17.3.tar.gz", upload-time = 2022-01-25T14:54:24Z, size = 18014, hashes = { sha256 = "a9a42a208573d4074c77d041447336cf4e3c1389a256fd3e113ef59cf29b7980" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/f6/01/274da83334c20dc1ae7a48b1ea4ae50d3571d4e6aea65bb0368f841701ad/fasteners-0.17.3-py3-none-any.whl", upload-time = 2022-01-25T14:54:23Z, size = 18441, hashes = { sha256 = "cae0772df265923e71435cc5057840138f4e8b6302f888a567d06ed8e1cbca03" } }]
+
+[[packages]]
+name = "h5py"
+version = "3.7.0"
+sdist = { url = "https://files.pythonhosted.org/packages/c5/40/7cf58e6230f0e76699f011c6d293dd47755997709a303a4e644823f3a753/h5py-3.7.0.tar.gz", upload-time = 2022-05-24T09:59:22Z, size = 392355, hashes = { sha256 = "3fcf37884383c5da64846ab510190720027dca0768def34dd8dcb659dbe5cbf3" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/54/a4/974b666f64c339e5b0b44fc782edee92ab3341c17367a040e41f5132c15b/h5py-3.7.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", upload-time = 2022-05-24T09:57:11Z, size = 4463873, hashes = { sha256 = "04e2e1e2fc51b8873e972a08d2f89625ef999b1f2d276199011af57bb9fc7851" } }]
+
+[[packages]]
+name = "mudata"
+version = "0.1.2"
+sdist = { url = "https://files.pythonhosted.org/packages/b7/7b/baab54fb96ee59d69ee514b2a2d3ebca5db6d3453951e13cc5b93eff091e/mudata-0.1.2.tar.gz", upload-time = 2022-05-11T17:00:01Z, size = 21011, hashes = { sha256 = "4636831e20d75cc47aa1eab14143d80399126a9bf526412e9d3e9ce17e8378aa" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/78/b5/8f9cfcdc842e7b9ec50ae35f4e1b7e6ae88614d376b9f6b9c4f3371ae63c/mudata-0.1.2-py3-none-any.whl", upload-time = 2022-05-11T16:59:59Z, size = 23331, hashes = { sha256 = "d7dcd046398193405e407ef3f8158ffb934e916db4c346d7792623f51ff15dfd" } }]
+
+[[packages]]
+name = "natsort"
+version = "8.1.0"
+sdist = { url = "https://files.pythonhosted.org/packages/f7/37/207acdf07c2229a799b7a042c0977ad2372f4adb3446fba8f7703e2840e1/natsort-8.1.0.tar.gz", upload-time = 2022-01-31T03:20:44Z, size = 145241, hashes = { sha256 = "c7c1f3f27c375719a4dfcab353909fe39f26c2032a062a8c80cc844eaaca0445" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/a9/76/0f624b7326f4458a249580c55e5654756084ec4572ce37a05f799b96bc24/natsort-8.1.0-py3-none-any.whl", upload-time = 2022-01-31T03:20:42Z, size = 37325, hashes = { sha256 = "f59988d2f24e77b6b56f8a8f882d5df6b3b637e09e075abc67b486d59fba1a4b" } }]
+
+[[packages]]
+name = "numcodecs"
+version = "0.10.0"
+sdist = { url = "https://files.pythonhosted.org/packages/09/a9/7e6eb6c1465121b80bbb9dd3599a01ac6ece3fc446ab1290bda25730b6c5/numcodecs-0.10.0.tar.gz", upload-time = 2022-06-23T09:10:38Z, size = 4519035, hashes = { sha256 = "2dd42564e7772a9385923b02a363e3cedf053ce93094a9064485e7f69a6f1c92" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/55/a0/bd5feb893542b431b99bfca1200d3063558e244c8cc8ebd95f8dce3b73fc/numcodecs-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2022-06-23T09:10:15Z, size = 6685393, hashes = { sha256 = "9c9f1cc74c999006fd913d636436edbddd2f0b2115b7633cf97f9ba298743dca" } }]
+
+[[packages]]
+name = "numpy"
+version = "1.23.0"
+sdist = { url = "https://files.pythonhosted.org/packages/03/c6/14a17e10813b8db20d1e800ff9a3a898e65d25f2b0e9d6a94616f1e3362c/numpy-1.23.0.tar.gz", upload-time = 2022-06-22T22:26:20Z, size = 10714532, hashes = { sha256 = "bd3fa4fe2e38533d5336e1272fc4e765cabbbde144309ccee8675509d5cd7b05" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/7a/88/8404fbe4f6472e4e54106a8faacae1279a244422bc88f5ee3e33ba2dd72b/numpy-1.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2022-06-22T22:14:41Z, size = 17033638, hashes = { sha256 = "d54b3b828d618a19779a84c3ad952e96e2c2311b16384e973e671aa5be1f6187" } }]
+
+[[packages]]
+name = "packaging"
+version = "21.3"
+sdist = { url = "https://files.pythonhosted.org/packages/df/9e/d1a7217f69310c1db8fdf8ab396229f55a699ce34a203691794c5d1cad0c/packaging-21.3.tar.gz", upload-time = 2021-11-18T00:39:13Z, size = 84848, hashes = { sha256 = "dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", upload-time = 2021-11-18T00:39:10Z, size = 40750, hashes = { sha256 = "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" } }]
+
+[[packages]]
+name = "pandas"
+version = "1.4.3"
+sdist = { url = "https://files.pythonhosted.org/packages/f4/00/2de395c769335956b8650f990ef2a15e860be83b544c408ff95713446329/pandas-1.4.3.tar.gz", upload-time = 2022-06-23T13:31:50Z, size = 4941520, hashes = { sha256 = "2ff7788468e75917574f080cd4681b27e1a7bf36461fe968b49a87b5a54d007c" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/ea/e4/a1cbaca4069fdd92c930bb1c5eebd9ea9c55717a9bf60bd41708c8a33f5a/pandas-1.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2022-06-23T13:30:31Z, size = 11593339, hashes = { sha256 = "6f803320c9da732cc79210d7e8cc5c8019aad512589c910c66529eb1b1818230" } }]
+
+[[packages]]
+name = "pyparsing"
+version = "3.0.9"
+sdist = { url = "https://files.pythonhosted.org/packages/71/22/207523d16464c40a0310d2d4d8926daffa00ac1f5b1576170a32db749636/pyparsing-3.0.9.tar.gz", upload-time = 2022-05-10T23:26:05Z, size = 1999906, hashes = { sha256 = "2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/6c/10/a7d0fa5baea8fe7b50f448ab742f26f52b80bfca85ac2be9d35cdd9a3246/pyparsing-3.0.9-py3-none-any.whl", upload-time = 2022-05-10T23:26:03Z, size = 98338, hashes = { sha256 = "5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" } }]
+
+[[packages]]
+name = "python-dateutil"
+version = "2.8.2"
+sdist = { url = "https://files.pythonhosted.org/packages/4c/c4/13b4776ea2d76c115c1d1b84579f3764ee6d57204f6be27119f13a61d0a9/python-dateutil-2.8.2.tar.gz", upload-time = 2021-07-14T08:19:19Z, size = 357324, hashes = { sha256 = "0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl", upload-time = 2021-07-14T08:19:18Z, size = 247702, hashes = { sha256 = "961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" } }]
+
+[[packages]]
+name = "pytz"
+version = "2022.1"
+sdist = { url = "https://files.pythonhosted.org/packages/2f/5f/a0f653311adff905bbcaa6d3dfaf97edcf4d26138393c6ccd37a484851fb/pytz-2022.1.tar.gz", upload-time = 2022-03-20T00:37:10Z, size = 320473, hashes = { sha256 = "1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/60/2e/dec1cc18c51b8df33c7c4d0a321b084cf38e1733b98f9d15018880fb4970/pytz-2022.1-py2.py3-none-any.whl", upload-time = 2022-03-20T00:37:06Z, size = 503520, hashes = { sha256 = "e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" } }]
+
+[[packages]]
+name = "scipy"
+version = "1.8.1"
+sdist = { url = "https://files.pythonhosted.org/packages/26/b5/9330f004b9a3b2b6a31f59f46f1617ce9ca15c0e7fe64288c20385a05c9d/scipy-1.8.1.tar.gz", upload-time = 2022-05-18T12:47:44Z, size = 38196215, hashes = { sha256 = "9e3fb1b0e896f14a85aa9a28d5f755daaeeb54c897b746df7a55ccb02b340f33" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/bc/fe/72b611ba221c3367b06163992af4807515d6e0e09b3b9beee8ec22162d6f/scipy-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2022-05-18T12:30:13Z, size = 42180853, hashes = { sha256 = "a0aa8220b89b2e3748a2836fbfa116194378910f1a6e78e4675a095bcd2c762d" } }]
+
+[[packages]]
+name = "six"
+version = "1.16.0"
+sdist = { url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", upload-time = 2021-05-05T14:18:18Z, size = 34041, hashes = { sha256 = "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", upload-time = 2021-05-05T14:18:17Z, size = 11053, hashes = { sha256 = "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" } }]
+
+[[packages]]
+name = "typing-extensions"
+version = "4.3.0"
+sdist = { url = "https://files.pythonhosted.org/packages/9e/1d/d128169ff58c501059330f1ad96ed62b79114a2eb30b8238af63a2e27f70/typing_extensions-4.3.0.tar.gz", upload-time = 2022-07-01T14:56:14Z, size = 47430, hashes = { sha256 = "e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/ed/d6/2afc375a8d55b8be879d6b4986d4f69f01115e795e36827fd3a40166028b/typing_extensions-4.3.0-py3-none-any.whl", upload-time = 2022-07-01T14:56:11Z, size = 25596, hashes = { sha256 = "25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02" } }]
+
+[[packages]]
+name = "xlrd"
+version = "1.2.0"
+sdist = { url = "https://files.pythonhosted.org/packages/aa/05/ec9d4fcbbb74bbf4da9f622b3b61aec541e4eccf31d3c60c5422ec027ce2/xlrd-1.2.0.tar.gz", upload-time = 2018-12-15T17:47:48Z, size = 554079, hashes = { sha256 = "546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/b0/16/63576a1a001752e34bf8ea62e367997530dc553b689356b9879339cf45a4/xlrd-1.2.0-py2.py3-none-any.whl", upload-time = 2018-12-15T17:47:45Z, size = 103251, hashes = { sha256 = "e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" } }]
+
+[[packages]]
+name = "zarr"
+version = "2.12.0"
+sdist = { url = "https://files.pythonhosted.org/packages/51/8b/2480d3758e5e70765c2c9f3c13ca8fb576f1ad73c31f6a696d6720b7f562/zarr-2.12.0.tar.gz", upload-time = 2022-06-23T08:47:33Z, size = 3571172, hashes = { sha256 = "515a31ee4bad6bb48ae05178c588ae49a02a35defa01dd9a3293306903368165" } }
+wheels = [{ url = "https://files.pythonhosted.org/packages/56/a6/817d25f7b77b695b1014030bfff90288846862a50d72c1ee85a8a233c040/zarr-2.12.0-py3-none-any.whl", upload-time = 2022-06-23T08:47:31Z, size = 185751, hashes = { sha256 = "3713f28f3fafa838f62bb442504da8e89bb969ecd1da09c9a96927536ec9be34" } }]