Merge pull request #6 from e-sensing/arrow

backend: enhance arrow compatibility libs

Author: M3nin0

.github/workflows/tests.yaml

@@ -0,0 +1,96 @@
+name: Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      reason:
+        required: false
+        description: 'Reason'
+        default: 'Manual trigger'
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      # Report every version, so a single break does not hide the others.
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
+
+    env:
+      # Fail the build on R warnings that would otherwise pass unnoticed
+      R_KEEP_PKG_SOURCE: yes
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up R
+        uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - name: Install R dependencies
+        uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          # `arrow` and `kohonen` are required by pysits but are not part of a
+          # default `install.packages("sits")`, so they are listed explicitly.
+          packages: |
+            any::sits
+            any::arrow
+            any::kohonen
+            any::jsonlite
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Create a clean virtual environment
+        # Mirrors how users install pysits, so a broken dependency resolution
+        # is caught here rather than by them.
+        run: python -m venv .venv
+
+      - name: Install pysits
+        # `--only-binary` makes pip fail instead of falling back to a source
+        # build when no wheel matches this interpreter. Those builds have
+        # shipped binaries that segfault. Only pysits itself is built here.
+        run: |
+          .venv/bin/python -m pip install --upgrade pip
+          .venv/bin/python -m pip install \
+            --only-binary=:all: --no-binary pysits \
+            ".[dev]"
+
+      - name: Report resolved dependency versions
+        run: .venv/bin/python -m pip list
+
+      - name: Verify Arrow interoperability between R and Python
+        # pyarrow and R arrow package each bundle libarrow. If they stop
+        # sharing an allocator, transfers are silently corrupted rather than
+        # failing, so this asserts a known value survives a round-trip.
+        run: |
+          .venv/bin/python - <<'EOF'
+          import pyarrow as pa
+
+          import pysits
+          from pysits.conversions.tibble import (
+              pandas_sits_to_tibble,
+              tibble_sits_to_pandas,
+          )
+
+          probe = [0.5, 1.5, 2.5]
+          assert pa.array(probe).to_pylist() == probe, "pyarrow returns corrupted data"
+
+          samples = pysits.samples_modis_ndvi
+          expected = samples["time_series"][0]["NDVI"].tolist()
+          actual = tibble_sits_to_pandas(pandas_sits_to_tibble(samples.head(1)))
+          actual = actual["time_series"][0]["NDVI"].tolist()
+
+          assert actual == expected, f"round-trip corrupted data: {actual} != {expected}"
+          print("Arrow round-trip is lossless.")
+          EOF
+
+      - name: Run tests
+        run: .venv/bin/python -m pytest

pyproject.toml

@@ -58,8 +58,13 @@ classifiers = [
 ]
 
 dependencies = [
-    "rpy2>=3.5.17,<4.0.0",
+    # 3.6.0 is the first release distributing wheels. Earlier ones are
+    # source-only, so pip would compile rpy2 against the local R installation
+    "rpy2>=3.6.0,<4.0.0",
     "pandas>=2.2.3",
+    # pandas ships no wheels for Python 3.14 before 2.3.3. Without this floor
+    # pip builds pandas from source there, and the result segfaults.
+    "pandas>=2.3.3; python_version>='3.14'",
     "pillow>=11.1.0",
     "matplotlib>=3.10.1",
     "geopandas>=1.1.0",
@@ -90,6 +95,9 @@ dev = [
     "pytest-cov>=6.1.1",
     "pytest-randomly>=3.16.0",
     "cloudpickle>=3.1.1",
+    # The test suite covers the xarray exporters and reads rasters directly
+    "pysits[xarray]",
+    "rasterio>=1.3.0",
 ]
 
 [tool.ruff]

pysits/__init__.py

@@ -17,6 +17,10 @@
 
 """pysits module."""
 
+# Importing `backend.arrow` first selects the libarrow allocator, which must
+# happen before anything imports `pyarrow` or loads the R `arrow` package.
+# Keep this import above the others. See `pysits.backend.arrow`.
+from .backend import arrow as _arrow  # noqa: F401
 from .conversions.dsl.mask import MaskValue
 from .conversions.dsl.tuning import hparam
 from .settings import __version__

pysits/backend/arrow.py

@@ -0,0 +1,106 @@
+#
+# Copyright (C) 2025 sits developers.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Arrow interoperability between R and Python.
+
+``pysits`` moves data between R and Python as Arrow IPC streams. That puts two
+independent builds of the libarrow C++ library in a single process: the one
+bundled in the R ``arrow`` package and the one bundled in ``pyarrow``. Their
+symbols collapse into a single namespace, so a call made through ``pyarrow``
+can be served by R's libarrow.
+
+Each build keeps a private ``mimalloc`` heap. When the two are mixed, a buffer
+allocated by one library is read back as zeroed memory by the other, and every
+transfer is silently corrupted instead of failing. Both builds also support the
+system allocator, which is shared, so selecting it keeps them interoperable.
+
+The allocator is chosen by libarrow the first time it initializes, so importing
+this module sets ``ARROW_DEFAULT_MEMORY_POOL``. It is imported before anything
+that pulls in ``pyarrow`` or the R ``arrow`` package.
+"""
+
+import os
+
+#
+# Allocator selected to keep R libarrow and pyarrow interoperable
+#
+ARROW_MEMORY_POOL_ENVVAR = "ARROW_DEFAULT_MEMORY_POOL"
+ARROW_MEMORY_POOL = "system"
+
+# Set on import: libarrow reads this once, when it initializes. A value already
+# present in the environment is left alone so users can override it.
+os.environ.setdefault(ARROW_MEMORY_POOL_ENVVAR, ARROW_MEMORY_POOL)
+
+#
+# Values transferred to R and back to verify the round-trip is lossless. A
+# mismatched pair of libarrow builds reads them back as zeros.
+#
+PROBE_COLUMN = "pysits_arrow_probe"
+PROBE_VALUES = [0.5, 1.5, 2.5]
+
+
+def check_arrow_memory_pool() -> None:
+    """Refuse to load R ``arrow`` package under an unshared allocator.
+
+    Called before the R ``arrow`` package is loaded. Corruption appears only
+    once both libarrow builds are in use, and is silent when it does, so the
+    configuration is rejected up front rather than probed for afterwards.
+
+    Raises:
+        RuntimeError: If the selected allocator is not the shared one.
+    """
+    selected = os.environ.get(ARROW_MEMORY_POOL_ENVVAR)
+
+    if selected == ARROW_MEMORY_POOL:
+        return
+
+    raise RuntimeError(
+        f"{ARROW_MEMORY_POOL_ENVVAR} is set to '{selected}', but pysits "
+        f"requires '{ARROW_MEMORY_POOL}'.\n\n"
+        "The R `arrow` package and `pyarrow` each bundle their own build of "
+        "the libarrow C++ library. Loaded together, they must use the system "
+        "allocator to share buffers. With any other allocator, data sent "
+        "between R and Python is silently replaced by zeros.\n\n"
+        f"Unset {ARROW_MEMORY_POOL_ENVVAR}, or set it before starting Python:\n\n"
+        f"    export {ARROW_MEMORY_POOL_ENVVAR}={ARROW_MEMORY_POOL}"
+    )
+
+
+def arrow_interop_error(observed: list) -> RuntimeError:
+    """Build the error raised when an Arrow round-trip loses data.
+
+    Args:
+        observed (list): Values read back from R for `PROBE_VALUES`.
+
+    Returns:
+        RuntimeError: Error describing the cause and how to resolve it.
+    """
+    import pyarrow as pa
+
+    pool = pa.default_memory_pool().backend_name
+
+    return RuntimeError(
+        "Data sent to R is coming back corrupted, so pysits cannot run.\n\n"
+        f"Sent {PROBE_VALUES}, received {observed}.\n\n"
+        "The R `arrow` package and `pyarrow` each bundle their own build of "
+        "the libarrow C++ library. Loaded together, they must use the system "
+        f"allocator to share buffers, but pyarrow is using '{pool}'.\n\n"
+        "This happens when pyarrow is initialized before pysits with a "
+        "different allocator. Either import pysits before pyarrow, or set the "
+        "environment variable before starting Python:\n\n"
+        f"    export {ARROW_MEMORY_POOL_ENVVAR}={ARROW_MEMORY_POOL}"
+    )

pysits/backend/pkgs.py

@@ -17,6 +17,7 @@
 
 """backend packages."""
 
+from pysits.backend.arrow import check_arrow_memory_pool
 from pysits.backend.loaders import load_package
 from pysits.settings import __sitsver__
 
@@ -33,4 +34,9 @@
 r_pkg_kohonen = load_package("kohonen")
 r_pkg_sf = load_package("sf")
 r_pkg_htmlwidgets = load_package("htmlwidgets")
+
+# `arrow` brings a second libarrow build into the process, so the allocator has
+# to be compatible before it is loaded. See `pysits.backend.arrow`.
+check_arrow_memory_pool()
+
 r_pkg_arrow = load_package("arrow")

pysits/conversions/tibble.py

@@ -36,6 +36,7 @@
 from rpy2.robjects.vectors import DataFrame as RDataFrame
 from shapely import wkt
 
+from pysits.backend.arrow import PROBE_COLUMN, PROBE_VALUES, arrow_interop_error
 from pysits.backend.functions import r_fnc_class, r_fnc_set_column
 from pysits.backend.pkgs import r_pkg_base, r_pkg_sf, r_pkg_sits
 from pysits.models.frame import NestedFrame
@@ -90,6 +91,39 @@ def _sf_to_shapely(sf_object: RDataFrame) -> list:
 #
 # Arrow IPC helpers
 #
+def _check_arrow_interop() -> None:
+    """Verify that Arrow data is not corrupted once R's libarrow is loaded.
+
+    R libarrow only initializes on first use, so a probe is sent through R to
+    force it, and ``pyarrow`` is then re-tested. A mismatched pair of libarrow
+    builds corrupts the conversion of Python objects to Arrow arrays, returning
+    zeros instead of raising, which would silently damage every nested column.
+
+    Raises:
+        RuntimeError: If either transfer loses data.
+    """
+    probe = PandasDataFrame({PROBE_COLUMN: PROBE_VALUES})
+
+    # Forces R libarrow to initialize, and checks the Python -> R direction
+    r_probe = rpy2_globalenv["pysits_read_ipc_raw"](
+        robjects.vectors.ByteVector(_dataframe_to_ipc_bytes(probe))
+    )
+
+    # Load content from R
+    observed = list(r_probe.rx2(PROBE_COLUMN))
+
+    # If it is different, then given an error
+    if observed != PROBE_VALUES:
+        raise arrow_interop_error(observed)
+
+    # Both libarrow builds are live now: check the conversion of Python objects
+    # to Arrow arrays, which is the path nested columns depend on.
+    observed = pa.array(PROBE_VALUES).to_pylist()
+
+    if observed != PROBE_VALUES:
+        raise arrow_interop_error(observed)
+
+
 def _ensure_r_ipc_functions():
     """Define R-side IPC reader/writer/unnester functions once."""
     if "pysits_write_ipc_raw" in rpy2_globalenv:
@@ -162,6 +196,8 @@ def _ensure_r_ipc_functions():
         }
     """)
 
+    _check_arrow_interop()
+
 
 def _dataframe_to_ipc_bytes(df: PandasDataFrame) -> bytes:
     """Serialize a pandas DataFrame to Arrow IPC bytes.