Merge pull request #155 from pyranges/fix-join-overlaps-nonzero-index

Fix join_overlaps row duplication with non-default index

Author: endrebak

CHANGELOG.txt

@@ -1,3 +1,7 @@
+# 1.3.9 (30.05.26)
+- fix `join_overlaps` duplicating rows in `left`/`outer` joins when the input index did not start at 0
+- preserve the input index in `join_overlaps` output (self for inner/left/outer, other for right), consistent with `overlap`/`nearest_ranges`
+
 # 1.3.8 (21.04.26)
 - repo name changed from pyranges_1.x to pyranges1
 - updated references to it

README.md

@@ -22,6 +22,10 @@ available at https://pyranges1.readthedocs.io/
 ## Recent Changelog
 
 ```
+# 1.3.9 (30.05.26)
+- fix `join_overlaps` duplicating rows in `left`/`outer` joins when the input index did not start at 0
+- preserve the input index in `join_overlaps` output (self for inner/left/outer, other for right), consistent with `overlap`/`nearest_ranges`
+
 # 1.3.8 (21.04.26)
 - repo name changed from pyranges_1.x to pyranges1
 - updated references to it

docs/how_to_map.rst

@@ -424,27 +424,27 @@ by pairing (overlapping) CDS and exon intervals with the same ``Parent`` value:
   index    |    Chromosome             Start    End      Strand    Feature
   int64    |    str                    int64    int64    str       category
   -------  ---  ---------------------  -------  -------  --------  ----------
-  0        |    rna-DGYR_LOCUS13733    1        382      +         CDS
-  1        |    rna-DGYR_LOCUS13734    258      484      +         CDS
-  2        |    rna-DGYR_LOCUS13734    484      625      +         CDS
-  3        |    rna-DGYR_LOCUS13734    625      798      +         CDS
+  4        |    rna-DGYR_LOCUS13733    1        382      +         CDS
+  11       |    rna-DGYR_LOCUS13734    258      484      +         CDS
+  12       |    rna-DGYR_LOCUS13734    484      625      +         CDS
+  13       |    rna-DGYR_LOCUS13734    625      798      +         CDS
   ...      |    ...                    ...      ...      ...       ...
-  52       |    rna-DGYR_LOCUS12552-2  139      237      +         CDS
-  53       |    rna-DGYR_LOCUS12552-2  237      337      +         CDS
-  54       |    rna-DGYR_LOCUS12552-2  337      520      +         CDS
-  55       |    rna-DGYR_LOCUS12552-2  520      640      +         CDS
+  146      |    rna-DGYR_LOCUS12552-2  139      237      +         CDS
+  147      |    rna-DGYR_LOCUS12552-2  237      337      +         CDS
+  148      |    rna-DGYR_LOCUS12552-2  337      520      +         CDS
+  149      |    rna-DGYR_LOCUS12552-2  520      640      +         CDS
   PyRanges with 56 rows, 5 columns, and 1 index columns.
   Contains 17 chromosomes and 1 strands.
 
   >>> cds_local[cds_local.Chromosome == some_id]
     index  |    Chromosome               Start      End  Strand    Feature
     int64  |    str                      int64    int64  str       category
   -------  ---  ---------------------  -------  -------  --------  ----------
-       51  |    rna-DGYR_LOCUS12552-2       97      139  +         CDS
-       52  |    rna-DGYR_LOCUS12552-2      139      237  +         CDS
-       53  |    rna-DGYR_LOCUS12552-2      237      337  +         CDS
-       54  |    rna-DGYR_LOCUS12552-2      337      520  +         CDS
-       55  |    rna-DGYR_LOCUS12552-2      520      640  +         CDS
+      145  |    rna-DGYR_LOCUS12552-2       97      139  +         CDS
+      146  |    rna-DGYR_LOCUS12552-2      139      237  +         CDS
+      147  |    rna-DGYR_LOCUS12552-2      237      337  +         CDS
+      148  |    rna-DGYR_LOCUS12552-2      337      520  +         CDS
+      149  |    rna-DGYR_LOCUS12552-2      520      640  +         CDS
   PyRanges with 5 rows, 5 columns, and 1 index columns.
   Contains 1 chromosomes and 1 strands.
 
@@ -454,30 +454,30 @@ With the similar logic, we can easily map the start and stop codon positions to
   index    |    Chromosome             Start    End      Strand    Feature
   int64    |    str                    int64    int64    str       str
   -------  ---  ---------------------  -------  -------  --------  ---------
-  0        |    rna-DGYR_LOCUS13733    1        4        +         start
-  1        |    rna-DGYR_LOCUS13734    258      261      +         start
-  2        |    rna-DGYR_LOCUS13734    484      487      +         start
-  3        |    rna-DGYR_LOCUS13734    625      628      +         start
+  4        |    rna-DGYR_LOCUS13733    1        4        +         start
+  11       |    rna-DGYR_LOCUS13734    258      261      +         start
+  12       |    rna-DGYR_LOCUS13734    484      487      +         start
+  13       |    rna-DGYR_LOCUS13734    625      628      +         start
   ...      |    ...                    ...      ...      ...       ...
-  52       |    rna-DGYR_LOCUS12552-2  139      142      +         start
-  53       |    rna-DGYR_LOCUS12552-2  237      240      +         start
-  54       |    rna-DGYR_LOCUS12552-2  337      340      +         start
-  55       |    rna-DGYR_LOCUS12552-2  520      523      +         start
+  146      |    rna-DGYR_LOCUS12552-2  139      142      +         start
+  147      |    rna-DGYR_LOCUS12552-2  237      240      +         start
+  148      |    rna-DGYR_LOCUS12552-2  337      340      +         start
+  149      |    rna-DGYR_LOCUS12552-2  520      523      +         start
   PyRanges with 56 rows, 5 columns, and 1 index columns.
   Contains 17 chromosomes and 1 strands.
 
   >>> cds.slice_ranges(-3).assign(Feature='stop').map_to_local(exons, ref_on='Parent', match_by='Parent')
   index    |    Chromosome             Start    End      Strand    Feature
   int64    |    str                    int64    int64    str       str
   -------  ---  ---------------------  -------  -------  --------  ---------
-  0        |    rna-DGYR_LOCUS13733    379      382      +         stop
-  1        |    rna-DGYR_LOCUS13734    481      484      +         stop
-  2        |    rna-DGYR_LOCUS13734    622      625      +         stop
-  3        |    rna-DGYR_LOCUS13734    795      798      +         stop
+  4        |    rna-DGYR_LOCUS13733    379      382      +         stop
+  11       |    rna-DGYR_LOCUS13734    481      484      +         stop
+  12       |    rna-DGYR_LOCUS13734    622      625      +         stop
+  13       |    rna-DGYR_LOCUS13734    795      798      +         stop
   ...      |    ...                    ...      ...      ...       ...
-  52       |    rna-DGYR_LOCUS12552-2  234      237      +         stop
-  53       |    rna-DGYR_LOCUS12552-2  334      337      +         stop
-  54       |    rna-DGYR_LOCUS12552-2  517      520      +         stop
-  55       |    rna-DGYR_LOCUS12552-2  637      640      +         stop
+  146      |    rna-DGYR_LOCUS12552-2  234      237      +         stop
+  147      |    rna-DGYR_LOCUS12552-2  334      337      +         stop
+  148      |    rna-DGYR_LOCUS12552-2  517      520      +         stop
+  149      |    rna-DGYR_LOCUS12552-2  637      640      +         stop
   PyRanges with 56 rows, 5 columns, and 1 index columns.
   Contains 17 chromosomes and 1 strands.

docs/tutorial.rst

@@ -572,7 +572,7 @@ of the overlapping intervals, similar to a SQL join operation:
     index  |    Chromosome           Start      End  Strand      ID                  Start_b    End_b  ID_b
     int64  |    category             int64    int64  category    str                   int64    int64  str
   -------  ---  -----------------  -------  -------  ----------  ----------------  ---------  -------  ----------------
-       15  |    CAJFCJ010000025.1     2755     3055  -           cds-CAD5125115.1       2753     2851  cds-CAD5125114.1
+      135  |    CAJFCJ010000025.1     2755     3055  -           cds-CAD5125115.1       2753     2851  cds-CAD5125114.1
   PyRanges with 1 rows, 8 columns, and 1 index columns.
   Contains 1 chromosomes and 1 strands.
 
@@ -581,8 +581,8 @@ The object ``j`` contains the columns of both objects, with the suffix "_b" to d
 It may be a bit too wide for our taste. Let's just look at a few columns to understand the overlap:
 
   >>> j[['ID', 'Start', 'End', 'ID_b', 'Start_b', 'End_b']]
-                    ID  Start   End              ID_b  Start_b  End_b
-  15  cds-CAD5125115.1   2755  3055  cds-CAD5125114.1     2753   2851
+                     ID  Start   End              ID_b  Start_b  End_b
+  135  cds-CAD5125115.1   2755  3055  cds-CAD5125114.1     2753   2851
 
 Above, we used a pandas syntax to select columns. Because the returned object does not have all genomic location
 columns, it is a pandas DataFrame.

pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyranges1"
-version = "1.3.8"
+version = "1.3.9"
 description = "GenomicRanges for Python."
 requires-python = ">=3.12.0"
 readme = "README.md"

pyranges1/core/pyranges_main.py

@@ -1215,7 +1215,10 @@ def join_overlaps(  # type: ignore[override]
 
         Note
         ----
-        The indices of the two input PyRanges are not preserved in output.
+        The index of the output is taken from the primary side of the join: ``self`` for
+        "inner", "left" and "outer", and ``other`` for "right". As a result the index may
+        contain duplicates (when an interval has several overlaps) and, for "outer"/"right",
+        missing-side rows keep the index of the side they originate from.
         The chromosome column from other will never be reported as it is always the same as in self.
         Whether the strand column from other is reported depends on the strand_behavior.
 
@@ -1278,11 +1281,11 @@ def join_overlaps(  # type: ignore[override]
           index  |    Chromosome        Start        End  Name         Start_b      End_b  Name_b
           int64  |    str             float64    float64  str          float64    float64  str
         -------  ---  ------------  ---------  ---------  ---------  ---------  ---------  --------
-              1  |    chr1                  5          7  interval2          6          7  b
+              2  |    chr1                  5          7  interval2          6          7  b
               0  |    chr1                  3          6  interval1        nan        nan  nan
               1  |    chr1                  8          9  interval3        nan        nan  nan
               0  |    nan                 nan        nan  nan                1          2  a
-        PyRanges with 4 rows, 7 columns, and 1 index columns (with 2 index duplicates).
+        PyRanges with 4 rows, 7 columns, and 1 index columns (with 1 index duplicates).
         Contains 1 chromosomes.
         Invalid ranges:
           * 1 starts or ends are nan. See indexes: 0

pyranges1/methods/join.py

@@ -33,39 +33,41 @@ def _both_dfs(
     )
     expected_columns = [*df.head(0).join(df2.head(0), how="inner", rsuffix=suffix).columns]
     df2.columns = expected_columns[df.shape[1] :]
+    # _self_indexes / _other_indexes are positional (from the overlap engine). Take the
+    # matching rows, then temporarily reset both indexes to a RangeIndex so the column-wise
+    # join pairs them by position rather than by label.
     _df = df.take(_self_indexes)  # type: ignore[arg-type]
-    _df.index = pd.Index(np.arange(len(_df)))
+    _df.index = pd.RangeIndex(len(_df))
     _df2 = pd.DataFrame(df2).take(_other_indexes)  # type: ignore[arg-type]
-    _df2.index = pd.Index(np.arange(len(_df2)))
+    _df2.index = pd.RangeIndex(len(_df2))
     j = _df.join(_df2, how="inner")
+    # Restore the original input index, like overlap()/nearest_ranges()/etc. The primary
+    # side of the join keeps its index: self for inner/left/outer, other for right.
+    j.index = df2.index[_other_indexes] if join_type == "right" else df.index[_self_indexes]
+
     if join_type == "inner":
-        j.index = pd.Index(_self_indexes)
         return j
 
     if join_type == "left":
-        missing_rows = _missing_rows_left(_self_indexes, df, j)
-        return pd.concat([j, missing_rows])
+        return pd.concat([j, _missing_rows_left(_self_indexes, df)])
 
     if join_type == "right":
-        missing_rows = _missing_rows_right(_other_indexes, df2, j)
-        return pd.concat([j, missing_rows])
+        return pd.concat([j, _missing_rows_right(_other_indexes, df2)])
 
     if join_type == "outer":
-        missing_rows_l = _missing_rows_left(_self_indexes, df, j)
-        missing_rows_r = _missing_rows_right(_other_indexes, df2, j)
-        return pd.concat([j, missing_rows_l, missing_rows_r])
+        return pd.concat([j, _missing_rows_left(_self_indexes, df), _missing_rows_right(_other_indexes, df2)])
 
     msg = f"Invalid join type: {join_type}"
     raise ValueError(msg)
 
 
-def _missing_rows_left(_self_indexes, df, j) -> pd.DataFrame:
-    missing_indices_left = df.index.difference(_self_indexes)
-    j.index = _self_indexes
-    return df.reindex(missing_indices_left)
+def _missing_rows_left(_self_indexes, df) -> pd.DataFrame:
+    # Rows of self without an overlap are the positions not present in _self_indexes.
+    # df.take keeps their original index labels, preserving the input index.
+    missing_positions = np.setdiff1d(np.arange(len(df)), _self_indexes)
+    return df.take(missing_positions)
 
 
-def _missing_rows_right(_other_indexes, df2, j) -> pd.DataFrame:
-    missing_indices_right = df2.index.difference(_other_indexes)
-    j.index = _other_indexes
-    return df2.reindex(missing_indices_right)
+def _missing_rows_right(_other_indexes, df2) -> pd.DataFrame:
+    missing_positions = np.setdiff1d(np.arange(len(df2)), _other_indexes)
+    return pd.DataFrame(df2).take(missing_positions)

pyranges1/range_frame/range_frame.py

@@ -366,7 +366,8 @@ def join_overlaps(
         -------
         RangeFrame
             A new RangeFrame containing the joined intervals with columns from both input RangeFrames.
-            The indices of the input RangeFrames are not preserved in the output.
+            The output keeps the index of the primary side of the join (``self`` for inner/left/outer
+            joins, ``other`` for right joins), so it may contain duplicates.
 
         Notes
         -----

tests/unit/test_join.py

@@ -1,8 +1,55 @@
+import pandas as pd
+import pytest
 from numpy import nan
 
 import pyranges1 as pr
 
 
+@pytest.mark.parametrize("index", [[0, 1], [1, 0], [1, 2], [2, 1]])
+def test_join_left_nonzero_index(index) -> None:
+    """A left join must not depend on the labels of self's index (issue: rows duplicated
+    when the left index did not start at 0). The original self index is also preserved,
+    consistent with overlap()/nearest_ranges()."""
+    left = pd.DataFrame(
+        {"Chromosome": ["1", "2"], "Start": [100, 300], "End": [200, 400]},
+        index=index,
+    )
+    right = pd.DataFrame({"Chromosome": ["1"], "Start": [150], "End": [160]})
+
+    j = pr.PyRanges(left).join_overlaps(pr.PyRanges(right), join_type="left")
+
+    expected = pr.PyRanges(
+        {
+            "Chromosome": ["1", "2"],
+            "Start": [100, 300],
+            "End": [200, 400],
+            "Start_b": [150.0, nan],
+            "End_b": [160.0, nan],
+        },
+        index=index,
+    )
+    assert j.equals(expected)
+
+
+def test_join_preserves_input_index() -> None:
+    """join_overlaps preserves the original input index (self for inner/left/outer,
+    other for right), like the rest of the overlap-family methods."""
+    left = pd.DataFrame(
+        {"Chromosome": ["1", "2"], "Start": [100, 300], "End": [200, 400]},
+        index=[10, 20],
+    )
+    right = pd.DataFrame({"Chromosome": ["1"], "Start": [150], "End": [160]}, index=[77])
+    gl, gr = pr.PyRanges(left), pr.PyRanges(right)
+
+    assert gl.join_overlaps(gr, join_type="inner").index.tolist() == [10]
+    assert gl.join_overlaps(gr, join_type="left").index.tolist() == [10, 20]
+    assert gl.join_overlaps(gr, join_type="right").index.tolist() == [77]
+    assert gl.join_overlaps(gr, join_type="outer").index.tolist() == [10, 20]
+    # inputs must not be mutated
+    assert left.index.tolist() == [10, 20]
+    assert right.index.tolist() == [77]
+
+
 def test_join_issue_4_right() -> None:
     import numpy as np