0.0.110 (#223)

Co-authored-by: endre bakken stovner <endrebakkenstovner@endres-MacBook-Air.local>

Author: endrebak

CHANGELOG.txt

@@ -1,3 +1,7 @@
+# 0.0.110 (20.09.21)
+- fix count_overlaps with keep_nonoverlapping=False
+- fix subtract with more than 1024 intervals (new fix)
+
 # 0.0.109 (16.09.21)
 - fix overlap invert behavior
 - add intersect invert flag

pyranges/methods/coverage.py

@@ -33,18 +33,18 @@ def _number_overlapping(scdf, ocdf, **kwargs):
 
     df = scdf.copy()
 
-    if keep_nonoverlapping:
-        _missing_indexes = np.setdiff1d(scdf.index, _self_indexes)
-        missing = pd.DataFrame(data={"Index": _missing_indexes, "Count": 0}, index=_missing_indexes)
-        counts_per_read = pd.concat([counts_per_read, missing])
-    else:
-        df = df.loc[_self_indexes]
+    _missing_indexes = np.setdiff1d(scdf.index, _self_indexes)
+    missing = pd.DataFrame(data={"Index": _missing_indexes, "Count": 0}, index=_missing_indexes)
+    counts_per_read = pd.concat([counts_per_read, missing])
 
     counts_per_read = counts_per_read.set_index("Index").sort_index()
 
     df.insert(df.shape[1], column_name, counts_per_read)
 
-    return df
+    if keep_nonoverlapping:
+        return df
+    else:
+        return df[df[column_name] != 0]
 
 
 

pyranges/methods/subtraction.py

@@ -1,6 +1,8 @@
 import pandas as pd
 import numpy as np
 
+from ncls import NCLS
+
 
 def add_rows_per_group(df):
     last_rows = df.groupby("__ix__").last().reset_index()
@@ -9,79 +11,79 @@ def add_rows_per_group(df):
     df = df.sort_values("__ix__", ascending=True)
     return df
 
-def _subtraction(scdf, **kwargs):
-    if scdf.empty:
-        return scdf
+# def _subtraction(scdf, **kwargs):
+#     if scdf.empty:
+#         return scdf
 
-    falses = np.zeros(len(scdf), dtype=bool)
-    scdf.insert(scdf.shape[1], "__first__", falses)
-    scdf.insert(scdf.shape[1], "__last__", falses)
+#     falses = np.zeros(len(scdf), dtype=bool)
+#     scdf.insert(scdf.shape[1], "__first__", falses)
+#     scdf.insert(scdf.shape[1], "__last__", falses)
 
-    scdf = add_rows_per_group(scdf)
+#     scdf = add_rows_per_group(scdf)
 
-    scdf.insert(scdf.shape[1], "NewStart", scdf.End__deleteme__.shift(fill_value=-1))
-    scdf.insert(scdf.shape[1], "NewEnd", scdf.Start__deleteme__)
-    scdf.insert(scdf.shape[1], "__ix2__", np.arange(len(scdf)))
+#     scdf.insert(scdf.shape[1], "NewStart", scdf.End__deleteme__.shift(fill_value=-1))
+#     scdf.insert(scdf.shape[1], "NewEnd", scdf.Start__deleteme__)
+#     scdf.insert(scdf.shape[1], "__ix2__", np.arange(len(scdf)))
 
-    first_rows = scdf.groupby(scdf.__ix__, as_index=False).first()
-    scdf.loc[scdf.__ix2__.isin(first_rows.__ix2__), "__first__"] = True
+#     first_rows = scdf.groupby(scdf.__ix__, as_index=False).first()
+#     scdf.loc[scdf.__ix2__.isin(first_rows.__ix2__), "__first__"] = True
 
-    scdf.loc[:, "NewStart"] = np.where(scdf.__first__, scdf.Start, scdf.NewStart)
+#     scdf.loc[:, "NewStart"] = np.where(scdf.__first__, scdf.Start, scdf.NewStart)
 
-    scdf.loc[scdf.__first__ & ~(scdf.Start__deleteme__ >= scdf.Start), "NewStart"] = -1
-    scdf.loc[:, "NewEnd"] = np.where(scdf.__last__, scdf.End, scdf.NewEnd)
-    scdf.loc[:, "NewStart"] = np.where(scdf.__last__, scdf.End__deleteme__, scdf.NewStart)
-    scdf.loc[scdf.__last__ & ~(scdf.End__deleteme__ <= scdf.End), ["NewEnd", "NewStart"]] = -1
+#     scdf.loc[scdf.__first__ & ~(scdf.Start__deleteme__ >= scdf.Start), "NewStart"] = -1
+#     scdf.loc[:, "NewEnd"] = np.where(scdf.__last__, scdf.End, scdf.NewEnd)
+#     scdf.loc[:, "NewStart"] = np.where(scdf.__last__, scdf.End__deleteme__, scdf.NewStart)
+#     scdf.loc[scdf.__last__ & ~(scdf.End__deleteme__ <= scdf.End), ["NewEnd", "NewStart"]] = -1
 
 
-    scdf = scdf[~((scdf.NewStart == -1) | (scdf.NewEnd == -1))]
-    scdf = scdf.drop(["Start", "End"], axis=1)
-    scdf.rename(columns={"NewStart": "Start", "NewEnd": "End"}, inplace=True)
+#     scdf = scdf[~((scdf.NewStart == -1) | (scdf.NewEnd == -1))]
+#     scdf = scdf.drop(["Start", "End"], axis=1)
+#     scdf.rename(columns={"NewStart": "Start", "NewEnd": "End"}, inplace=True)
 
-    remove_mask = scdf.Start >= scdf.End
+#     remove_mask = scdf.Start >= scdf.End
 
-    return scdf[~remove_mask]
+#     return scdf[~remove_mask]
 
-# def _subtraction(scdf, ocdf, **kwargs):
+def _subtraction(scdf, ocdf, **kwargs):
 
-#     if ocdf.empty or scdf.empty:
-#         return scdf
+    if ocdf.empty or scdf.empty:
+        return scdf
 
-#     strandedness = kwargs["strandedness"]
-#     strand = True if strandedness else False
+    strandedness = kwargs["strandedness"]
+    strand = True if strandedness else False
 
-#     chromosome = scdf.Chromosome.head(1).iloc[0]
-#     kwargs["chromosome"] = chromosome
+    chromosome = scdf.Chromosome.head(1).iloc[0]
+    kwargs["chromosome"] = chromosome
 
-#     if "Strand" in ocdf and strand:
-#         strand = scdf.Strand.head(1).iloc[0]
-#         kwargs["strand"] = strand
+    if "Strand" in ocdf and strand:
+        strand = scdf.Strand.head(1).iloc[0]
+        kwargs["strand"] = strand
 
-#     o = NCLS(ocdf.Start.values, ocdf.End.values, ocdf.index.values)
+    o = NCLS(ocdf.Start.values, ocdf.End.values, ocdf.index.values)
 
-#     idx_self, new_starts, new_ends = o.set_difference_helper(
-#         scdf.Start.values, scdf.End.values, scdf.index.values)
+    idx_self, new_starts, new_ends = o.set_difference_helper(
+        scdf.Start.values, scdf.End.values, scdf.index.values, scdf.__num__.values)
 
-#     missing_idx = pd.Index(scdf.index).difference(idx_self)
+    missing_idx = pd.Index(scdf.index).difference(idx_self)
 
-#     idx_to_drop = new_starts != -1
+    idx_to_drop = new_starts != -1
 
-#     new_starts = new_starts[idx_to_drop]
-#     new_ends = new_ends[idx_to_drop]
+    new_starts = new_starts[idx_to_drop]
+    new_ends = new_ends[idx_to_drop]
 
-#     idx_self = idx_self[idx_to_drop]
-#     new_starts = pd.Series(new_starts, index=idx_self)
-#     new_ends = pd.Series(new_ends, index=idx_self)
+    idx_self = idx_self[idx_to_drop]
+    new_starts = pd.Series(new_starts, index=idx_self)
+    new_ends = pd.Series(new_ends, index=idx_self)
 
-#     scdf = scdf.reindex(missing_idx.union(idx_self)).sort_index()
-#     new_starts = new_starts.sort_index()
-#     new_ends = new_ends.sort_index()
+    scdf = scdf.reindex(missing_idx.union(idx_self)).sort_index()
+    new_starts = new_starts.sort_index()
+    new_ends = new_ends.sort_index()
 
-#     if len(idx_self):
-#         scdf.loc[scdf.index.isin(idx_self), "Start"] = new_starts.values
-#         scdf.loc[scdf.index.isin(idx_self), "End"] = new_ends.values
+    if len(idx_self):
+        scdf.loc[scdf.index.isin(idx_self), "Start"] = new_starts.values
+        scdf.loc[scdf.index.isin(idx_self), "End"] = new_ends.values
 
-#     if not scdf.empty:
-#         return scdf
-#     else:
-#         return None
+    if not scdf.empty:
+        return scdf
+    else:
+        return None

pyranges/pyranges.py

@@ -4376,27 +4376,21 @@ def subtract(self, other, strandedness=None, nb_cpu=1):
 
         from pyranges.methods.subtraction import _subtraction
 
-        strand = True if strandedness in ["same", "opposite"] else False
-        kwargs = {"strandedness": strandedness, "strand": strand}
+        kwargs = {"strandedness": strandedness}
         kwargs["sparse"] = {"self": False, "other": True}
         kwargs = fill_kwargs(kwargs)
 
         strand = True if strandedness else False
-        if len(other) != 0:
-            cols_original_order = self.columns.copy()
-            other_clusters = other.merge(strand=strand)
-            self.__ix__ = np.arange(len(self))
-            no_overlap = self.overlap(other_clusters, strandedness=strandedness, invert=True)
-            j = self.sort().join(other_clusters, strandedness=strandedness, suffix="__deleteme__")
-            result = pyrange_apply_single(_subtraction, j, **kwargs)
-            result = {k: v for (k, v) in result.items()}
-            result = PyRanges(result)
-            result = pr.concat([result, no_overlap])
-            result = PyRanges({k: v[cols_original_order] for k, v in result.items()})
-        else:
-            result = self
+        other_clusters = other.merge(strand=strand)
+
+        self = self.count_overlaps(other_clusters, strandedness=strandedness, overlap_col="__num__")
+
+        result = pyrange_apply(_subtraction, self, other_clusters, **kwargs)
+
+        self = self.drop("__num__")
+
+        return PyRanges(result).drop("__num__")
 
-        return result
 
     def summary(self, to_stdout=True, return_df=False):
 

pyranges/version.py

@@ -1 +1 @@
-__version__ = "0.0.109"
+__version__ = "0.0.110"