GTF parse fix (#155)

* GTF parse fix

* GTF parse: fix tests also

* GTF parse: fix tests also

Author: endrebak

CHANGELOG.txt

@@ -1,3 +1,6 @@
+# 0.0.85 (17.09.20)
+- fix error when parsing gtf-files with whitespace in value-tags
+
 # 0.0.84 (18.08.20)
 - add option to report overlap in join
 

pyranges/readers.py

@@ -313,7 +313,7 @@ def read_gtf_full(f, as_df=False, nrows=None, skiprows=0, duplicate_attr=False):
 
     names = "Chromosome Source Feature Start End Score Strand Frame Attribute".split(
     )
-    # names = "Chromosome Start End Score Strand Source Feature Frame Attribute".split()
+
     df_iter = pd.read_csv(
         f,
         sep="\t",
@@ -352,8 +352,14 @@ def to_rows(anno):
         raise Exception("Invalid attribute string: {l}. If the file is in GFF3 format, use pr.read_gff3 instead.".format(l=l))
 
     for l in anno:
-        l = l.replace('"', '').replace(";", "").split()
-        rowdicts.append({k: v for k, v in zip(*([iter(l)] * 2))})
+        rowdicts.append({k: v
+                         for k, v in [kv.replace('"', '').split(None, 1)
+                                      # l[:-1] removes final ";" cheaply
+                                      for kv in l[:-1].split("; ")]})
+
+    # for l in anno:
+    #     l = l.replace('"', '').replace(";", "").split()
+    #     rowdicts.append({k: v for k, v in zip(*([iter(l)] * 2))})
 
     return pd.DataFrame.from_dict(rowdicts).set_index(anno.index)
 
@@ -362,8 +368,10 @@ def to_rows_keep_duplicates(anno):
     rowdicts = []
     for l in anno:
         rowdict = {}
-        l = l.replace('"', '').replace(";", "").split()
-        for k, v in zip(*([iter(l)] * 2)):
+
+        # l[:-1] removes final ";" cheaply
+        for k, v in (kv.replace('"', '').split(None, 1) for kv in l[:-1].split("; ")):
+
             if k not in rowdict:
                 rowdict[k] = v
             elif k in rowdict and isinstance(rowdict[k], list):
@@ -442,8 +450,6 @@ def read_gtf_restricted(f,
 def to_rows_gff3(anno):
     rowdicts = []
 
-    # anno = anno.str.replace(";$", "")
-
     for l in list(anno):
         l = (it.split("=") for it in l.split(";"))
         rowdicts.append({k: v for k, v in l})

pyranges/version.py

@@ -1 +1 @@
-__version__ = "0.0.84"
+__version__ = "0.0.85"

tests/test_io.py

@@ -1,14 +1,11 @@
 import pyranges as pr
 
-# def test_read_bam():
-
-#     pr.read_bam("tests/test_data/test_sorted.bam")
-
 
 def test_read_gtf():
 
     gr = pr.read_gtf("tests/test_data/ensembl.gtf", full=True)
-    assert len(gr.columns) == 28
+
+    assert len(gr.columns) == 26
 
     df = gr.df
     transcript = df.iloc[1]
@@ -19,7 +16,8 @@ def test_read_gtf():
 
     gr = pr.read_gtf("tests/test_data/ensembl.gtf",
                      full=True, duplicate_attr=True)
-    assert len(gr.columns) == 28
+    print(gr.columns)
+    assert len(gr.columns) == 26
 
     df = gr.df
     transcript = df.iloc[1]