Merge branch 'release/0.16.2'

siebrenf · siebrenf · commit 22b80eee1e08 · 2025-05-12T16:45:28.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Fixed
+- Ensembl release versions no longer includes unreleased versions 
+- unit tests
+- upgraded formatters (and fixed the marked grammar & spelling errors)
+
 ## [0.16.1] - 2023-06-14
 
 ### Fixed
@@ -267,7 +272,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - `search` and `install` now consistently use safe search terms (no spaces)
 - `search` now uses UTF-8, no longer crashing for \u2019 (some quotation mark).
 - `search` case insensitivity fixed for assembly names.
-- Bucketcache now stores less data, increasing responsiveness.
+- Bucketcache now stores fewer data, increasing responsiveness.
 
 ## [0.8.4] - 2020-07-29
 
@@ -382,7 +387,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Fixed
 
-- STAR is not longer enabled by default
+- STAR is no longer enabled by default
 
 ## [0.7.0] - 2019-11-18
 
@@ -461,7 +466,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Fixed genomes_dir argument to `genomepy install`
 - Fixed msgpack dependency
-- Fixed issue with `config generate` where config directory does note exist.
+- Fixed issue with `config generate` where config directory does not exist.
 
 ## [0.3.1]
 
@@ -478,7 +483,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Added `-r` and `--match/--no-match` option to select sequences by regex.
 
 [Unreleased]: https://github.com/vanheeringen-lab/genomepy/compare/master...develop
-[0.16.0]: https://github.com/vanheeringen-lab/genomepy/compare/0.16.0...0.16.1
+[0.16.1]: https://github.com/vanheeringen-lab/genomepy/compare/0.16.0...0.16.1
 [0.16.0]: https://github.com/vanheeringen-lab/genomepy/compare/0.15.0...0.16.0
 [0.15.0]: https://github.com/vanheeringen-lab/genomepy/compare/0.14.0...0.15.0
 [0.14.0]: https://github.com/vanheeringen-lab/genomepy/compare/0.13.1...0.14.0
diff --git a/README.md b/README.md
@@ -431,7 +431,7 @@ Check out our [Python API documentation here](https://vanheeringen-lab.github.io
 >>> for row in genomepy.search("GRCh38"):
 ...    print(row)
 ...    
-['GRCh38.p13', 'Ensembl', 'GCA_000001405.28', 9606, True, 'Homo sapiens', '2014-01-Ensembl/2021-03']
+['GRCh38.p14', 'Ensembl', 'GCA_000001405.28', 9606, True, 'Homo sapiens', '2014-01-Ensembl/2021-03']
 ['hg38', 'UCSC', 'GCA_000001405.15', 9606, [True, True, False, True], 'Homo sapiens', 'Dec. 2013 (GRCh38/hg38)']
 ['GRCh38', 'NCBI', 'GCF_000001405.26', 9606, True, 'Homo sapiens', 'Genome Reference Consortium']
 ['GRCh38.p1', 'NCBI', 'GCF_000001405.27', 9606, True, 'Homo sapiens', 'Genome Reference Consortium']
diff --git a/docs/release_checklist.md b/docs/release_checklist.md
@@ -46,7 +46,7 @@
     genomepy clean
     genomepy search xenopus_tropicalis
     genomepy annotation hg38
-    genomepy annotation GRCh38.p13
+    genomepy annotation GRCh38.p14
     genomepy install -af -p gencode GRCm39
     genomepy install -af -p ensembl TAIR10
     genomepy install -af -p ucsc sacCer3 --UCSC-annotation ensGene
diff --git a/environment.yml b/environment.yml
@@ -10,9 +10,10 @@ dependencies:
   - click
   - colorama
   - filelock >=3.5
+  - libsqlite <3.49  # v3.49.1: no column "size" (this is a diskcache requirement)
   - loguru
   - mygene
-  - mysql-connector-python <=8.4.0
+  - mysql-connector-python <=8.4.0  # v9.0.0: 'mysql_native_password' cannot be loaded
   - norns >=0.1.6
   - numpy
   - pandas
@@ -39,18 +40,18 @@ dependencies:
   - minimap2
   - star
 
-  # Distribution
-  - python-build
-  - twine ==3.3.0
-  - keyring ==22.3.0
+#  # Distribution
+#  - python-build
+#  - twine =3.3
+#  - keyring =22.3
 
   # Testing
-  - autoflake ==1.4
-  - black ==22.6.0
-  - flake8 ==4.0.1
-  - flake8-bugbear ==22.7.1
-  - flakeheaven ==2.0.0
-  - isort ==5.10.1
+  - autoflake =1.7
+  - black =25.1
+  - flake8 =4.0
+  - flake8-bugbear =23.3
+#  - flakeheaven  # v3.3.0 does not work with python 3.12
+  - isort =6.0
   - pytest
   - pytest-rerunfailures
   - pytest-cov
diff --git a/genomepy/__about__.py b/genomepy/__about__.py
@@ -1,5 +1,6 @@
 """Metadata"""
-__version__ = "0.16.1"
+
+__version__ = "0.16.2"
 __author__ = (
     "Siebren Frölich, Maarten van der Sande, Tilman Schäfers and Simon van Heeringen"
 )
diff --git a/genomepy/annotation/__init__.py b/genomepy/annotation/__init__.py
@@ -1,4 +1,5 @@
 """Annotation class, modules & related functions"""
+
 import os
 import re
 from pathlib import Path
diff --git a/genomepy/annotation/mygene.py b/genomepy/annotation/mygene.py
@@ -1,4 +1,5 @@
 """Mygene.info functions and methods"""
+
 from typing import Iterable, Union
 
 import mygene
diff --git a/genomepy/annotation/sanitize.py b/genomepy/annotation/sanitize.py
@@ -1,4 +1,5 @@
 """Annotation.sanitize method"""
+
 from itertools import compress
 
 from genomepy.annotation.utils import _check_property, write_annot
diff --git a/genomepy/annotation/utils.py b/genomepy/annotation/utils.py
@@ -1,4 +1,5 @@
 """Utility functions with gene annotations"""
+
 import csv
 import os
 import shutil
diff --git a/genomepy/exceptions.py b/genomepy/exceptions.py
@@ -1,5 +1,4 @@
 class GenomeDownloadError(Exception):
-
     """Error while downloading genome."""
 
     pass  # noqa
diff --git a/genomepy/files.py b/genomepy/files.py
@@ -1,4 +1,5 @@
 """Utility functions with files"""
+
 import gzip
 import os
 import re
@@ -169,7 +170,7 @@ def extract_archive(
     -------
     tuple
         fname, str
-            up to date filename
+            up-to-date filename
     """
     if fname.endswith((".tgz", ".tar.gz")):
         return extract_tarball(fname, outfile=outfile)
@@ -223,7 +224,7 @@ def extract_gzip(fname: str, outfile: Optional[str] = None) -> Union[str, None]:
     -------
     tuple
         fname, str
-            up to date filename, or None if failed
+            up-to-date filename, or None if failed
     """
     if not outfile:
         outfile = fname[:-3]
@@ -256,7 +257,7 @@ def extract_zip(
     -------
     tuple
         fname, str
-            up to date filename
+            up-to-date filename
     """
     if not outfile:
         outfile = fname[:-4]
@@ -295,7 +296,7 @@ def gzip_and_name(fname, gzip_file=True) -> str:
     Returns
     -------
     str
-        up to date filename
+        up-to-date filename
     """
     if gzip_file:
         with open(fname, "rb") as f_in:
@@ -320,7 +321,7 @@ def bgzip_and_name(fname, bgzip_file=True) -> str:
     Returns
     -------
     str
-        up to date filename
+        up-to-date filename
     """
     if bgzip_file:
         ret = sp.check_call(f"bgzip {fname}", shell=True)
diff --git a/genomepy/functions.py b/genomepy/functions.py
@@ -1,4 +1,5 @@
 """Module-level functions"""
+
 import os
 import re
 import sys
@@ -299,7 +300,7 @@ def generate_env(fname: str = "exports.txt", genomes_dir: str = None):
     """
     Generate file with exports.
 
-    By default the export file generated is .config/genomepy/exports.txt.
+    By default, the export file generated is .config/genomepy/exports.txt.
     An alternative file name or file path is accepted too.
 
     Parameters
@@ -424,7 +425,7 @@ def _get_fasta_regex_func(
     """
     # define filter functions
     if keep_alt is False:
-        alt_pattern = re.compile("(alt)", re.I)  # case insensitive
+        alt_pattern = re.compile("(alt)", re.I)  # case-insensitive
 
         def alt_keep(header):
             return not bool(alt_pattern.search(header))
diff --git a/genomepy/genome/__init__.py b/genomepy/genome/__init__.py
@@ -1,4 +1,5 @@
 """Genome class, modules & related functions"""
+
 import os.path
 import re
 from glob import glob
diff --git a/genomepy/genome/sequences.py b/genomepy/genome/sequences.py
@@ -1,4 +1,5 @@
 """Functions to extract or convert genomic sequences"""
+
 import os
 import re
 from bisect import bisect
diff --git a/genomepy/online.py b/genomepy/online.py
@@ -1,4 +1,5 @@
 """Utility functions with internet connections"""
+
 import socket
 import urllib.error
 from ftplib import FTP, all_errors, error_temp
diff --git a/genomepy/plugins/__init__.py b/genomepy/plugins/__init__.py
@@ -1,4 +1,5 @@
 """Plugin class, modules & related functions"""
+
 import os
 import re
 
diff --git a/genomepy/providers/__init__.py b/genomepy/providers/__init__.py
@@ -1,4 +1,5 @@
 """Provider class, modules & related functions"""
+
 import os
 from typing import Optional
 
diff --git a/genomepy/providers/base.py b/genomepy/providers/base.py
@@ -1,4 +1,5 @@
 """BaseProvider class, the parent of the provider classes"""
+
 import gzip
 import os
 import re
@@ -285,7 +286,7 @@ def get_annotation_download_link(self, name: str, **kwargs) -> str:
 
     def download_annotation(self, name, genomes_dir=None, localname=None, **kwargs):
         """
-        Download annotation file to to a specific directory
+        Download annotation file to a specific directory
 
         Parameters
         ----------
diff --git a/genomepy/providers/ensembl.py b/genomepy/providers/ensembl.py
@@ -14,7 +14,7 @@ class EnsemblProvider(BaseProvider):
     """
     Ensembl genome provider.
 
-    Will search both ensembl.org as well as ensemblgenomes.org.
+    Will search both ensembl.org and ensemblgenomes.org.
     The bacteria division is not yet supported.
     """
 
@@ -116,9 +116,8 @@ def get_release(self, is_vertebrate: bool) -> int:
         ret = retry(request_json, 3, self._url, ext)
         return int(ret["releases"][0] if is_vertebrate else ret["version"])
 
-    @staticmethod
-    @disk_cache.memoize(expire=cache_exp_other)
-    def get_releases(is_vertebrate: bool):
+    @disk_cache.memoize(expire=cache_exp_other, ignore={"self"})
+    def get_releases(self, is_vertebrate: bool):
         """Retrieve all Ensembl or EnsemblGenomes release versions."""
         url = "http://ftp.ensemblgenomes.org/pub?"
         if is_vertebrate:
@@ -132,6 +131,10 @@ def get_releases(is_vertebrate: bool):
         if is_vertebrate:
             # ignore immature releases
             releases = [r for r in releases if r > 46]
+
+        # exclude pre-releases (returns 403: forbidden)
+        latest_release = self.get_release(is_vertebrate)
+        releases = [r for r in releases if r <= latest_release]
         return releases
 
     @lock
@@ -227,7 +230,7 @@ def get_annotation_download_links(self, name, **kwargs):
         name : str
             genome name
         **kwargs: dict, optional:
-            version : Ensembl version to use. By default the latest version is used
+            version : Ensembl version to use. By default, the latest version is used
 
         Returns
         -------
diff --git a/genomepy/providers/gencode.py b/genomepy/providers/gencode.py
@@ -127,7 +127,7 @@ def get_annotation_download_links(self, name, **kwargs):
 
     def download_annotation(self, name, genomes_dir=None, localname=None, **kwargs):
         """
-        Download annotation file to to a specific directory
+        Download annotation file to a specific directory
 
         Parameters
         ----------
diff --git a/genomepy/providers/ucsc.py b/genomepy/providers/ucsc.py
@@ -29,16 +29,16 @@
 # for more info, see http://genome.ucsc.edu/FAQ/FAQgenes.html
 ANNOTATIONS = ["ncbiRefSeq", "refGene", "ensGene", "knownGene"]
 
-# web adresses per mirror
+# web addresses per mirror
 ADRESSES = {
     "us": {
         "api": "https://api.genome.ucsc.edu/list/ucscGenomes",
-        "ftp": "https://hgdownload.soe.ucsc.edu/goldenPath",
+        "ftp": "https://hgdownload.soe.ucsc.edu/goldenPath",  # "ftp://hgdownload.soe.ucsc.edu/goldenPath",
         "sql": "genome-mysql.soe.ucsc.edu",
     },
     "eu": {
         "api": "https://genome-euro.ucsc.edu/cgi-bin/hubApi/list/ucscGenomes",
-        "ftp": "https://hgdownload-euro.soe.ucsc.edu/goldenPath",
+        "ftp": "https://hgdownload.soe.ucsc.edu/goldenPath",  # "ftp://hgdownload.soe.ucsc.edu/goldenPath",
         "sql": "genome-euro-mysql.soe.ucsc.edu",
     },
 }[config.get("ucsc_mirror", "us").lower()]
@@ -74,7 +74,9 @@ def __init__(self):
     @staticmethod
     def ping():
         """Can the provider be reached?"""
-        url_online = bool(check_url(ADRESSES["ftp"]))
+        url_online = bool(
+            check_url(ADRESSES["ftp"].replace("goldenPath", "downloads.html"))
+        )
         api_online = bool(check_url(ADRESSES["api"]))
         return url_online and api_online
 
@@ -441,7 +443,7 @@ def add_accessions1(genomes: dict) -> dict:
 
     For NCBI, RefSeq and GenBank accession IDs are stored directly in this table.
 
-    Updates the the genome dict "assembly_accession" field for each genome found.
+    Updates the genome dict "assembly_accession" field for each genome found.
     """
     # MySQL query
     database = "hgFixed"
@@ -487,7 +489,7 @@ def add_accessions2(genomes: dict) -> dict:
     """
     Some genomes have their assembly accession in the 'sourceName' field.
 
-    Updates the the genome dict "assembly_accession" field for each genome found.
+    Updates the genome dict "assembly_accession" field for each genome found.
     """
     re_acc = re.compile(r"GC[AF]_\d{9}\.\d+")
     for name in genomes:
diff --git a/genomepy/providers/url.py b/genomepy/providers/url.py
@@ -140,7 +140,7 @@ def fuzzy_annotation_search(search_name, search_list):
     """Returns all files containing both name and an annotation extension"""
     hits = []
     for ext in ["gtf", "gff"]:
-        # .*? = non greedy filler. 3? = optional 3 (for gff3). (\.gz)? = optional .gz
+        # .*? = non-greedy filler. 3? = optional 3 (for gff3). (\.gz)? = optional .gz
         expr = rf"{search_name}.*?\.{ext}3?(\.gz)?"  # noqa: W605
         for line in search_list:
             hit = re.search(expr, line, flags=re.IGNORECASE)
diff --git a/genomepy/seq.py b/genomepy/seq.py
@@ -1,4 +1,5 @@
 """as_seqdict function & overloads"""
+
 import os
 import re
 import sys
diff --git a/genomepy/utils.py b/genomepy/utils.py
@@ -1,4 +1,5 @@
 """Utility functions"""
+
 import itertools
 import os
 import re
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/format.sh b/tests/format.sh
diff --git a/tests/test_01_basic.py b/tests/test_01_basic.py
diff --git a/tests/test_04_online.py b/tests/test_04_online.py
diff --git a/tests/test_09_provider_ensembl.py b/tests/test_09_provider_ensembl.py
diff --git a/tests/test_11_provider.py b/tests/test_11_provider.py
diff --git a/tests/test_14_functions.py b/tests/test_14_functions.py
diff --git a/tests/test_15_argparse.py b/tests/test_15_argparse.py
diff --git a/tests/test_e01_link_options.py b/tests/test_e01_link_options.py
diff --git a/tests/test_e03_human_genomes.py b/tests/test_e03_human_genomes.py

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`"""Metadata"""`
`2`		`-__version__ = "0.16.1"`
	`2`	`+`
	`3`	`+__version__ = "0.16.2"`
`3`	`4`	`__author__ = (`
`4`	`5`	`"Siebren Frölich, Maarten van der Sande, Tilman Schäfers and Simon van Heeringen"`
`5`	`6`	`)`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`"""Annotation class, modules & related functions"""`
	`2`	`+`
`2`	`3`	`import os`
`3`	`4`	`import re`
`4`	`5`	`from pathlib import Path`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`"""Mygene.info functions and methods"""`
	`2`	`+`
`2`	`3`	`from typing import Iterable, Union`
`3`	`4`
`4`	`5`	`import mygene`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`"""Annotation.sanitize method"""`
	`2`	`+`
`2`	`3`	`from itertools import compress`
`3`	`4`
`4`	`5`	`from genomepy.annotation.utils import _check_property, write_annot`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`"""Utility functions with gene annotations"""`
	`2`	`+`
`2`	`3`	`import csv`
`3`	`4`	`import os`
`4`	`5`	`import shutil`