Merge branch 'release/0.10.0'

Author: siebrenf

.codeclimate.yml

@@ -21,14 +21,6 @@ checks:
       threshold: 250
 
 exclude_patterns:
-  - ".*"
-  - "*.md"
-  - "*.yml"
-  - "setup.py"
-  - "LICENSE"
-  - "**/*.yaml"
-  - "**/__*"
+  - "*"
+  - "!genomepy/*"
   - "genomepy/cli.py"
-  - "docs/*"
-  - "paper/*"
-  - "tests/*"

.flake8

@@ -1,5 +1,3 @@
 [flake8]
-exclude = .git,build
-max-line-length = 88
 extend-ignore = E203,E402,E501
 select = C,E,F,W,B,B950

.github/workflows/docs.yml

@@ -0,0 +1,38 @@
+name: Build and Deploy
+on:
+  push:
+    branches:
+      - master
+      - docs # backdoor branch to update docs
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v2.3.1
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies 🔨
+        run: |
+          $CONDA/bin/conda config --add channels conda-forge
+          $CONDA/bin/conda config --add channels bioconda
+          $CONDA/bin/conda install mamba
+          $CONDA/bin/mamba env update --file environment.yml --name base
+          $CONDA/bin/mamba env update --file docs/requirements.yaml --name base
+          $CONDA/bin/pip install .
+
+      - name: Build 🔧
+        run: |
+          $CONDA/bin/sphinx-build docs build
+          touch build/.nojekyll
+
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@4.1.4
+        with:
+          branch: gh-pages # The branch the action should deploy to.
+          folder: build # The folder the action should deploy.
+          clean: true

.gitignore

@@ -8,14 +8,15 @@
 coverage.xml
 cc-test-reporter
 Log.out
-genomepy/cfg/default.yaml
+genomepy/config/default.yaml
 
 # Directories
+docs/_autosummary
+docs/_build
+tests/data
 .pytest_cache
 build
-docs/_build
 genomepy.egg-info
-tests/data
 dist
 eggs
 tmp

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+files: 'setup\.py|genomepy/.*|tests/.*'
+repos:
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args:
+          - --check
+          - --recursive
+          - --remove-all-unused-imports
+          - --remove-duplicate-keys
+          - --remove-unused-variables
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.9.1
+    hooks:
+      - id: isort
+        args:
+          - --check
+          - --profile=black
+          - --conda-env=environment.yml
+
+  - repo: https://github.com/ambv/black
+    rev: 21.5b2
+    hooks:
+    - id: black
+      args:
+        - --check
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    - id: flake8
+      additional_dependencies:
+        - flake8-bugbear==21.4.3

.travis.yml

@@ -1,3 +1,9 @@
+# run travis on these branches (and all PRs)
+branches:
+  only:
+  - master
+  - develop
+
 # use the minimal travis environment since we test in conda
 language: minimal
 
@@ -10,47 +16,55 @@ env:
     - CC_TEST_REPORTER_ID=951f438ac8a0fa93801ff0bf69922df59fe03800bf7ea8ab77a3c26cda444979
   jobs:
     - PYTHON_VERSION=3.6
-    - PYTHON_VERSION=3.9
+
+cache:
+  directories:
+    - $HOME/miniconda
 
 before_install:
-  # install miniconda
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
+  # install miniconda or use the cache
+  - if test -e $HOME/miniconda/bin; then
+      echo "Miniconda already installed";
+
+      export PATH=$HOME/miniconda/bin:$PATH;
+      conda config --set always_yes yes;
+      mamba env update -n genomepy -f environment.yml --prune;
+    fi
+  - if ! test -e $HOME/miniconda/bin; then
+      echo "Installing miniconda";
+
+      CONDA_OS=$([ "$TRAVIS_OS_NAME" = "linux" ] && echo "Linux" || echo "MacOSX");
+      wget https://repo.continuum.io/miniconda/Miniconda3-latest-${CONDA_OS}-x86_64.sh -O miniconda.sh;
+      chmod +x miniconda.sh;
+      ./miniconda.sh -b -p $HOME/miniconda -f;
+
+      export PATH=$HOME/miniconda/bin:$PATH;
+      conda config --set always_yes yes;
+      conda install conda-forge::mamba;
+      mamba env create -n genomepy python=$PYTHON_VERSION -f environment.yml;
     fi
-  - chmod +x miniconda.sh
-  - ./miniconda.sh -b -p $HOME/miniconda -f
-  - export PATH=$HOME/miniconda/bin:$PATH
-  - conda config --set always_yes yes
-  - conda install conda-forge::mamba
 
 install:
-  - mamba env create -n genomepy python=$PYTHON_VERSION -f environment.yml
   - source activate genomepy
   - python setup.py develop
   - python setup.py build
 
 before_script:
   # install codeclimate test coverage
-#  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-#      wget -O cc-test-reporter https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64;
-#      chmod +x ./cc-test-reporter;
-#      ./cc-test-reporter before-build;
-#    fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       wget -O cc-test-reporter https://codeclimate.com/downloads/test-reporter/test-reporter-latest-darwin-amd64;
       chmod +x ./cc-test-reporter;
       ./cc-test-reporter before-build;
     fi
 
 script:
-  - black --check setup.py genomepy/ tests/
-  - flake8 setup.py genomepy/ tests/
-  - pytest -vv --disable-pytest-warnings tests/*
+  - pytest -vv --disable-pytest-warnings
     --reruns 1 --reruns-delay 10
     --cov=genomepy --cov-config=tests/.coveragerc --cov-report=xml
 
+before_cache:
+  - conda clean --all
+
 after_script:
   # send the coverage data to Code Climate
   - if [ -f ./cc-test-reporter ]; then

CHANGELOG.md

@@ -6,6 +6,53 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+## [0.10.0] - 2021-07-30
+
+### Added
+- Annotation class, containing
+  - regex filter (`genomepy.Annotation.filter_regex()`)
+  - sanitize functions (`genomepy.Annotation.sanitize()`)
+    - option to skip filtering and/or matching the annotation to the genome (also on CLI)
+  - gene name remapping to various formats (`genomepy.Annotation.map_genes()`)
+    - using MyGene.info. Can be queried separately (`genomepy.annotation.query_mygene()`)
+  - contig name remapping to other provider formats (`genomepy.Annotation.map_locations()`)
+  - get the annotations, or gene locations, as dataframes (`genomepy.Annotation.gtf`, `bed` or `gene_coords()` respectively)
+  - get the gene names as a list (`genomepy.Annotation.genes("gtf")` or `genomepy.Annotation.genes("bed")`)
+- `genomepy install` now attempts to install the NCBI assembly report
+- NCBI provider also indexes the NCBI `genbank_historical` summary
+- `genomepy search` now shows if the genome has an annotation
+    - this slows down the results a bit
+    - to compensate, results are now shown as soon as they are found
+    - for UCSC, availability of any of the 4 annotations is shown
+- `genomepy annotation` shows the first line(s) of each gene annotation.gtf
+- for developers:
+  - pre-commit-hooks for linting
+  - formatting/linting script `tests/format.sh` (optional argument `lint`)
+  - isort & autoflake formatters
+
+### Changed
+- provider module split per provider
+- ProviderBase overhauled, now called Provider
+- regex filtering separated from `Provider.download_genome`
+- utils module split into utils, files and online
+- now using loguru for pretty logging
+- accession `search` improved
+  - now finds GCA and GCF accessions
+  - now ignores patch levels
+- `genomepy install` automatic provider selection refactored
+    - `Provider.online_providers` returns a generator (faster!)
+- `genomepy install` uses a combined filter function (faster!)
+- `genomepy install` only zips annotation files if the genome is zipped (with the bgzip flag) (faster!)
+- NCBI provider should be parsed faster (faster!)
+- new dependency: pandas
+- tests no longer format code
+
+### Fixed
+- broken URLs should keep genomepy occupied for less long (check_url will immediately return on "Not Found" errors 404/450) (faster!)
+- the `Genome` class now passes arguments to the parent `Fasta` class
+- the `Genome` class now regenerates the sizes and gaps files similarly to the `Fasta` class and its index (when the genome is younger) (faster!)
+- somewhat more pythonic tests
+
 ## [0.9.3] - 2021-02-03
 
 ### Changed
@@ -286,6 +333,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Added `-r` and `--match/--no-match` option to select sequences by regex.
 
 [Unreleased]: https://github.com/vanheeringen-lab/genomepy/compare/master...develop
+[0.10.0]: https://github.com/vanheeringen-lab/genomepy/compare/0.9.3...0.10.0
 [0.9.3]: https://github.com/vanheeringen-lab/genomepy/compare/0.9.2...0.9.3
 [0.9.2]: https://github.com/vanheeringen-lab/genomepy/compare/0.9.1...0.9.2
 [0.9.1]: https://github.com/vanheeringen-lab/genomepy/compare/0.9.0...0.9.1