Merge pull request #138 from JannisStraus/137-dicom-anonymization-missing

137 fix mypy typing and bump up versions

Author: Goku1110

.flake8

@@ -11,20 +11,19 @@ permissions:
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.8"
-    - name: Install poetry
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        export PATH="/root/.local/bin:$PATH"
-        poetry run pip install -U pip
-        poetry install
-    - name: Publish package
-      run: poetry publish --build --username __token__ --password ${{ secrets.PYPI_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          export PATH="/root/.local/bin:$PATH"
+          poetry run pip install -U pip
+          poetry install
+      - name: Publish package
+        run: poetry publish --build --username __token__ --password ${{ secrets.PYPI_TOKEN }}

.github/workflows/python-publish.yml

@@ -1,27 +1,38 @@
+# See https://gitlab.uk-essen.de/ship-ai/pre-commit-hooks for the newest version
+
 repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.0.1
-  hooks:
-  - id: end-of-file-fixer
-  - id: trailing-whitespace
-- repo: https://github.com/psf/black
-  rev: 22.3.0
-  hooks:
-  - id: black
-- repo: https://github.com/pycqa/isort
-  rev: 5.12.0
-  hooks:
-  - id: isort
-    exclude: _external
-- repo: https://github.com/PyCQA/flake8
-  rev: 3.9.2
-  hooks:
-  - id: flake8
-    exclude: _external
-    additional_dependencies: ["flake8-bugbear==21.4.3"]
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.910
-  hooks:
-  - id: mypy
-    exclude: _external
-    args: ["--install-types", "--non-interactive"]
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.11.6
+    hooks:
+      - id: ruff
+        args: [--fix]
+        exclude: examples
+      - id: ruff-format
+        exclude: examples
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+      - id: mypy
+        language_version: python3.10
+        additional_dependencies: [types-python-dateutil, types-requests]
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        types_or: [html, css, json, javascript, xml, yaml]
+        exclude: node_modules
+
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: v2.5.1
+    hooks:
+      - id: pyproject-fmt
+        types: [toml]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: check-merge-conflict
+      - id: check-added-large-files

.mypy.ini

@@ -24,7 +24,7 @@
 
 __all__ = [
     "Ahoy",
+    "DicomDownloader",
     "Miner",
     "Pirate",
-    "DicomDownloader",
 ]

.pre-commit-config.yaml

@@ -40,15 +40,15 @@ class Ahoy:
 
     def __init__(
         self,
-        auth_url: str = None,
+        auth_url: Optional[str] = None,
         auth_type: Optional[str] = "token",
-        refresh_url: str = None,
-        username: str = None,
+        refresh_url: Optional[str] = None,
+        username: Optional[str] = None,
         auth_method: Optional[str] = "password",
-        token: str = None,
+        token: Optional[str] = None,
         max_login_attempts: int = 5,
-        token_refresh_delta: Union[int, timedelta] = None,
-        session: requests.Session = None,
+        token_refresh_delta: Optional[Union[int, timedelta]] = None,
+        session: Optional[requests.Session] = None,
     ) -> None:
         self.auth_type = auth_type
         self.auth_method = auth_method
@@ -82,7 +82,7 @@ def __exit__(
         self.close()
 
     def change_environment_variable_name(
-        self, user_env: str = None, pass_env: str = None
+        self, user_env: Optional[str] = None, pass_env: Optional[str] = None
     ) -> None:
         """
         Change the name of the variables used to retrieve username and password.

fhir_pyrate/__init__.py

@@ -14,7 +14,18 @@
 from contextlib import contextmanager
 from functools import partial
 from types import TracebackType
-from typing import Dict, Generator, List, Optional, TextIO, Tuple, Type, Union
+from typing import (
+    ClassVar,
+    Dict,
+    FrozenSet,
+    Generator,
+    List,
+    Optional,
+    TextIO,
+    Tuple,
+    Type,
+    Union,
+)
 
 import pandas as pd
 import pydicom
@@ -67,7 +78,7 @@ def fileno(file_or_fd: TextIO) -> Optional[int]:
 @contextmanager
 def stdout_redirected(
     to: Union[str, TextIO] = os.devnull, stdout: Optional[TextIO] = None
-) -> Generator:
+) -> Generator[Optional[TextIO], None, None]:
     if platform.system() == "Windows":
         yield None
         return
@@ -136,23 +147,25 @@ class DicomDownloader:
     :param num_processes: The number of processes to run for downloading
     """
 
-    ACCEPTED_FORMATS = {
-        ".dcm",
-        ".nia",
-        ".nii",
-        ".nii.gz",
-        ".hdr",
-        ".img",
-        ".img.gz",
-        ".tif",
-        ".TIF",
-        ".tiff",
-        ".TIFF",
-        ".mha",
-        ".mhd",
-        ".nrrd",
-        ".nhdr",
-    }
+    ACCEPTED_FORMATS: ClassVar[FrozenSet[str]] = frozenset(
+        {
+            ".dcm",
+            ".nia",
+            ".nii",
+            ".nii.gz",
+            ".hdr",
+            ".img",
+            ".img.gz",
+            ".tif",
+            ".TIF",
+            ".tiff",
+            ".TIFF",
+            ".mha",
+            ".mhd",
+            ".nrrd",
+            ".nhdr",
+        }
+    )
 
     def __init__(
         self,
@@ -239,7 +252,7 @@ def __exit__(
     @staticmethod
     def get_download_id(
         study_uid: str,
-        series_uid: str = None,
+        series_uid: Optional[str] = None,
         always_download_in_study_folder: bool = False,
     ) -> str:
         """
@@ -259,7 +272,7 @@ def get_download_id(
 
     def get_download_path(self, download_id: str) -> pathlib.Path:
         """
-        Builds the folder hierarchy where the data will be stored. The hierarchy depends on the
+        Build the folder hierarchy where the data will be stored. The hierarchy depends on the
         `hierarchical_storage` parameter. Given a download ID
         263a1dad02916f5eca3c4eec51dc9d281735b47b8eb8bc2343c56e6ccd and `hierarchical_storage` = 2,
         the data will be stored in 26/3a/1dad02916f5eca3c4eec51dc9d281735b47b8eb8bc2343c56e6ccd.
@@ -277,13 +290,13 @@ def get_download_path(self, download_id: str) -> pathlib.Path:
     def download_data(
         self,
         study_uid: str,
-        series_uid: str = None,
+        series_uid: Optional[str] = None,
         output_dir: Union[str, pathlib.Path] = "out",
         save_metadata: bool = True,
         existing_ids: Optional[List[str]] = None,
     ) -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]:
         """
-        Downloads the data related to the StudyInstanceUID and SeriesInstanceUID (if given,
+        Download the data related to the StudyInstanceUID and SeriesInstanceUID (if given,
         otherwise the entire study will be downloaded).
 
         :param study_uid: The StudyInstanceUID
@@ -333,7 +346,7 @@ def download_data(
             base_dict[self.series_instance_uid_field] = series_uid
 
         # Init the readers/writers
-        series_reader = sitk.ImageSeriesReader()  # type: ignore
+        series_reader = sitk.ImageSeriesReader()
         with tempfile.TemporaryDirectory() as tmp_dir:
             # Create the download dir
             current_tmp_dir = pathlib.Path(tmp_dir)
@@ -361,11 +374,11 @@ def download_data(
             progress_bar.close()
 
             # Get Series ID names from folder
-            series_uids = sitk.ImageSeriesReader.GetGDCMSeriesIDs(str(current_tmp_dir))  # type: ignore
+            series_uids = sitk.ImageSeriesReader.GetGDCMSeriesIDs(str(current_tmp_dir))
             logger.info(f"Study ID has {len(series_uids)} series.")
             for series in series_uids:
                 # Get the DICOMs corresponding to the series
-                files = series_reader.GetGDCMSeriesFileNames(  # type: ignore
+                files = series_reader.GetGDCMSeriesFileNames(
                     str(current_tmp_dir), series
                 )
                 current_dict = base_dict.copy()
@@ -374,11 +387,12 @@ def download_data(
                 )
                 try:
                     # Read the series
-                    with simpleitk_warning_file.open("w") as f, stdout_redirected(
-                        f, stdout=sys.stderr
+                    with (
+                        simpleitk_warning_file.open("w") as f,
+                        stdout_redirected(f, stdout=sys.stderr),
                     ):
-                        series_reader.SetFileNames(files)  # type: ignore
-                        image = series_reader.Execute()  # type: ignore
+                        series_reader.SetFileNames(files)
+                        image = series_reader.Execute()
                     with simpleitk_warning_file.open("r") as f:
                         content = f.read()
                     if "warning" in content.lower():
@@ -431,9 +445,9 @@ def download_data(
                         series_download_dir / f"{series}_meta.dcm",
                     )
                 dcm_info = pydicom.dcmread(str(files[0]), stop_before_pixels=True)
-                current_dict[
-                    self.deid_study_instance_uid_field
-                ] = dcm_info.StudyInstanceUID
+                current_dict[self.deid_study_instance_uid_field] = (
+                    dcm_info.StudyInstanceUID
+                )
                 current_dict[self.deid_series_instance_uid_field] = series
                 downloaded_series_info.append(current_dict)
 
@@ -442,7 +456,7 @@ def download_data(
     def fix_mapping_dataframe(
         self,
         df: pd.DataFrame,
-        mapping_df: pd.DataFrame = None,
+        mapping_df: Optional[pd.DataFrame] = None,
         output_dir: Union[str, pathlib.Path] = "out",
         study_uid_col: str = "study_instance_uid",
         series_uid_col: str = "series_instance_uid",
@@ -464,7 +478,8 @@ def fix_mapping_dataframe(
         output_dir = pathlib.Path(output_dir)
         if not output_dir.exists() or not len(list(output_dir.glob("*"))):
             warnings.warn(
-                "Cannot fix the mapping file if the output directory does not exist."
+                "Cannot fix the mapping file if the output directory does not exist.",
+                stacklevel=2,
             )
             return None
         if mapping_df is None:
@@ -547,7 +562,7 @@ def download_data_from_dataframe(
         output_dir: Union[str, pathlib.Path] = "out",
         study_uid_col: str = "study_instance_uid",
         series_uid_col: Optional[str] = "series_instance_uid",
-        mapping_df: pd.DataFrame = None,
+        mapping_df: Optional[pd.DataFrame] = None,
         download_full_study: bool = False,
         save_metadata: bool = True,
     ) -> Tuple[pd.DataFrame, pd.DataFrame]:
@@ -593,7 +608,8 @@ def download_data_from_dataframe(
             warnings.warn(
                 "download_full_study = False will only download a specified series but "
                 "have not provided a valid Series UID column of the DataFrame, "
-                "as a result the full study will be downloaded."
+                "as a result the full study will be downloaded.",
+                stacklevel=2,
             )
 
         # Create list of rows