Merge pull request #123 from ImageMarkup/score-error

Author: brianhelba

isic_challenge_scoring/__init__.py

@@ -2,9 +2,11 @@
 
 from isic_challenge_scoring.classification import ClassificationMetric, ClassificationScore
 from isic_challenge_scoring.segmentation import SegmentationScore
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
-__all__ = ['ClassificationScore', 'SegmentationScore', 'ScoreException', 'ClassificationMetric']
+ScoreException = ScoreError
+
+__all__ = ['ClassificationScore', 'SegmentationScore', 'ScoreError', 'ClassificationMetric']
 
 try:
     __version__ = version('isic-challenge-scoring')

isic_challenge_scoring/__main__.py

@@ -7,7 +7,7 @@
 
 from isic_challenge_scoring.classification import ClassificationMetric, ClassificationScore
 from isic_challenge_scoring.segmentation import SegmentationScore
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
 DirectoryPath = click_pathlib.Path(exists=True, file_okay=False, dir_okay=True, readable=True)
 FilePath = click_pathlib.Path(exists=True, file_okay=True, dir_okay=False, readable=True)
@@ -26,7 +26,7 @@ def cli(output: str) -> None:
 def segmentation(ctx: click.Context, truth_dir: pathlib.Path, prediction_dir: pathlib.Path) -> None:
     try:
         score = SegmentationScore.from_dir(truth_dir, prediction_dir)
-    except ScoreException as e:
+    except ScoreError as e:
         raise click.ClickException(str(e))
 
     output: str = cast(click.Context, ctx.parent).params['output']
@@ -53,7 +53,7 @@ def classification(
         score = ClassificationScore.from_file(
             truth_file, prediction_file, ClassificationMetric(metric)
         )
-    except ScoreException as e:
+    except ScoreError as e:
         raise click.ClickException(str(e))
 
     output: str = cast(click.Context, ctx.parent).params['output']

isic_challenge_scoring/load_csv.py

@@ -3,7 +3,7 @@
 import numpy as np
 import pandas as pd
 
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
 
 def parse_truth_csv(csv_file_stream: TextIO) -> Tuple[pd.DataFrame, pd.DataFrame]:
@@ -31,19 +31,19 @@ def parse_csv(csv_file_stream: TextIO, categories: pd.Index) -> pd.DataFrame:
         if csv_file_stream.read(2000).count('\n') < 2:
             # Heuristic: if there aren't 2 newlines in the first 2000 characters, it's probably
             # invalid, and we don't want to hang or crash the parser
-            raise ScoreException('No newlines detected in CSV.')
+            raise ScoreError('No newlines detected in CSV.')
         csv_file_stream.seek(0)
 
         try:
             probabilities = pd.read_csv(csv_file_stream, header=0, index_col=False)
         except (pd.errors.ParserError, pd.errors.EmptyDataError) as e:
             # TODO: Test something that generates a ParserError
-            raise ScoreException(f'Could not parse CSV: "{str(e)}".')
+            raise ScoreError(f'Could not parse CSV: "{str(e)}".')
     except UnicodeDecodeError:
-        raise ScoreException('Could not parse CSV: could not decode file as UTF-8.')
+        raise ScoreError('Could not parse CSV: could not decode file as UTF-8.')
 
     if 'image' not in probabilities.columns:
-        raise ScoreException("Missing column in CSV: 'image'.")
+        raise ScoreError("Missing column in CSV: 'image'.")
 
     # Pandas represents strings as 'O' (object)
     if probabilities['image'].dtype != np.dtype('O'):
@@ -57,31 +57,31 @@ def parse_csv(csv_file_stream: TextIO, categories: pd.Index) -> pd.DataFrame:
 
     if not probabilities['image'].is_unique:
         duplicate_images = probabilities['image'][probabilities['image'].duplicated()].unique()
-        raise ScoreException(f'Duplicate image rows detected in CSV: {duplicate_images.tolist()}.')
+        raise ScoreError(f'Duplicate image rows detected in CSV: {duplicate_images.tolist()}.')
 
     # The duplicate check is the same as performed by 'verify_integrity'
     probabilities.set_index('image', drop=True, inplace=True, verify_integrity=False)
 
     missing_columns = categories.difference(probabilities.columns)
     if not missing_columns.empty:
-        raise ScoreException(f'Missing columns in CSV: {missing_columns.tolist()}.')
+        raise ScoreError(f'Missing columns in CSV: {missing_columns.tolist()}.')
 
     extra_columns = probabilities.columns.difference(categories)
     if not extra_columns.empty:
-        raise ScoreException(f'Extra columns in CSV: {extra_columns.tolist()}.')
+        raise ScoreError(f'Extra columns in CSV: {extra_columns.tolist()}.')
 
     # sort by the order in categories
     probabilities = probabilities.reindex(categories, axis='columns')
 
     missing_rows = probabilities[probabilities.isnull().any(axis='columns')].index
     if not missing_rows.empty:
-        raise ScoreException(f'Missing value(s) in CSV for images: {missing_rows.tolist()}.')
+        raise ScoreError(f'Missing value(s) in CSV for images: {missing_rows.tolist()}.')
 
     non_float_columns = probabilities.dtypes[
         probabilities.dtypes.apply(lambda x: x != np.float64)
     ].index
     if not non_float_columns.empty:
-        raise ScoreException(
+        raise ScoreError(
             f'CSV contains non-floating-point value(s) in columns: {non_float_columns.tolist()}.'
         )
     # TODO: identify specific failed rows
@@ -90,7 +90,7 @@ def parse_csv(csv_file_stream: TextIO, categories: pd.Index) -> pd.DataFrame:
         probabilities.applymap(lambda x: x < 0.0 or x > 1.0).any(axis='columns')
     ].index
     if not out_of_range_rows.empty:
-        raise ScoreException(
+        raise ScoreError(
             f'Values in CSV are outside the interval [0.0, 1.0] for images: '
             f'{out_of_range_rows.tolist()}.'
         )
@@ -111,11 +111,11 @@ def validate_rows(
     """
     missing_images = truth_probabilities.index.difference(prediction_probabilities.index)
     if not missing_images.empty:
-        raise ScoreException(f'Missing images in CSV: {missing_images.tolist()}.')
+        raise ScoreError(f'Missing images in CSV: {missing_images.tolist()}.')
 
     extra_images = prediction_probabilities.index.difference(truth_probabilities.index)
     if not extra_images.empty:
-        raise ScoreException(f'Extra images in CSV: {extra_images.tolist()}.')
+        raise ScoreError(f'Extra images in CSV: {extra_images.tolist()}.')
 
 
 def sort_rows(probabilities: pd.DataFrame) -> None:

isic_challenge_scoring/load_image.py

@@ -6,7 +6,7 @@
 from PIL import Image, UnidentifiedImageError
 import numpy as np
 
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
 
 @dataclass
@@ -48,9 +48,9 @@ def find_prediction_file(self, prediction_path: pathlib.Path) -> None:
             ]
 
         if not prediction_file_candidates:
-            raise ScoreException(f'No matching submission for: {self.truth_file.name}')
+            raise ScoreError(f'No matching submission for: {self.truth_file.name}')
         elif len(prediction_file_candidates) > 1:
-            raise ScoreException(f'Multiple matching submissions for: {self.truth_file.name}')
+            raise ScoreError(f'Multiple matching submissions for: {self.truth_file.name}')
 
         self.prediction_file = prediction_file_candidates[0]
 
@@ -62,7 +62,7 @@ def load_truth_image(self) -> None:
     def load_prediction_image(self) -> None:
         self.prediction_image = load_segmentation_image(self.prediction_file)
         if self.prediction_image.shape[0:2] != self.truth_image.shape[0:2]:
-            raise ScoreException(
+            raise ScoreError(
                 f'Image {self.prediction_file.name} has dimensions '
                 f'{self.prediction_image.shape[0:2]}; expected {self.truth_image.shape[0:2]}.'
             )
@@ -81,12 +81,12 @@ def load_segmentation_image(image_path: pathlib.Path) -> np.ndarray:
                 image = image.convert('L')
 
             if image.mode != 'L':
-                raise ScoreException(f'Image {image_path.name} is not single-channel (greyscale).')
+                raise ScoreError(f'Image {image_path.name} is not single-channel (greyscale).')
 
             np_image = np.asarray(image)
 
     except UnidentifiedImageError:
-        raise ScoreException(f'Could not decode image "{image_path.name}"')
+        raise ScoreError(f'Could not decode image "{image_path.name}"')
 
     return np_image
 
@@ -103,9 +103,9 @@ def assert_binary_image(image: np.ndarray, image_path: pathlib.Path) -> np.ndarr
         image /= high_value
         image *= 255
         if set(np.unique(image)) > {0, 255}:
-            raise ScoreException(f'Image {image_path.name} contains values other than 0 and 255.')
+            raise ScoreError(f'Image {image_path.name} contains values other than 0 and 255.')
     else:
-        raise ScoreException(f'Image {image_path.name} contains values other than 0 and 255.')
+        raise ScoreError(f'Image {image_path.name} contains values other than 0 and 255.')
 
     return image
 

isic_challenge_scoring/types.py

@@ -2,7 +2,7 @@
 from typing import Dict, List, Optional, Union
 
 
-class ScoreException(Exception):  # noqa: N818
+class ScoreError(Exception):
     pass
 
 

isic_challenge_scoring/unzip.py

@@ -6,7 +6,7 @@
 
 import zipfile_deflate64 as zipfile
 
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
 
 def extract_zip(zip_path: pathlib.Path, output_path: pathlib.Path, flatten: bool = True) -> None:
@@ -34,7 +34,7 @@ def extract_zip(zip_path: pathlib.Path, output_path: pathlib.Path, flatten: bool
             else:
                 zf.extractall(output_path)
     except zipfile.BadZipfile as e:
-        raise ScoreException(f'Could not read ZIP file "{zip_path.name}": {str(e)}.')
+        raise ScoreError(f'Could not read ZIP file "{zip_path.name}": {str(e)}.')
 
 
 def unzip_all(input_file: pathlib.Path) -> Tuple[pathlib.Path, tempfile.TemporaryDirectory]:

tests/test_load_csv.py

@@ -4,7 +4,7 @@
 import pytest
 
 from isic_challenge_scoring import load_csv
-from isic_challenge_scoring.types import ScoreException
+from isic_challenge_scoring.types import ScoreError
 
 
 def test_parse_truth_csv(categories):
@@ -97,7 +97,7 @@ def test_parse_csv_no_newlines(categories):
         prediction_file_stream.write(f'{i:030f},')
     prediction_file_stream.seek(0)
 
-    with pytest.raises(ScoreException, match=r'^No newlines detected in CSV\.$'):
+    with pytest.raises(ScoreError, match=r'^No newlines detected in CSV\.$'):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -106,7 +106,7 @@ def test_parse_csv_empty(categories):
     prediction_file_stream = io.StringIO('\n\n')
 
     with pytest.raises(
-        ScoreException, match=r'^Could not parse CSV: "No columns to parse from file"\.$'
+        ScoreError, match=r'^Could not parse CSV: "No columns to parse from file"\.$'
     ):
         load_csv.parse_csv(prediction_file_stream, categories)
 
@@ -115,7 +115,7 @@ def test_parse_csv_invalid_unicode(categories):
     prediction_file_stream = io.TextIOWrapper(io.BytesIO(b'\xef'))
 
     with pytest.raises(
-        ScoreException, match=r'^Could not parse CSV: could not decode file as UTF-8\.$'
+        ScoreError, match=r'^Could not parse CSV: could not decode file as UTF-8\.$'
     ):
         load_csv.parse_csv(prediction_file_stream, categories)
 
@@ -129,7 +129,7 @@ def test_parse_csv_mismatched_headers(categories):
     )
 
     # Pandas should drop extra columns without headers, but this is a common invalid case
-    with pytest.raises(ScoreException, match=r'^Missing columns in CSV:'):
+    with pytest.raises(ScoreError, match=r'^Missing columns in CSV:'):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -138,7 +138,7 @@ def test_parse_csv_missing_columns(categories):
         'image,MEL,BCC,AKIEC,BKL,DF\n' 'ISIC_0000123,1.0,0.0,0.0,0.0,0.0\n'
     )
 
-    with pytest.raises(ScoreException, match=r"^Missing columns in CSV: \['NV', 'VASC'\]\.$"):
+    with pytest.raises(ScoreError, match=r"^Missing columns in CSV: \['NV', 'VASC'\]\.$"):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -148,7 +148,7 @@ def test_parse_csv_extra_columns(categories):
         'ISIC_0000123,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0\n'
     )
 
-    with pytest.raises(ScoreException, match=r"^Extra columns in CSV: \['BAZ', 'FOO'\]\.$"):
+    with pytest.raises(ScoreError, match=r"^Extra columns in CSV: \['BAZ', 'FOO'\]\.$"):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -157,7 +157,7 @@ def test_parse_csv_misnamed_columns(categories):
         'image,MEL,FOO,BCC,AKIEC,BKL,BAZ,VASC\n' 'ISIC_0000123,1.0,0.0,0.0,0.0,0.0,0.0,0.0\n'
     )
 
-    with pytest.raises(ScoreException, match=r"^Missing columns in CSV: \['DF', 'NV'\]\.$"):
+    with pytest.raises(ScoreError, match=r"^Missing columns in CSV: \['DF', 'NV'\]\.$"):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -200,7 +200,7 @@ def test_parse_csv_missing_index(categories):
         'MEL,NV,BCC,AKIEC,BKL,DF,VASC\n' '1.0,0.0,0.0,0.0,0.0,0.0,0.0\n'
     )
 
-    with pytest.raises(ScoreException, match=r"^Missing column in CSV: 'image'\.$"):
+    with pytest.raises(ScoreError, match=r"^Missing column in CSV: 'image'\.$"):
         load_csv.parse_csv(prediction_file_stream, categories)
 
 
@@ -224,7 +224,7 @@ def test_parse_csv_missing_values(categories):
     )
 
     with pytest.raises(
-        ScoreException,
+        ScoreError,
         match=r"^Missing value\(s\) in CSV for images: \['ISIC_0000124', 'ISIC_0000125'\]\.$",
     ):
         load_csv.parse_csv(prediction_file_stream, categories)
@@ -239,7 +239,7 @@ def test_parse_csv_non_float_columns(categories):
     )
 
     with pytest.raises(
-        ScoreException,
+        ScoreError,
         match=r"^CSV contains non-floating-point value\(s\) in columns: \['BCC', 'VASC'\]\.$",
     ):
         load_csv.parse_csv(prediction_file_stream, categories)
@@ -254,7 +254,7 @@ def test_parse_csv_out_of_range_values(categories):
     )
 
     with pytest.raises(
-        ScoreException,
+        ScoreError,
         match=r'^Values in CSV are outside the interval \[0\.0, 1\.0\] for images: '
         r"\['ISIC_0000123', 'ISIC_0000125'\]\.$",
     ):
@@ -271,7 +271,7 @@ def test_parse_csv_duplicate_images(categories):
     )
 
     with pytest.raises(
-        ScoreException,
+        ScoreError,
         match=r"^Duplicate image rows detected in CSV: \['ISIC_0000123', 'ISIC_0000124'\]\.$",
     ):
         load_csv.parse_csv(prediction_file_stream, categories)
@@ -298,7 +298,7 @@ def test_validate_rows_missing_images(categories):
     prediction_probabilities = pd.DataFrame(
         [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], index=['ISIC_0000123'], columns=categories
     )
-    with pytest.raises(ScoreException, match=r"^Missing images in CSV: \['ISIC_0000124'\]\.$"):
+    with pytest.raises(ScoreError, match=r"^Missing images in CSV: \['ISIC_0000124'\]\.$"):
         load_csv.validate_rows(truth_probabilities, prediction_probabilities)
 
     truth_probabilities = pd.DataFrame(
@@ -310,7 +310,7 @@ def test_validate_rows_missing_images(categories):
         [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], index=['ISIC_0000120'], columns=categories
     )
     with pytest.raises(
-        ScoreException, match=r"^Missing images in CSV: \['ISIC_0000123', 'ISIC_0000124'\]\.$"
+        ScoreError, match=r"^Missing images in CSV: \['ISIC_0000123', 'ISIC_0000124'\]\.$"
     ):
         load_csv.validate_rows(truth_probabilities, prediction_probabilities)
 
@@ -324,7 +324,7 @@ def test_validate_rows_missing_images(categories):
         index=['ISIC_0000123', 'ISIC_0000125'],
         columns=categories,
     )
-    with pytest.raises(ScoreException, match=r"^Missing images in CSV: \['ISIC_0000124'\]\.$"):
+    with pytest.raises(ScoreError, match=r"^Missing images in CSV: \['ISIC_0000124'\]\.$"):
         load_csv.validate_rows(truth_probabilities, prediction_probabilities)
 
 
@@ -342,7 +342,7 @@ def test_validate_rows_extra_images(categories):
         columns=categories,
     )
     with pytest.raises(
-        ScoreException, match=r"^Extra images in CSV: \['ISIC_0000126', 'ISIC_0000127'\]\.$"
+        ScoreError, match=r"^Extra images in CSV: \['ISIC_0000126', 'ISIC_0000127'\]\.$"
     ):
         load_csv.validate_rows(truth_probabilities, prediction_probabilities)
 

tests/test_load_image.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from isic_challenge_scoring import ScoreException, load_image
+from isic_challenge_scoring import ScoreError, load_image
 
 
 @pytest.mark.parametrize(
@@ -43,5 +43,5 @@ def test_load_segmentation_image_valid(test_images_path, test_image_name):
 @pytest.mark.parametrize('test_image_name', ['empty.png', 'random_bytes.png', 'rgb.png'])
 def test_load_segmentation_image_invalid(test_images_path, test_image_name):
     image_path = test_images_path / test_image_name
-    with pytest.raises(ScoreException):
+    with pytest.raises(ScoreError):
         load_image.load_segmentation_image(image_path)