Remove chardet, pytest-pep8 from dependencies, update music21 and fix haydn_op20 loader

.github/environment-ci.yml

@@ -21,7 +21,6 @@ dependencies:
 
   # optional, but required for testing
   - pytest>=7.4.3
-  - pytest-pep8>=1.0.6
   - pytest-cov>=4.1.0
   - pytest-mock>=3.11.1
   - pytest-localserver>=0.8.1
@@ -39,4 +38,4 @@ dependencies:
     - smart_open[all]>=6.4.0
     - testcontainers>=3.7.1
     - mir_eval>=0.7
-    - moviepy>=2.2.1
+    - moviepy>=2.2.1

docs/requirements.txt

@@ -1,9 +1,8 @@
-chardet>=5.0.0
 dali-dataset>=1.0
 Deprecated>=1.2.13
 librosa>=0.10.1
 numpy>=1.21.6
 sphinx>=5.2.0
 sphinx-togglebutton>=0.3.2
 sphinx-rtd-theme>=1.3.0
-tqdm>=4.66.1
+tqdm>=4.66.1

mirdata/datasets/haydn_op20.py

@@ -141,7 +141,9 @@ def _split_score_annotations(fhandle: TextIO):
     """
     score = music21.converter.parse(fhandle.name, format="humdrum")
 
-    rna = {rn.offset: rn for rn in list(score.flat.getElementsByClass("RomanNumeral"))}
+    rna = {
+        rn.offset: rn for rn in list(score.flatten().getElementsByClass("RomanNumeral"))
+    }
     score.remove(list(rna.values()), recurse=True)
     rna_clean = [(offset, rn) for offset, rn in rna.items() if rn]
     return score, rna_clean

mirdata/download_utils.py

@@ -1,6 +1,5 @@
 """Utilities for downloading from the web."""
 
-import chardet
 import glob
 import logging
 import os
@@ -341,23 +340,21 @@ def extractall_unicode(zfile, out_dir):
 
     for m in zfile.infolist():
         data = zfile.read(m)  # extract zipped data into memory
-
         filename = m.filename
 
-        # if block to deal with irmas and good-sounds archives
-        # check if the zip archive does not have the encoding info set
-        # encode-decode filename only if it's different than the original name
-        if (m.flag_bits & ZIP_FILENAME_UTF8_FLAG == 0) and filename.encode(
-            "cp437"
-        ).decode(errors="ignore") != filename:
-            filename_bytes = filename.encode("cp437")
-            if filename_bytes.decode("utf-8", "replace") != filename_bytes.decode(
-                errors="ignore"
-            ):
-                guessed_encoding = chardet.detect(filename_bytes)["encoding"] or "utf8"
-                filename = filename_bytes.decode(guessed_encoding, "replace")
-            else:
-                filename = filename_bytes.decode("utf-8", "replace")
+        # Some dataset archives, including IRMAS and GOOD-SOUNDS, store UTF-8
+        # filenames without setting the ZIP UTF-8 flag.
+        if m.flag_bits & ZIP_FILENAME_UTF8_FLAG == 0:
+            try:
+                filename_bytes = filename.encode("cp437")
+            except UnicodeEncodeError:
+                filename_bytes = None
+
+            if filename_bytes is not None:
+                try:
+                    filename = filename_bytes.decode("utf-8")
+                except UnicodeDecodeError:
+                    pass
 
         disk_file_name = os.path.join(out_dir, filename)
 

pyproject.toml

@@ -22,7 +22,6 @@ classifiers = [
     "Programming Language :: Python :: 3.11"
 ]
 dependencies = [
-    "chardet>=5.0.0",
     "Deprecated>=1.2.14",
     "h5py>=3.7.0",
     "librosa>=0.10.1",
@@ -45,14 +44,12 @@ tests = [
     "attrs>=23.1.0",
     "pytest>=7.2.0",
     "pytest-cov>=4.1.0",
-    "pytest-pep8>=1.0.6",
     "pytest-mock>=3.10.0",
     "pytest-localserver>=0.7.1",
     "testcontainers>=2.3",
     "future>=0.18.3",
     "coveralls>=3.3.1",
     "types-PyYAML",
-    "types-chardet",
     "black>=23.3.0",
     "flake8>=5.0.4",
     "mypy>=0.982",
@@ -71,8 +68,8 @@ docs = [
 compmusic_hindustani_rhythm = ["openpyxl==3.0.10"]
 dali = ["dali-dataset==1.1"]
 compmusic_carnatic_rhythm = ["openpyxl==3.0.10"]
-haydn_op20 = ["music21==6.7.1"]
-cipi = ["music21==6.7.1"]
+haydn_op20 = ["music21>=7.3.3"]
+cipi = ["music21>=7.3.3"]
 gcs = ["smart_open[gcs]"]
 s3 = ["smart_open[s3]"]
 http = ["smart_open[http]"]

tests/test_download_utils.py

@@ -3,7 +3,6 @@
 from pathlib import Path
 import shutil
 import zipfile
-import re
 
 from mirdata import download_utils, core
 
@@ -504,34 +503,47 @@ def test_download_tar_file_ignorechecksum(
     _clean("a")
 
 
-def test_extractall_unicode(mocker, mock_download_from_remote, mock_unzip):
+def test_extractall_unicode(tmp_path, mocker, mock_download_from_remote, mock_unzip):
     zip_files = ("tests/resources/utfissue.zip", "tests/resources/utfissuewin.zip")
     expected_files_all = (
         ["pic👨‍👩‍👧‍👦🎂.jpg", "Benoît.txt", "Icon"],
-        ["pic👨‍👩‍👧‍👦🎂.jpg", "BenoŒt.txt", "Icon"],
+        ["pic👨‍👩‍👧‍👦🎂.jpg", "Benoît.txt", "Icon"],
     )
     for zipf, expected_files in zip(zip_files, expected_files_all):
         zfile = zipfile.ZipFile(zipf, "r")
-        download_utils.extractall_unicode(zfile, os.path.dirname("tests/resources/"))
+        download_utils.extractall_unicode(zfile, str(tmp_path))
         zfile.close()
         for expected_file in expected_files:
             expected_file_location = os.path.join(
-                "tests", "resources", "utfissue", expected_file
+                str(tmp_path), "utfissue", expected_file
             )
             assert os.path.exists(expected_file_location)
-            os.remove(expected_file_location)
 
 
-def test_extractall_cp437(mocker, mock_download_from_remote, mock_unzip):
+def test_extractall_unicode_keeps_uncodable_filename(tmp_path, mocker):
+    member = mocker.Mock()
+    member.filename = "plainĀ.txt"
+    member.flag_bits = 0
+
+    zfile = mocker.Mock()
+    zfile.infolist.return_value = [member]
+    zfile.read.return_value = b"content"
+
+    download_utils.extractall_unicode(zfile, str(tmp_path))
+
+    expected_file_location = os.path.join(str(tmp_path), "plainĀ.txt")
+    assert os.path.exists(expected_file_location)
+    assert Path(expected_file_location).read_bytes() == b"content"
+
+
+def test_extractall_cp437(tmp_path, mocker, mock_download_from_remote, mock_unzip):
     zfile = zipfile.ZipFile("tests/resources/utfissue.zip", "r")
-    zfile.extractall(os.path.dirname("tests/resources/"))
+    zfile.extractall(str(tmp_path))
     zfile.close()
     expected_files = ["pic👨‍👩‍👧‍👦🎂.jpg", "Benoît.txt", "Icon"]
     for expected_file in expected_files:
-        expected_file_location = os.path.join("tests", "resources", expected_file)
+        expected_file_location = os.path.join(str(tmp_path), expected_file)
         assert not os.path.exists(expected_file_location)
-    shutil.rmtree(os.path.join("tests", "resources", "__MACOSX"))
-    shutil.rmtree(os.path.join("tests", "resources", "utfissue"))
 
 
 def test_index_duplicate_prevention(mocker, mock_path):