RWC_popular update to 2.0 (#699)

* updating rwc_popular to version 2.0

* fixing error in index

* removing preview for index

* fixing test folder path

* fixing test track id name

* adding test covering else in metadata

* adding bibitex for vocal-inst activity, section annotation

* remove archive annotations from loader, edit index script

* edit tests/, black formatting

* fix metadata path

* black formatting

* fix-rwc popular test

* black formatting for test

* adding midi to rwc popular

* Move shared RWC utilities to rwc_popular loader

---------

Co-authored-by: Magdalena Fuentes <mgfuenteslujambio@gmail.com>

Author: yujin-kimmm

mirdata/datasets/rwc_popular.py

@@ -0,0 +1,138 @@
+import argparse
+import csv
+import hashlib
+import json
+import os
+
+RWC_POPULAR_INDEX_PATH = "../mirdata/datasets/indexes/rwc_popular_index_2.0.json"
+
+
+def md5(file_path):
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as fhandle:
+        for chunk in iter(lambda: fhandle.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def make_rwc_popular_index(data_path):
+    dataset_root = os.path.join(data_path, "rwc_popular")
+    metadata_file = os.path.join(
+        dataset_root,
+        "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2",
+        "metadata.csv",
+    )
+
+    with open(metadata_file, "r", encoding="utf-8") as fhandle:
+        reader = csv.DictReader(fhandle, delimiter=";")
+        piece = []
+        for row in reader:
+            rwcid = (row.get("RWCID") or "").strip()
+            if not rwcid.startswith("RWC_P"):
+                continue
+
+            piece.append(rwcid[-3:])
+
+    track_ids = sorted([f"RWC_P{p}" for p in piece])
+
+    rwc_popular_index = {"version": "2.0", "tracks": {}, "metadata": {}}
+    for track_id in track_ids:
+        pid = track_id[-3:]
+        rm_id = f"RM-P{pid}"
+
+        audio_rel = os.path.join("RWC-P", f"{track_id}.wav")
+        audio_abs = os.path.join(dataset_root, audio_rel)
+        if not os.path.exists(audio_abs):
+            raise FileNotFoundError(f"Audio file not found: {audio_abs}")
+        audio_rel = os.path.relpath(audio_abs, dataset_root).replace("\\", "/")
+        audio_checksum = md5(audio_abs)
+
+        beats_rel = os.path.join(
+            "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2",
+            "01_annotations_preprocessed",
+            "beats",
+            "RWC-P",
+            f"{track_id}.csv",
+        )
+        beats_abs = os.path.join(dataset_root, beats_rel)
+        if os.path.exists(beats_abs):
+            beats_idx = beats_rel.replace("\\", "/")
+            beats_md5 = md5(beats_abs)
+        else:
+            beats_idx, beats_md5 = None, None
+
+        chords_rel = os.path.join(
+            "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2",
+            "01_annotations_preprocessed",
+            "chords",
+            "RWC-P",
+            f"{track_id}.csv",
+        )
+        chords_abs = os.path.join(dataset_root, chords_rel)
+        if os.path.exists(chords_abs):
+            chords_idx = chords_rel.replace("\\", "/")
+            chords_md5 = md5(chords_abs)
+        else:
+            chords_idx, chords_md5 = None, None
+
+        melody_rel = os.path.join(
+            "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2",
+            "01_annotations_preprocessed",
+            "melody",
+            "RWC-P",
+            f"{track_id}.csv",
+        )
+
+        melody_abs = os.path.join(dataset_root, melody_rel)
+        if os.path.exists(melody_abs):
+            melody_idx = melody_rel.replace("\\", "/")
+            melody_md5 = md5(melody_abs)
+        else:
+            melody_idx, melody_md5 = None, None
+
+        midi_rel = os.path.join(
+            "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2",
+            "01_annotations_preprocessed",
+            "MIDI_aligned",
+            "RWC-P",
+            f"{track_id}.mid",
+        )
+        midi_abs = os.path.join(dataset_root, midi_rel)
+        if os.path.exists(midi_abs):
+            midi_idx = midi_rel.replace("\\", "/")
+            midi_md5 = md5(midi_abs)
+        else:
+            midi_idx, midi_md5 = None, None
+
+        rwc_popular_index["tracks"][track_id] = {
+            "audio": (audio_rel, audio_checksum),
+            "beats": (beats_idx, beats_md5),
+            "chords": (chords_idx, chords_md5),
+            "melody": (melody_idx, melody_md5),
+            "midi": (midi_idx, midi_md5),
+        }
+
+    metadata_rel = os.path.relpath(metadata_file, dataset_root).replace("\\", "/")
+    rwc_popular_index["metadata"]["rwc-popular-metadata"] = (
+        metadata_rel,
+        md5(metadata_file),
+    )
+
+    with open(RWC_POPULAR_INDEX_PATH, "w", encoding="utf-8") as fhandle:
+        json.dump(rwc_popular_index, fhandle, indent=2)
+
+    print(f"Wrote {len(track_ids)} tracks to {RWC_POPULAR_INDEX_PATH}")
+
+
+def main(args):
+    make_rwc_popular_index(args.data_path)
+
+
+if __name__ == "__main__":
+    PARSER = argparse.ArgumentParser(description="Make RWC-Popular index file.")
+    PARSER.add_argument(
+        "--data_path",
+        type=str,
+        help="Path to parent directory of rwc_popular.",
+    )
+    main(PARSER.parse_args())

scripts/make_rwc_popular_index.py

@@ -1,56 +1,65 @@
 import os
 import numpy as np
+import pretty_midi
 
 from mirdata.datasets import rwc_popular
 from mirdata import annotations
 from tests.test_utils import run_track_tests
 
 
 def test_track():
-    default_trackid = "RM-P001"
+    default_trackid = "RWC_P001"
     data_home = os.path.normpath("tests/resources/mir_datasets/rwc_popular")
     dataset = rwc_popular.Dataset(data_home, version="test")
     track = dataset.track(default_trackid)
 
     expected_attributes = {
-        "track_id": "RM-P001",
+        "track_id": "RWC_P001",
         "audio_path": os.path.join(
             os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
-            "audio/rwc-p-m01/1.wav",
-        ),
-        "sections_path": os.path.join(
-            os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
-            "annotations/AIST.RWC-MDB-P-2001.CHORUS/RM-P001.CHORUS.TXT",
+            "RWC-P/1.wav",
         ),
         "beats_path": os.path.join(
             os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
-            "annotations/AIST.RWC-MDB-P-2001.BEAT/RM-P001.BEAT.TXT",
+            "rwc-annotations-main/01_annotations_preprocessed/beats/RWC-P/RWC_P001.csv",
         ),
         "chords_path": os.path.join(
             os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
-            "annotations/AIST.RWC-MDB-P-2001.CHORD/RWC_Pop_Chords/N001-M01-T01.lab",
+            "rwc-annotations-main/01_annotations_preprocessed/chords/RWC-P/RWC_P001.csv",
+        ),
+        "f0_path": os.path.join(
+            os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
+            "rwc-annotations-main/01_annotations_preprocessed/melody/RWC-P/RWC_P001.csv",
         ),
-        "voca_inst_path": os.path.join(
+        "midi_path": os.path.join(
             os.path.normpath("tests/resources/mir_datasets/rwc_popular/"),
-            "annotations/AIST.RWC-MDB-P-2001.VOCA_INST/RM-P001.VOCA_INST.TXT",
+            "rwc-annotations-main/01_annotations_preprocessed/MIDI_aligned/RWC-P/RWC_P001.mid",
         ),
-        "piece_number": "No. 1",
-        "suffix": "M01",
-        "track_number": "Tr. 01",
+        "piece_number": "1",
+        "cd_number": "1",
+        "track_number": "1",
         "title": "Eien no replica",
         "artist": "Kazuo Nishi",
         "singer_information": "Male",
-        "duration": 209,
-        "tempo": "135",
-        "instruments": "Gt",
+        "singing_language": "Japanese",
+        "tempo": "135.0",
+        "variation": "",
+        "live_instrument": "Gt",
         "drum_information": "Drum sequences",
+        "composer": "",
+        "composition_type": "",
+        "main_genre": "Popular",
+        "sub_genre": "J-pop",
+        "audio_start": "0.0236",
+        "audio_end": "204.4778",
+        "duration": "207.16829931972788",
     }
 
     expected_property_types = {
         "beats": annotations.BeatData,
-        "sections": annotations.SectionData,
         "chords": annotations.ChordData,
-        "vocal_instrument_activity": annotations.EventData,
+        "melody": annotations.F0Data,
+        "midi": pretty_midi.PrettyMIDI,
         "audio": tuple,
     }
 
@@ -65,7 +74,7 @@ def test_track():
 def test_load_chords():
     chords_path = (
         "tests/resources/mir_datasets/rwc_popular/"
-        + "annotations/AIST.RWC-MDB-P-2001.CHORD/RWC_Pop_Chords/N001-M01-T01.lab"
+        + "rwc-annotations-main/01_annotations_preprocessed/chords/RWC-P/RWC_P001.csv"
     )
     chord_data = rwc_popular.load_chords(chords_path)
 
@@ -75,81 +84,98 @@ def test_load_chords():
     assert type(chord_data.labels) is list
 
     # check values
-    assert np.array_equal(
-        chord_data.intervals[:, 0], np.array([0.000, 0.104, 3.646, 43.992, 44.494])
-    )
-    assert np.array_equal(
-        chord_data.intervals[:, 1], np.array([0.104, 1.858, 5.387, 44.494, 47.636])
+    assert np.array_equal(chord_data.intervals[:3, 0], np.array([0.0, 0.104, 1.858]))
+    assert np.array_equal(chord_data.intervals[:3, 1], np.array([0.104, 1.858, 3.646]))
+    assert chord_data.labels[:3] == ["N", "Ab:min", "Gb:maj"]
+
+
+def test_load_beats():
+    beats_path = (
+        "tests/resources/mir_datasets/rwc_popular/"
+        + "rwc-annotations-main/01_annotations_preprocessed/beats/RWC-P/RWC_P001.csv"
     )
+    beat_data = rwc_popular.load_beats(beats_path)
+
+    # check types
+    assert type(beat_data) is annotations.BeatData
+    assert type(beat_data.times) is np.ndarray
+    assert type(beat_data.positions) is np.ndarray
+
+    # check values
     assert np.array_equal(
-        chord_data.labels, ["N", "Ab:min", "E:maj", "Bb:maj(*3)", "C:min7"]
+        beat_data.times[:5], np.array([0.060, 0.510, 0.950, 1.390, 1.840])
     )
+    assert np.array_equal(beat_data.positions[:5], np.array([1, 2, 3, 4, 1]))
 
 
-def test_load_vocal_activity():
-    vocinst_path = (
+def test_load_melody():
+    melody_path = (
         "tests/resources/mir_datasets/rwc_popular/"
-        + "annotations/AIST.RWC-MDB-P-2001.VOCA_INST/RM-P001.VOCA_INST.TXT"
+        + "rwc-annotations-main/01_annotations_preprocessed/melody/RWC-P/RWC_P001.csv"
     )
-    vocinst_data = rwc_popular.load_vocal_activity(vocinst_path)
+    melody_data = rwc_popular.load_melody(melody_path)
 
     # check types
-    assert type(vocinst_data) is annotations.EventData
-    assert type(vocinst_data.intervals) is np.ndarray
-    assert type(vocinst_data.events) is list
+    assert type(melody_data) is annotations.F0Data
+    assert type(melody_data.times) is np.ndarray
+    assert type(melody_data.frequencies) is np.ndarray
+    assert type(melody_data.voicing) is np.ndarray
 
     # check values
     assert np.array_equal(
-        vocinst_data.intervals[:, 0],
-        np.array(
-            [
-                0.000,
-                10.293061224,
-                11.883492063,
-                12.087845804,
-                13.587460317,
-                13.819387755,
-                20.668707482,
-                20.832653061,
-            ]
-        ),
+        melody_data.times[:5], np.array([0.0, 0.01, 0.02, 0.03, 0.04])
     )
     assert np.array_equal(
-        vocinst_data.intervals[:, 1],
-        np.array(
-            [
-                10.293061224,
-                11.883492063,
-                12.087845804,
-                13.587460317,
-                13.819387755,
-                20.668707482,
-                20.832653061,
-                26.465306122,
-            ]
-        ),
-    )
-    assert np.array_equal(
-        vocinst_data.events,
-        np.array(
-            ["b", "m:withm", "b", "m:withm", "b", "m:withm", "b", "s:electricguitar"]
-        ),
+        melody_data.frequencies[:5], np.array([0.0, 0.0, 0.0, 0.0, 0.0])
     )
+    assert np.array_equal(melody_data.voicing[:5], np.array([0.0, 0.0, 0.0, 0.0, 0.0]))
+    assert len(melody_data.times) == 200
+    assert melody_data.times[-1] == 1.99
+
+    # check that times are uniform (difference of 0.01s for 10ms frames)
+    time_diffs = np.diff(melody_data.times)
+    assert np.allclose(time_diffs, 0.01, atol=1e-6)
 
 
 def test_load_metadata():
     data_home = "tests/resources/mir_datasets/rwc_popular"
     dataset = rwc_popular.Dataset(data_home, version="test")
     metadata = dataset._metadata
-    assert metadata["RM-P001"] == {
-        "piece_number": "No. 1",
-        "suffix": "M01",
-        "track_number": "Tr. 01",
-        "title": "Eien no replica",
-        "artist": "Kazuo Nishi",
-        "singer_information": "Male",
-        "duration": 209,
-        "tempo": "135",
-        "instruments": "Gt",
-        "drum_information": "Drum sequences",
-    }
+
+    assert "RWC_P001" in metadata
+    assert metadata["RWC_P001"]["piece_number"] == "1"
+    assert metadata["RWC_P001"]["cd_number"] == "1"
+    assert metadata["RWC_P001"]["track_number"] == "1"
+    assert metadata["RWC_P001"]["title"] == "Eien no replica"
+    assert metadata["RWC_P001"]["artist"] == "Kazuo Nishi"
+    assert metadata["RWC_P001"]["singer_information"] == "Male"
+    assert metadata["RWC_P001"]["singing_language"] == "Japanese"
+    assert metadata["RWC_P001"]["tempo"] == "135.0"
+    assert metadata["RWC_P001"]["live_instrument"] == "Gt"
+    assert metadata["RWC_P001"]["drum_information"] == "Drum sequences"
+    assert metadata["RWC_P001"]["main_genre"] == "Popular"
+    assert metadata["RWC_P001"]["sub_genre"] == "J-pop"
+    assert metadata["RWC_P001"]["duration"] == "207.16829931972788"
+
+
+def test_load_metadata_else(tmp_path):
+    data_home = tmp_path / "rwc_popular"
+    metadata_dir = (
+        data_home / "rwc-annotations-2b84581b0c4c80514aadf7e9025a309c91e02cc2"
+    )
+    metadata_dir.mkdir(parents=True)
+    metadata_file = metadata_dir / "metadata.csv"
+
+    metadata_file.write_text(
+        "RWCID;CollID;PieceNo;CDNo;TrackNo;Title;Artist;SingerInformation;"
+        "SingingLanguage;Tempo;Variation;LiveInstruments;DrumInformation;"
+        "Composer;CompositionType;GenreMain;GenreSub;audio_start;audio_end;duration\n"
+        "RWC_R001;C;1;1;1;Test;Test Artist;Female;Japanese;120.0;"
+        ";Gt;Drum sequences;;;Popular;J-pop;0.1;10.0;10.0\n",
+        encoding="utf-8",
+    )
+
+    dataset = rwc_popular.Dataset(str(data_home), version="test")
+    metadata = dataset._metadata
+
+    assert metadata == {}