Eval/tts multilingual parakeet metrics (#15826)

* [TTS] Add Japanese Katakana CER metric to MagpieTTS eval

Adds a reading-based CER for Japanese, computed on the Katakana reading
(via pyopenjtalk g2p) of both reference and ASR hypothesis. Robust to
kanji/kana spelling differences that inflate raw character CER.

- text_to_katakana(): lazy-imports pyopenjtalk, returns '' if unavailable
  (graceful no-op for non-ja or environments without the dep).
- katakana_cer / gt_katakana / pred_katakana computed only when language=='ja',
  saved per-utterance in filewise metrics.
- katakana_cer_filewise_avg + katakana_cer_cumulative aggregated globally
  (only emitted for ja datasets), added to the results CSV header/rows.

Signed-off-by: quanpham <youngkwan199@gmail.com>

* feat(tts): add multilingual eval text processors

Signed-off-by: quanpham <youngkwan199@gmail.com>

* refactor(tts): reuse shared eval text processors

Signed-off-by: quanpham <youngkwan199@gmail.com>

* Address Japanese text processor review comments

Signed-off-by: quanpham <youngkwan199@gmail.com>

* Fix MagpieTTS evaluation formatting

Signed-off-by: quanpham <youngkwan199@gmail.com>

---------

Signed-off-by: quanpham <youngkwan199@gmail.com>

Author: quapham

examples/tts/magpietts_inference.py

@@ -134,6 +134,8 @@ def append_metrics_to_csv(csv_path: str, checkpoint_name: str, dataset: str, met
         metrics.get('eou_silence_rate', ''),
         metrics.get('eou_noise_rate', ''),
         metrics.get('eou_error_rate', ''),
+        metrics.get('katakana_cer_filewise_avg', ''),
+        metrics.get('katakana_cer_cumulative', ''),
     ]
     with open(csv_path, "a") as f:
         f.write(",".join(str(v) for v in values) + "\n")
@@ -230,7 +232,8 @@ def run_inference_and_evaluation(
         "ssim_pred_gt_avg_alternate,ssim_pred_context_avg_alternate,"
         "ssim_gt_context_avg_alternate,cer_gt_audio_cumulative,wer_gt_audio_cumulative,"
         "utmosv2_avg,total_gen_audio_seconds,frechet_codec_distance,"
-        "eou_cutoff_rate,eou_silence_rate,eou_noise_rate,eou_error_rate"
+        "eou_cutoff_rate,eou_silence_rate,eou_noise_rate,eou_error_rate,"
+        "katakana_cer_filewise_avg,katakana_cer_cumulative"
     )
 
     for dataset in datasets:

nemo/collections/tts/modules/magpietts_inference/evaluate_generated_audio.py

@@ -36,6 +36,7 @@
 from nemo.collections.tts.metrics.eou_classifier import EoUClassification, EoUClassifier, EoUType
 from nemo.collections.tts.metrics.frechet_codec_distance import FrechetCodecDistance
 from nemo.collections.tts.parts.utils.tts_dataset_utils import (
+    JapaneseTextProcessor,
     NemoTranscriber,
     NemoTranscriberWithPrompt,
     WhisperTranscriber,
@@ -57,11 +58,18 @@
     )
 
 
+KATAKANA_METRICS_TO_SAVE = [
+    'katakana_cer',
+    'gt_katakana',
+    'pred_katakana',
+]
+
 FILEWISE_METRICS_TO_SAVE = [
     'cer',
     'wer',
     'pred_context_ssim',
     'pred_text',
+    'gt_audio_text',
     'gt_text',
     'gt_audio_filepath',
     'pred_audio_filepath',
@@ -408,6 +416,17 @@ def evaluate_dir(
         detailed_cer = word_error_rate_detail(hypotheses=[pred_text], references=[gt_text], use_cer=True)
         detailed_wer = word_error_rate_detail(hypotheses=[pred_text], references=[gt_text], use_cer=False)
 
+        # Japanese: additional reading-based CER on Katakana (pyopenjtalk g2p), robust to
+        # kanji/kana spelling differences between reference and ASR hypothesis.
+        gt_katakana = pred_katakana = None
+        katakana_cer = None
+        if isinstance(text_processor, JapaneseTextProcessor):
+            gt_katakana = text_processor.text_to_katakana(gt_text)
+            pred_katakana = text_processor.text_to_katakana(pred_text)
+            katakana_cer = word_error_rate_detail(hypotheses=[pred_katakana], references=[gt_katakana], use_cer=True)[
+                0
+            ]
+
         logging.info(f"{ridx} GT Text: {gt_text}")
         logging.info(f"{ridx} Pr Text: {pred_text}")
         # Format cer and wer to 2 decimal places
@@ -502,6 +521,9 @@ def evaluate_dir(
                 'detailed_wer': detailed_wer,
                 'cer': detailed_cer[0],
                 'wer': detailed_wer[0],
+                'katakana_cer': katakana_cer,
+                'gt_katakana': gt_katakana,
+                'pred_katakana': pred_katakana,
                 'pred_gt_ssim': pred_gt_ssim,
                 'pred_context_ssim': pred_context_ssim,
                 'gt_context_ssim': gt_context_ssim,
@@ -593,7 +615,11 @@ def evaluate(
     elapsed = time.time() - start_time
     logging.info(f"evaluate() completed in {elapsed:.1f}s ({elapsed / 60:.1f} min)")
 
-    filtered_filewise = [{k: m[k] for k in FILEWISE_METRICS_TO_SAVE if k in m} for m in filewise_metrics]
+    filewise_metrics_to_save = list(FILEWISE_METRICS_TO_SAVE)
+    if (language or "").replace("_", "-").lower().split("-")[0] == "ja":
+        filewise_metrics_to_save[2:2] = KATAKANA_METRICS_TO_SAVE
+
+    filtered_filewise = [{k: m[k] for k in filewise_metrics_to_save if k in m} for m in filewise_metrics]
     return avg_metrics, filtered_filewise
 
 
@@ -651,6 +677,15 @@ def compute_global_metrics(
     avg_metrics['wer_cumulative'] = word_error_rate_detail(hypotheses=pred_texts, references=gt_texts, use_cer=False)[
         0
     ]
+    # Japanese reading-based CER (Katakana via pyopenjtalk); only present for ja datasets.
+    kata = [m for m in filewise_metrics if m.get('katakana_cer') is not None]
+    if kata:
+        avg_metrics['katakana_cer_filewise_avg'] = sum(m['katakana_cer'] for m in kata) / len(kata)
+        avg_metrics['katakana_cer_cumulative'] = word_error_rate_detail(
+            hypotheses=[m['pred_katakana'] for m in kata],
+            references=[m['gt_katakana'] for m in kata],
+            use_cer=True,
+        )[0]
     avg_metrics['ssim_pred_gt_avg'] = sum(m['pred_gt_ssim'] for m in filewise_metrics) / n
     avg_metrics['ssim_pred_context_avg'] = sum(m['pred_context_ssim'] for m in filewise_metrics) / n
     avg_metrics['ssim_gt_context_avg'] = sum(m['gt_context_ssim'] for m in filewise_metrics) / n

nemo/collections/tts/parts/utils/tts_dataset_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import functools
+import importlib
 import logging
 import os
 import random
@@ -944,6 +945,50 @@ def process_text_for_wer(self, text: str) -> str:
         return text
 
 
+class NoSpaceTextProcessor(TextProcessor):
+    """WER text processor for languages where ASR/tokenization spaces should be ignored."""
+
+    def __init__(self):
+        super().__init__()
+        self.default_processor = DefaultTextProcessor()
+
+    def normalize_text(self, text: str) -> str:
+        return text
+
+    def process_text_for_wer(self, text: str) -> str:
+        text = self.default_processor.process_text_for_wer(text)
+        text = text.replace(" ", "")
+        return text
+
+
+class JapaneseTextProcessor(NoSpaceTextProcessor):
+    """Japanese WER text processor with Katakana reading conversion."""
+
+    def __init__(self):
+        super().__init__()
+        try:
+            self.pyopenjtalk = importlib.import_module("pyopenjtalk")
+        except ImportError as e:
+            raise ImportError(
+                "JapaneseTextProcessor requires pyopenjtalk for Katakana CER computation. "
+                "Install pyopenjtalk or do not request Japanese evaluation text processing."
+            ) from e
+
+    def text_to_katakana(self, text: str) -> str:
+        """Convert Japanese text to its Katakana reading via pyopenjtalk.
+
+        Used for an additional, reading-based Japanese CER metric that is robust to
+        kanji/kana spelling variation between the reference and the ASR hypothesis.
+        """
+        if not text:
+            return ""
+        try:
+            return self.pyopenjtalk.g2p(text, kana=True).strip()
+        except Exception as e:  # noqa: BLE001
+            logging.warning(f"pyopenjtalk failed for '{text[:40]}': {e}")
+            return ""
+
+
 class EnglishTextProcessor(TextProcessor):
     """English text processing, which catches some edge cases not covered by normal text normalization.
 
@@ -980,8 +1025,13 @@ def process_text_for_wer(self, text: str) -> str:
 
 
 def get_text_processor(language: str) -> TextProcessor:
+    language = (language or "").replace("_", "-").lower().split("-")[0]
     if language == "en":
         return EnglishTextProcessor()
+    if language == "ja":
+        return JapaneseTextProcessor()
+    elif language == "zh":
+        return NoSpaceTextProcessor()
     else:
         logging.info(f"Text processing not implemented for language {language}; using default processor")
         return DefaultTextProcessor()