Skip to content

Commit 1f216a5

Browse files
clean up process text for cer function
Signed-off-by: Paarth Neekhara <paarth.n@gmail.com>
1 parent a93dde1 commit 1f216a5

1 file changed

Lines changed: 12 additions & 24 deletions

File tree

nemo/collections/tts/parts/utils/helpers.py

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -808,30 +808,18 @@ def g2p_backward_compatible_support(g2p_target: str) -> str:
808808

809809

810810
def process_text_for_cer(input_text):
811-
"""
812-
Normalizes text for CER/WER calculation.
813-
"""
814-
# Convert text to lowercase
815-
lower_case_text = input_text.lower()
816-
817-
# Remove commas from text
818-
no_comma_text = lower_case_text.replace(",", "")
819-
# Replace "-" with spaces
820-
no_dash_text = no_comma_text.replace("-", " ")
821-
no_dash_text = no_dash_text.replace("'", "")
822-
no_dash_text = no_dash_text.replace(";", "")
823-
no_dash_text = no_dash_text.replace(".", "")
824-
825-
# Replace double spaces with single space
826-
single_space_text = " ".join(no_dash_text.split())
827-
828-
single_space_text = single_space_text.translate(str.maketrans('', '', string.punctuation))
829-
830-
# Handle some common errors in ASR transcripts
831-
single_space_text = single_space_text.replace("h t t p", "http")
832-
single_space_text = single_space_text.replace("w w w", "www")
833-
834-
return single_space_text
811+
"""Normalizes text for CER/WER calculation."""
812+
text = input_text.lower()
813+
for char in [",", "'", ";", "."]:
814+
text = text.replace(char, "")
815+
text = text.replace("-", " ")
816+
text = " ".join(text.split())
817+
# Strip any remaining punctuation characters
818+
text = text.translate(str.maketrans('', '', string.punctuation))
819+
# Fix common ASR transcript artifacts
820+
text = text.replace("h t t p", "http")
821+
text = text.replace("w w w", "www")
822+
return text
835823

836824

837825
def transcribe_with_whisper(

0 commit comments

Comments
 (0)