TTS Testing Fix: Make Runs more Deterministic

examples/tts/magpietts_inference.py

@@ -45,12 +45,14 @@
 import copy
 import json
 import os
+import random
 import shutil
 from dataclasses import fields
 from pathlib import Path
 from typing import List, Optional, Tuple
 
 import numpy as np
+import torch
 
 from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
 from nemo.collections.tts.models.magpietts import ModelInferenceParameters
@@ -369,13 +371,29 @@ def run_inference_and_evaluation(
     return None, None
 
 
+def seed_all(seed: int):
+    """
+    Attempts to make script deterministic
+    """
+    torch.manual_seed(seed)
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.backends.cudnn.benchmark = False
+    torch.use_deterministic_algorithms(True)
+
+
 def create_argument_parser() -> argparse.ArgumentParser:
     """Create the CLI argument parser."""
     parser = argparse.ArgumentParser(
         description='MagpieTTS Inference and Evaluation',
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=__doc__,
     )
+    parser.add_argument(
+        '--deterministic',
+        action='store_true',
+        help='Attempts to make results deterministic to the best that can be done. Used for testing',
+    )
 
     # Model loading arguments
     model_group = parser.add_argument_group('Model Loading')
@@ -515,6 +533,8 @@ def main(argv=None):
     """
     parser = create_argument_parser()
     args = parser.parse_args(argv)
+    if args.deterministic:
+        seed_all(seed=9)
 
     dataset_meta_info = load_evalset_config(args.datasets_json_path)
     datasets = filter_datasets(dataset_meta_info, args.datasets)

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_FrameStacking.sh

@@ -15,6 +15,7 @@
 # Tests a 4x-stacked model with local transformer inference.
 
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \
     --datasets an4_val_ci \
@@ -28,6 +29,6 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --checkpoint_files /home/TestData/tts/2602_FrameStacking4x/frame-stacking-4x-english-nanocodec.ckpt \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.12 \
-    --ssim_target 0.6 \
+    --cer_target 0.07 \
+    --ssim_target 0.66 \
     --use_local_transformer

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_MoE_ZeroShot.sh

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --nemo_files "/home/TestData/tts/2602_MoE/moe16_sinkhorn_top1_valLoss5.0469_step2625132_epoch524.nemo" \
     --codecmodel_path "/home/TestData/tts/21fps_causal_codecmodel.nemo" \
     --datasets_json_path "examples/tts/evalset_config.json" \
@@ -24,5 +25,5 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --apply_attention_prior \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.08 \
-    --ssim_target 0.72
+    --cer_target 0.07 \
+    --ssim_target 0.73

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_SeenSpeakers.sh

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \
     --datasets an4_val_ci \
@@ -28,5 +29,4 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --use_local_transformer \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.3 \
-    --ssim_target 0.0
+    --cer_target 0.1

tests/functional_tests/L2_TTS_InferEvaluate_Magpietts_ZeroShot.sh

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \
     --datasets an4_val_ci \
@@ -28,5 +29,5 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --use_local_transformer \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.25 \
-    --ssim_target 0.7
+    --cer_target 0.09 \
+    --ssim_target 0.74

tests/functional_tests/L2_TTS_InferEvaluatelongform_Magpietts_MoE_ZeroShot.sh

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --nemo_files "/home/TestData/tts/2602_MoE/moe16_sinkhorn_top1_valLoss5.0469_step2625132_epoch524.nemo" \
     --codecmodel_path "/home/TestData/tts/21fps_causal_codecmodel.nemo" \
     --datasets_json_path "examples/tts/evalset_config.json" \
@@ -23,5 +24,5 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --apply_attention_prior \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.08 \
-    --ssim_target 0.40
+    --cer_target 0.03 \
+    --ssim_target 0.49

tests/functional_tests/L2_TTS_InferEvaluatelongform_Magpietts_ZeroShot.sh

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.coverage --source=/workspace/nemo examples/tts/magpietts_inference.py \
+    --deterministic \
     --codecmodel_path /home/TestData/tts/21fps_causal_codecmodel.nemo \
     --datasets_json_path examples/tts/evalset_config.json \
     --datasets an4_val_ci_longform_tiny \
@@ -28,5 +29,5 @@ TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1 coverage run -a --data-file=/workspace/.cover
     --use_local_transformer \
     --run_evaluation \
     --clean_up_disk \
-    --cer_target 0.2 \
-    --ssim_target 0.5
+    --cer_target 0.02 \
+    --ssim_target 0.44