Standardize audio manifest container paths

Add shared helpers (build_container_audio_path, get_container_audio_root) and thread --audio-prefix through the audio benchmark prepare scripts so prepared JSONL manifests use in-container paths rooted at /data (overridable via --audio-prefix or NEMO_SKILLS_AUDIO_ROOT) instead of hardcoded /dataset paths.

Document the prepare/eval mount contract and add tests covering the path helpers and per-benchmark manifest paths.

Signed-off-by: Dongji Gao <dongjig@nvidia.com>

Author: DongjiGao

docs/evaluation/speech-audio.md

@@ -37,12 +37,40 @@ MMAU-Pro (Multimodal Audio Understanding - Pro) is a comprehensive benchmark for
 
 These benchmarks require audio files for meaningful evaluation. **Audio files are downloaded by default** to ensure proper evaluation.
 
+### Audio path convention
+
+Prepared audio manifests should write audio paths using the in-container audio
+root, not the host filesystem path. The default in-container root is `/data`.
+Override it with `--audio-prefix` when a different mount point is needed; if
+`--audio-prefix` is omitted, prepare scripts fall back to
+`NEMO_SKILLS_AUDIO_ROOT` and then `/data`.
+
+`--audio-prefix` is the global in-container audio root. Do not include the
+benchmark name in it; the prepare script appends the benchmark directory.
+For example, use `--audio-prefix /data`, not
+`--audio-prefix /data/contextasr-bench`.
+
+For example, preparing data with `--audio-prefix /data` writes manifest paths
+like:
+
+```text
+/data/asr-leaderboard/...
+/data/contextasr-bench/...
+```
+
+At evaluation time, mount the host prepared-data directory to the same
+in-container root:
+
+```bash
+ns eval ... --data_dir=/lustre/.../skills_data --mount-paths=/lustre/.../skills_data:/data
+```
+
 ### Data Preparation
 
 To prepare the dataset with audio files:
 
 ```bash
-ns prepare_data asr-leaderboard --data_dir=/path/to/data --cluster=<cluster>
+ns prepare_data asr-leaderboard --data_dir=/path/to/data --cluster=<cluster> --audio-prefix=/data
 ```
 
 Prepare specific datasets only:
@@ -74,7 +102,7 @@ eval(
     model="/workspace/checkpoint",
     server_entrypoint="/workspace/megatron-lm/server.py",
     server_container="/path/to/container.sqsh",
-    data_dir="/dataset",
+    data_dir="/data",
     installation_command="pip install -r requirements/audio.txt",
     server_args="--inference-max-requests 1 --model-config /workspace/checkpoint/config.yaml",
 )
@@ -98,7 +126,7 @@ eval(benchmarks="asr-leaderboard", split="librispeech_clean", ...)
         --model=/workspace/path/to/checkpoint \
         --server_entrypoint=/workspace/megatron-lm/server.py \
         --server_container=/path/to/container.sqsh \
-        --data_dir=/dataset \
+        --data_dir=/data \
         --installation_command="pip install -r requirements/audio.txt"
     ```
 
@@ -120,7 +148,7 @@ eval(
     model="/workspace/checkpoint",
     server_entrypoint="/workspace/megatron-lm/server.py",
     server_container="/path/to/container.sqsh",
-    data_dir="/dataset",
+    data_dir="/data",
     installation_command="pip install sacrebleu",
     server_args="--inference-max-requests 1 --model-config /workspace/checkpoint/config.yaml",
 )
@@ -150,7 +178,7 @@ eval(benchmarks="mmau-pro.closed_form", ...)
         --model=/workspace/path/to/checkpoint \
         --server_entrypoint=/workspace/megatron-lm/server.py \
         --server_container=/path/to/container.sqsh \
-        --data_dir=/dataset \
+        --data_dir=/data \
         --installation_command="pip install sacrebleu"
     ```
 
@@ -534,7 +562,7 @@ used directly. If the file is missing, data is downloaded there automatically.
 To use a custom audio path prefix (e.g., for container mount points):
 
 ```bash
-ns prepare_data contextasr-bench --data_dir=/path/to/ContextASR-Bench --audio-prefix /data/contextasr
+ns prepare_data contextasr-bench --data_dir=/path/to/ContextASR-Bench --audio-prefix /data
 ```
 
 ### Running ContextASR-Bench Evaluation

nemo_skills/dataset/asr-leaderboard/prepare.py

@@ -19,7 +19,7 @@
 same data source used by the official leaderboard, ensuring apples-to-apples
 WER comparison.
 
-Audio paths in JSONL: /dataset/asr-leaderboard/data/{dataset}/{sample_id}.flac
+Audio paths in JSONL: {audio_prefix}/asr-leaderboard/data/{dataset}/{sample_id}.flac
 
 Usage:
     ns prepare_data asr-leaderboard
@@ -36,6 +36,12 @@
 from datasets import Audio, load_dataset
 from tqdm import tqdm
 
+from nemo_skills.dataset.utils import (
+    DEFAULT_CONTAINER_AUDIO_ROOT,
+    build_container_audio_path,
+    get_container_audio_root,
+)
+
 HF_REPO = "hf-audio/esb-datasets-test-only-sorted"
 SYSTEM_MESSAGE = "You are a helpful assistant. /no_think"
 AUDIO_SAMPLE_RATE = 16000
@@ -69,7 +75,9 @@ def extract_audio(audio_info):
         return None, None
 
 
-def format_entry(entry, dataset_name, audio_dir, text_field, id_field, with_audio):
+def format_entry(
+    entry, dataset_name, audio_dir, text_field, id_field, with_audio, audio_root=DEFAULT_CONTAINER_AUDIO_ROOT
+):
     """Format a dataset entry into JSONL and optionally save the audio file."""
     text = entry[text_field].strip()
     if not text:
@@ -87,7 +95,11 @@ def format_entry(entry, dataset_name, audio_dir, text_field, id_field, with_audi
             sf.write(str(audio_dir / audio_filename), audio_array, sampling_rate)
 
     user_message = {"role": "user", "content": "Transcribe the following audio."}
-    audio_meta = {"path": f"/dataset/asr-leaderboard/data/{dataset_name}/{audio_filename}"}
+    audio_meta = {
+        "path": build_container_audio_path(
+            "asr-leaderboard", "data", dataset_name, audio_filename, audio_prefix=audio_root
+        )
+    }
     if duration is not None:
         audio_meta["duration"] = float(duration)
     user_message["audio"] = audio_meta
@@ -105,7 +117,7 @@ def format_entry(entry, dataset_name, audio_dir, text_field, id_field, with_audi
     return formatted
 
 
-def prepare_dataset(dataset_name, output_dir, with_audio=True):
+def prepare_dataset(dataset_name, output_dir, with_audio=True, audio_root=DEFAULT_CONTAINER_AUDIO_ROOT):
     """Download, decode, and write a single ASR dataset to JSONL + audio files."""
     if dataset_name not in DATASET_CONFIGS:
         raise ValueError(f"Unknown dataset: {dataset_name}. Available: {list(DATASET_CONFIGS.keys())}")
@@ -127,7 +139,9 @@ def prepare_dataset(dataset_name, output_dir, with_audio=True):
     count = 0
     with open(output_file, "w", encoding="utf-8") as fout:
         for entry in tqdm(dataset, desc=dataset_name):
-            formatted = format_entry(entry, dataset_name, audio_dir, text_field, id_field, with_audio)
+            formatted = format_entry(
+                entry, dataset_name, audio_dir, text_field, id_field, with_audio, audio_root=audio_root
+            )
             if formatted is None:
                 continue
             fout.write(json.dumps(formatted) + "\n")
@@ -146,26 +160,48 @@ def main():
         choices=list(DATASET_CONFIGS.keys()) + ["all"],
         help="Datasets to prepare (default: all)",
     )
+    parser.add_argument(
+        "--data_dir",
+        type=str,
+        default=None,
+        help=(
+            "Output directory. If provided, outputs go under <data_dir>/asr-leaderboard. "
+            "If omitted, writes into this package's dataset directory."
+        ),
+    )
     parser.add_argument(
         "--no-audio",
         action="store_true",
         help="Skip saving audio files (JSONL still includes audio paths)",
     )
+    parser.add_argument(
+        "--audio-prefix",
+        type=str,
+        default=None,
+        help="In-container audio root written into JSONL paths. Defaults to $NEMO_SKILLS_AUDIO_ROOT or /data.",
+    )
     args = parser.parse_args()
 
-    data_dir = Path("/dataset/asr-leaderboard")
-    output_dir = data_dir if data_dir.exists() else Path(__file__).parent
+    if args.data_dir:
+        output_dir = Path(args.data_dir) / "asr-leaderboard"
+    else:
+        output_dir = Path(__file__).parent
     output_dir.mkdir(parents=True, exist_ok=True)
 
     with_audio = not args.no_audio
-    if not with_audio:
+    audio_root = get_container_audio_root(args.audio_prefix)
+
+    if args.no_audio:
         print("Running without saving audio files.")
+    else:
+        print("Running with audio. Saving to data/{dataset}/")
+    print(f"Audio paths in JSONL will use: {audio_root}/asr-leaderboard/data/...")
 
     datasets_to_prepare = list(DATASET_CONFIGS.keys()) if "all" in args.datasets else args.datasets
 
     total_samples = 0
     for dataset_name in datasets_to_prepare:
-        total_samples += prepare_dataset(dataset_name, output_dir, with_audio=with_audio)
+        total_samples += prepare_dataset(dataset_name, output_dir, with_audio=with_audio, audio_root=audio_root)
 
     combined_file = output_dir / "test.jsonl"
     print(f"\nCreating combined file: {combined_file}")

nemo_skills/dataset/audiobench/prepare.py

@@ -35,6 +35,12 @@
 import soundfile as sf
 from tqdm import tqdm
 
+from nemo_skills.dataset.utils import (
+    DEFAULT_CONTAINER_AUDIO_ROOT,
+    build_container_audio_path,
+    get_container_audio_root,
+)
+
 # AudioBench datasets categorized by evaluation type
 JUDGE_DATASETS = [
     "alpaca_audio_test",
@@ -140,6 +146,7 @@ def create_manifest_entry(
     dataset_name: str,
     sample_id: int,
     category: str,
+    audio_root: str = DEFAULT_CONTAINER_AUDIO_ROOT,
 ) -> Dict:
     """Create a nemo-skills compatible manifest entry.
 
@@ -158,9 +165,9 @@ def create_manifest_entry(
     reference = sample.get("reference", sample.get("answer", ""))
     task_type = sample.get("task_type", "unknown")
 
-    # Create absolute audio path with /data/ prefix for cluster deployment
-    # Format: /data/audiobench/{category}/audio/{dataset_name}/{filename}
-    audio_rel_path = f"/data/audiobench/{category}/audio/{dataset_name}/{audio_filename}"
+    audio_rel_path = build_container_audio_path(
+        "audiobench", category, "audio", dataset_name, audio_filename, audio_prefix=audio_root
+    )
 
     # Create audio metadata (both singular and plural forms for compatibility)
     audio_metadata = {"path": audio_rel_path, "duration": duration}
@@ -209,6 +216,7 @@ def process_dataset(
     save_audio: bool = True,
     split: str = "test",
     max_samples: int = -1,
+    audio_root: str = DEFAULT_CONTAINER_AUDIO_ROOT,
 ) -> tuple[int, List[Dict]]:
     """Process a single AudioBench dataset.
 
@@ -459,6 +467,7 @@ def process_dataset(
                 dataset_name=dataset_name,
                 sample_id=idx,
                 category=category,
+                audio_root=audio_root,
             )
 
             manifest_entries.append(entry)
@@ -519,6 +528,12 @@ def main():
         default=-1,
         help="Maximum number of samples to process per dataset (-1 for all)",
     )
+    parser.add_argument(
+        "--audio-prefix",
+        type=str,
+        default=None,
+        help="In-container audio root written into JSONL paths. Defaults to $NEMO_SKILLS_AUDIO_ROOT or /data.",
+    )
     parser.set_defaults(save_audio=True)
 
     args = parser.parse_args()
@@ -531,6 +546,7 @@ def main():
         output_dir = Path(__file__).parent
 
     output_dir.mkdir(parents=True, exist_ok=True)
+    audio_root = get_container_audio_root(args.audio_prefix)
 
     print("\n" + "=" * 60)
     print("AudioBench Dataset Preparation")
@@ -539,6 +555,7 @@ def main():
     print(f"Output directory: {output_dir}")
     print(f"Save audio files: {args.save_audio}")
     print(f"Split: {args.split}")
+    print(f"Audio paths in JSONL will use: {audio_root}/audiobench/...")
     print("=" * 60 + "\n")
 
     # Determine which datasets to process
@@ -585,6 +602,7 @@ def main():
                 save_audio=args.save_audio,
                 split=args.split,
                 max_samples=args.max_samples,
+                audio_root=audio_root,
             )
             total_samples += num_samples
             total_datasets += 1

nemo_skills/dataset/contextasr-bench/prepare.py

@@ -39,6 +39,9 @@
 import tarfile
 from pathlib import Path
 
+from nemo_skills.dataset.utils import build_container_audio_path
+
+BENCHMARK_NAME = "contextasr-bench"
 HF_REPO_ID = "MrSupW/ContextASR-Bench"
 JSONL_FILENAME = "ContextASR-Speech_English.jsonl"
 AUDIO_TAR_PREFIX = "audio/ContextASR-Speech/English/ContextASR-Speech_English"
@@ -142,6 +145,11 @@ def build_messages(prompt_text, audio_path, duration):
     ]
 
 
+def resolve_audio_prefix(audio_prefix: str | None = None) -> str:
+    """Return the in-container ContextASR audio prefix used in JSONL paths."""
+    return build_container_audio_path(BENCHMARK_NAME, audio_prefix=audio_prefix)
+
+
 def format_entry(sample, mode, audio_prefix):
     """Format a single dataset sample into a JSONL record for a given mode."""
     audio_path = f"{audio_prefix}/{sample['audio']}"
@@ -191,8 +199,9 @@ def main():
         default=None,
         help=(
             "Override audio path prefix written into JSONL files. "
-            "Defaults to the data_dir value. Useful for container mount points "
-            "(e.g., --audio-prefix /data/contextasr-bench)."
+            "This is the global in-container audio root; the script appends "
+            f"{BENCHMARK_NAME}/. Defaults to $NEMO_SKILLS_AUDIO_ROOT or /data "
+            f"(e.g., --audio-prefix /data writes /data/{BENCHMARK_NAME}/...)."
         ),
     )
     parser.add_argument(
@@ -220,8 +229,7 @@ def main():
         print(f"Data not found at {data_dir}. Downloading there...")
         download_dataset(data_dir)
 
-    audio_prefix = args.audio_prefix if args.audio_prefix else str(data_dir)
-    audio_prefix = audio_prefix.rstrip("/")
+    audio_prefix = resolve_audio_prefix(args.audio_prefix)
 
     jsonl_path = data_dir / JSONL_FILENAME
 
@@ -236,14 +244,15 @@ def main():
     print(f"Loaded {len(samples)} samples")
 
     if not args.no_audio:
-        sample_audio = Path(audio_prefix) / samples[0]["audio"]
+        sample_audio = data_dir / samples[0]["audio"]
         if not sample_audio.exists():
             print(
                 f"WARNING: Sample audio file not found at {sample_audio}. "
-                f"Audio paths may need adjustment via --audio-prefix."
+                f"Audio paths may need adjustment via --data_dir."
             )
         else:
             print(f"Audio files verified (sample check: {sample_audio})")
+    print(f"Audio paths in JSONL will use: {audio_prefix}/...")
 
     modes = {
         "contextless": output_dir / "contextless" / "test.jsonl",