Skip to content

Commit 2014751

Browse files
committed
fix(launcher): use the schema's draft_model global var, and validate the schema
`pipeline.global_vars` is a fixed-field dataclass (`GlobalVariables` in tools/launcher/core.py), not a free-form mapping, so an unknown key fails at launch with "No parameter named 'X' exists". The Nemotron-3.5 DSpark warm-start example (#2149) invented `drafter:`, so the example in the repo could not run at all: Error processing argument 'pipeline.global_vars.drafter=...': Invalid argument: No parameter named 'drafter' exists for <function launch> `draft_model` is the field that already exists for exactly this purpose. Renaming the key and its one reference fixes the example. The Kimi-K2.5 specdec_bench example had the same latent break with `draft_model_dir:`; only the global-var key is renamed there, the script's `--draft_model_dir` flag is unchanged. This is the second time the class has shipped -- the comment on `GlobalVariables.draft_model` records the first (OMNIML-5024) -- so check_launcher_yaml now rejects unknown global_vars keys and references to keys that are never defined. It reads the field names out of core.py with a regex rather than importing it, since importing pulls in nemo_run, which the pre-commit environment does not have. Verified both ways: the check passes on every launcher YAML in the tree, and reintroducing `drafter:` reproduces the error as a pre-commit failure naming the valid keys. Signed-off-by: h-guo18 <67671475+h-guo18@users.noreply.github.com>
1 parent 2b296b2 commit 2014751

3 files changed

Lines changed: 53 additions & 5 deletions

File tree

tools/launcher/examples/moonshotai/Kimi-K2.5/specdec_bench.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@ pipeline:
3232

3333
global_vars:
3434
hf_model: /hf-local/nvidia/Kimi-K2.5-NVFP4
35-
# Trained+exported DFLASH draft; override: pipeline.global_vars.draft_model_dir=<path>
36-
draft_model_dir: /hf-local/nvidia/Kimi-K2.5-DFlash
35+
# Trained+exported DFLASH draft; override: pipeline.global_vars.draft_model=<path>
36+
draft_model: /hf-local/nvidia/Kimi-K2.5-DFlash
3737

3838
task_0:
3939
script: common/specdec_bench/run.sh
4040
args:
41-
- --draft_model_dir <<global_vars.draft_model_dir>>
41+
- --draft_model_dir <<global_vars.draft_model>>
4242
- --speculative_algorithm DFLASH
4343
- --engine VLLM
4444
- --mtbench /hf-local/HuggingFaceH4/mt_bench_prompts/raw/question.jsonl

tools/launcher/examples/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/hf_streaming_dspark_warmstart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ pipeline:
5656
# config.json layer-type vocabulary already patched for tf5 (see header).
5757
hf_model: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16
5858
# The published drafter the warm start continues from.
59-
drafter: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark
59+
draft_model: /hf-local/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark
6060

6161
# Build /scratchspace/data/train.jsonl. Point data.data_path at the full
6262
# Spec-Decoding-Dataset-v2 corpus to reproduce; eagle_utils also accepts a
@@ -80,7 +80,7 @@ pipeline:
8080
# causal attention and attention sink all come from this recipe — see header.
8181
- --config modules/Model-Optimizer/modelopt_recipes/huggingface/models/nvidia/Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml
8282
- model.model_name_or_path=<<global_vars.hf_model>>
83-
- dflash.dflash_init_checkpoint=<<global_vars.drafter>>
83+
- dflash.dflash_init_checkpoint=<<global_vars.draft_model>>
8484
- data.data_path=/scratchspace/data/train.jsonl
8585
# The stock Nemotron template has no {% generation %} tags; without a tagged copy
8686
# answer_only_loss trains on an all-zero mask (see header).

tools/precommit/check_launcher_yaml.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,52 @@ def _try_load_recipe(recipe_path: Path, source: Path) -> list[str]:
112112
return []
113113

114114

115+
def _global_vars_schema() -> set[str] | None:
116+
"""Field names accepted by ``GlobalVariables``, or None if it can't be read.
117+
118+
Parsed out of ``core.py`` rather than imported: importing it pulls in ``nemo_run``,
119+
which is not a dependency of the pre-commit environment.
120+
"""
121+
core = _LAUNCHER_DIR / "core.py"
122+
try:
123+
source = core.read_text(encoding="utf-8")
124+
except OSError:
125+
return None
126+
match = re.search(r"^class GlobalVariables.*?(?=^@|\Z)", source, re.MULTILINE | re.DOTALL)
127+
if not match:
128+
return None
129+
return set(re.findall(r"^\s{4}(\w+)\s*:", match.group(0), re.MULTILINE))
130+
131+
132+
def _check_global_vars(pipeline: dict, path: Path) -> list[str]:
133+
"""Reject ``global_vars`` keys the launcher's dataclass cannot accept.
134+
135+
``global_vars`` is a fixed-field dataclass, not a free-form mapping, so an unknown key
136+
fails at launch with ``No parameter named 'X' exists`` — after the user has set up a
137+
cluster environment. This has now bitten twice (OMNIML-5024, then the Nemotron-3.5
138+
DSpark warm-start example), so it is checked here instead.
139+
"""
140+
schema = _global_vars_schema()
141+
global_vars = pipeline.get("global_vars")
142+
if schema is None or not isinstance(global_vars, dict):
143+
return []
144+
errors = [
145+
f"{path}: global_vars key {key!r} is not a field of GlobalVariables "
146+
f"(valid: {', '.join(sorted(schema))})"
147+
for key in global_vars
148+
if key not in schema
149+
]
150+
# A reference to a key that is never defined interpolates to the literal
151+
# ``<<global_vars.X>>`` and reaches the job as a nonsense path.
152+
refs = sorted(set(re.findall(r"<<global_vars\.(\w+)>>", path.read_text("utf-8"))))
153+
errors.extend(
154+
f"{path}: <<global_vars.{ref}>> is referenced but never defined"
155+
for ref in refs
156+
if ref not in global_vars
157+
)
158+
return errors
159+
160+
115161
def _scan_launcher_yaml(path: Path) -> list[str]:
116162
errors: list[str] = []
117163
try:
@@ -124,6 +170,8 @@ def _scan_launcher_yaml(path: Path) -> list[str]:
124170
if not isinstance(pipeline, dict):
125171
return []
126172

173+
errors.extend(_check_global_vars(pipeline, path))
174+
127175
for task in pipeline.values():
128176
if not isinstance(task, dict):
129177
continue

0 commit comments

Comments
 (0)