refactor(recipes): isolate Qwen-VL null tokenizer

Signed-off-by: Chen Cui <chcui@nvidia.com>

Author: cuichenx

src/megatron/bridge/perf_recipes/qwen_vl/common.py

@@ -32,11 +32,21 @@
 from megatron.bridge.training.config import ConfigContainer
 
 
+def _use_model_vocab_null_tokenizer(cfg: ConfigContainer) -> None:
+    """Use a model-sized synthetic tokenizer for Qwen-VL performance runs."""
+    if cfg.model.vocab_size is None:
+        raise ValueError("Qwen-VL performance recipes require a model vocabulary size.")
+    cfg.tokenizer.tokenizer_type = "NullTokenizer"
+    cfg.tokenizer.tokenizer_model = None
+    cfg.tokenizer.vocab_size = cfg.model.vocab_size
+
+
 def _qwen35_vl_common(cfg: ConfigContainer) -> None:
     """Apply VLM-specific performance benchmark settings for Qwen3.5-VL.
 
     Must be called before ``_benchmark_common`` and after setting precision.
     """
+    _use_model_vocab_null_tokenizer(cfg)
     cfg.model.bias_activation_fusion = True
     cfg.model.recompute_granularity = None
     cfg.model.recompute_method = None
@@ -82,6 +92,7 @@ def _qwen35_vl_post_clear_scope_with_overlap(cfg: ConfigContainer) -> None:
 
 def _finalize_qwen3_vl(cfg: ConfigContainer) -> None:
     """Apply Qwen3-VL perf defaults that must override generic benchmark defaults."""
+    _use_model_vocab_null_tokenizer(cfg)
     # _benchmark_common sets apply_rope_fusion=True; Qwen3-VL asserts it must be False
     # (per-token absolute positional frequencies are incompatible with TE's fused RoPE).
     cfg.model.apply_rope_fusion = False

src/megatron/bridge/recipes/gpt/h100/vanilla_gpt.py

@@ -112,7 +112,6 @@ def vanilla_gpt_pretrain_1gpu_h100_bf16_config() -> ConfigContainer:
         tokenizer=TokenizerConfig(
             tokenizer_type="NullTokenizer",
             vocab_size=DEFAULT_NULL_TOKENIZER_VOCAB_SIZE,
-            use_tokenizer_vocab_size=True,
         ),
         checkpoint=CheckpointConfig(
             save_interval=500,

src/megatron/bridge/recipes/qwen_vl/h100/qwen35_vl.py

@@ -30,15 +30,12 @@
 from megatron.bridge.recipes.utils.dataset_utils import default_peft_config
 from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
 from megatron.bridge.recipes.utils.optimizer_utils import distributed_fused_adam_with_cosine_annealing
-from megatron.bridge.recipes.utils.tokenizer_utils import DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
 from megatron.bridge.training.config import ConfigContainer
 
 
 # =============================================================================
 # Qwen3.5-VL Pretrain Configurations (mock dataset)
 # =============================================================================
-# The mock VLM dataset gets language token IDs from the HF processor. Its
-# NullTokenizer is only a runtime placeholder and must not resize the model.
 def qwen35_vl_9b_pretrain_4gpu_h100_bf16_mock_config() -> ConfigContainer:
     """Return a pre-training config for Qwen3.5-VL 9B (dense)."""
     cfg = _pretrain_common()
@@ -74,9 +71,7 @@ def qwen35_vl_9b_pretrain_4gpu_h100_bf16_mock_config() -> ConfigContainer:
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -124,9 +119,7 @@ def qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config() -> ConfigContainer:
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -175,9 +168,7 @@ def qwen35_vl_35b_a3b_pretrain_8gpu_h100_bf16_mock_config() -> ConfigContainer:
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -227,9 +218,7 @@ def qwen35_vl_122b_a10b_pretrain_128gpu_h100_bf16_mock_config() -> ConfigContain
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -280,9 +269,7 @@ def qwen35_vl_397b_a17b_pretrain_512gpu_h100_bf16_mock_config() -> ConfigContain
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False

src/megatron/bridge/recipes/qwen_vl/h100/qwen3_vl.py

@@ -32,16 +32,13 @@
 from megatron.bridge.recipes.utils.dataset_utils import default_peft_config
 from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
 from megatron.bridge.recipes.utils.optimizer_utils import distributed_fused_adam_with_cosine_annealing
-from megatron.bridge.recipes.utils.tokenizer_utils import DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
 from megatron.bridge.training.config import ConfigContainer
 from megatron.bridge.training.flex_dispatcher_backend import apply_flex_dispatcher_backend
 
 
 # =============================================================================
 # Qwen3-VL Pretrain Configurations (mock dataset)
 # =============================================================================
-# The mock VLM dataset gets language token IDs from the HF processor. Its
-# NullTokenizer is only a runtime placeholder and must not resize the model.
 
 
 def qwen3_vl_8b_pretrain_4gpu_h100_bf16_mock_config() -> ConfigContainer:
@@ -79,9 +76,7 @@ def qwen3_vl_8b_pretrain_4gpu_h100_bf16_mock_config() -> ConfigContainer:
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -130,9 +125,7 @@ def qwen3_vl_30b_a3b_pretrain_8gpu_h100_bf16_mock_config() -> ConfigContainer:
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False
@@ -182,9 +175,7 @@ def qwen3_vl_235b_a22b_pretrain_256gpu_h100_bf16_mock_config() -> ConfigContaine
         persistent_workers=False,
         pad_to_max_length=True,
     )
-    cfg.tokenizer.tokenizer_type = "NullTokenizer"
-    cfg.tokenizer.vocab_size = DEFAULT_NULL_TOKENIZER_VOCAB_SIZE
-    cfg.tokenizer.use_tokenizer_vocab_size = False
+    cfg.tokenizer.tokenizer_model = hf_path
     cfg.train.eval_interval = 500
     cfg.train.eval_iters = 32
     cfg.ddp.overlap_grad_reduce = False

tests/unit_tests/recipes/qwen_vl/test_qwen35_vl_recipes.py

@@ -715,7 +715,9 @@ def test_each_qwen35_vl_pretrain_mock_recipe_builds_config(recipe_func: Callable
 
     _assert_basic_config(cfg)
 
-    assert cfg.tokenizer.tokenizer_type == "NullTokenizer"
+    assert cfg.tokenizer.tokenizer_type == "HuggingFaceTokenizer"
+    assert cfg.tokenizer.tokenizer_model == cfg.dataset.hf_processor_path
+    assert cfg.tokenizer.use_tokenizer_vocab_size is True
     assert getattr(cfg.model, "tensor_model_parallel_size", 1) >= 1
     assert getattr(cfg.model, "pipeline_model_parallel_size", 1) >= 1
 

tests/unit_tests/recipes/qwen_vl/test_qwen3_vl_recipes.py

@@ -600,7 +600,9 @@ def test_each_qwen3_vl_pretrain_mock_recipe_builds_config(recipe_func: Callable,
 
     _assert_basic_config(cfg)
 
-    assert cfg.tokenizer.tokenizer_type == "NullTokenizer"
+    assert cfg.tokenizer.tokenizer_type == "HuggingFaceTokenizer"
+    assert cfg.tokenizer.tokenizer_model == cfg.dataset.hf_processor_path
+    assert cfg.tokenizer.use_tokenizer_vocab_size is True
     assert getattr(cfg.model, "tensor_model_parallel_size", 1) >= 1
     assert getattr(cfg.model, "pipeline_model_parallel_size", 1) >= 1
 

tests/unit_tests/recipes/test_all_perf_recipe_factories.py

@@ -21,7 +21,7 @@
 
 import pytest
 
-from megatron.bridge.training.config import ConfigContainer, GPTDatasetConfig, MockVLMSFTDatasetConfig
+from megatron.bridge.training.config import ConfigContainer, MockVLMSFTDatasetConfig
 from tests.unit_tests.recipes.recipe_test_utils import (
     discover_recipe_factories,
     exported_recipe_factory_keys,
@@ -85,7 +85,6 @@ def test_perf_recipe_factory_builds_config(recipe_factory: Callable[..., object]
     ):
         assert getattr(cfg, section) is not None
 
-    if "pretrain" in recipe_factory.__name__ and isinstance(cfg.dataset, GPTDatasetConfig):
-        assert cfg.tokenizer.use_tokenizer_vocab_size is True
     if "pretrain" in recipe_factory.__name__ and isinstance(cfg.dataset, MockVLMSFTDatasetConfig):
-        assert cfg.tokenizer.use_tokenizer_vocab_size is False
+        assert cfg.tokenizer.tokenizer_type == "NullTokenizer"
+        assert cfg.tokenizer.vocab_size == cfg.model.vocab_size

tests/unit_tests/recipes/test_all_recipe_factories.py

@@ -20,7 +20,8 @@
 
 import pytest
 
-from megatron.bridge.training.config import ConfigContainer, GPTDatasetConfig, MockVLMSFTDatasetConfig
+from megatron.bridge.recipes.common import _pretrain_common
+from megatron.bridge.training.config import ConfigContainer, MockVLMSFTDatasetConfig
 from tests.unit_tests.recipes.recipe_test_utils import (
     discover_recipe_factories,
     exported_recipe_factory_keys,
@@ -65,6 +66,11 @@ def test_all_recipe_factories_are_exported() -> None:
         assert getattr(_RECIPES_PACKAGE, factory.__name__) is factory
 
 
+def test_common_pretrain_uses_runtime_tokenizer_vocabulary() -> None:
+    """From-scratch pretraining derives the model vocabulary from its tokenizer."""
+    assert _pretrain_common().tokenizer.use_tokenizer_vocab_size is True
+
+
 @pytest.mark.parametrize("recipe_factory", _RUNNABLE_RECIPE_FACTORIES, ids=recipe_factory_id)
 def test_recipe_factory_builds_config(recipe_factory: Callable[..., object]) -> None:
     """Every supported recipe can be called with defaults without GPU or network access."""
@@ -88,10 +94,10 @@ def test_recipe_factory_builds_config(recipe_factory: Callable[..., object]) ->
     ):
         assert getattr(cfg, section) is not None
 
-    if "pretrain" in recipe_factory.__name__ and isinstance(cfg.dataset, GPTDatasetConfig):
-        assert cfg.tokenizer.use_tokenizer_vocab_size is True
     if "pretrain" in recipe_factory.__name__ and isinstance(cfg.dataset, MockVLMSFTDatasetConfig):
-        assert cfg.tokenizer.use_tokenizer_vocab_size is False
+        assert cfg.tokenizer.tokenizer_type == "HuggingFaceTokenizer"
+        assert cfg.tokenizer.tokenizer_model == cfg.dataset.hf_processor_path
+        assert cfg.tokenizer.use_tokenizer_vocab_size is True
 
 
 @pytest.mark.parametrize("recipe_factory", _UNSUPPORTED_RECIPE_FACTORIES, ids=recipe_factory_id)