There was an error while loading. Please reload this page.
1 parent e7f74fa commit b128da7Copy full SHA for b128da7
1 file changed
src/megatron/bridge/training/setup.py
@@ -649,7 +649,8 @@ def _validate_and_set_vocab_size(
649
ValueError: If model vocab size is invalid
650
"""
651
if use_tokenizer_vocab_size or model_vocab_size is None:
652
- # If model vocab size is not set, use the tokenizer's vocab size
+ # Use the tokenizer's vocab size when the model vocab is unset, or when
653
+ # use_tokenizer_vocab_size forces it for from-scratch pretraining.
654
# Enable padding since this came from tokenizer
655
return tokenizer_vocab_size, True
656
elif model_vocab_size < tokenizer_vocab_size:
0 commit comments