Skip to content

Commit b128da7

Browse files
committed
docs(training): clarify tokenizer vocab override comment
Signed-off-by: Chen Cui <chcui@nvidia.com>
1 parent e7f74fa commit b128da7

1 file changed

Lines changed: 2 additions & 1 deletion

File tree

src/megatron/bridge/training/setup.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -649,7 +649,8 @@ def _validate_and_set_vocab_size(
649649
ValueError: If model vocab size is invalid
650650
"""
651651
if use_tokenizer_vocab_size or model_vocab_size is None:
652-
# If model vocab size is not set, use the tokenizer's vocab size
652+
# Use the tokenizer's vocab size when the model vocab is unset, or when
653+
# use_tokenizer_vocab_size forces it for from-scratch pretraining.
653654
# Enable padding since this came from tokenizer
654655
return tokenizer_vocab_size, True
655656
elif model_vocab_size < tokenizer_vocab_size:

0 commit comments

Comments
 (0)