update files after clean up to ensure successfully running

Author: ZixianWangAMD

small_llm_pretraining/nemo/config_H100_1x8x4_8b.sh

@@ -71,9 +71,6 @@ export FROM_HF=1
 # Model: Whether we want to save a checkpoint. Must be 1 if NPAR > 1. If 1, then we save a checkpoint at the end.
 export SAVE_CKPT=0
 
-# export SRC_PATH="/data/training/Llama-3.1-8B/original/"
-# export DST_PATH="/data/training/Llama-3.1-8B/nemo/"
-export CONT_IMAGE_URL="rocm/mlperf:llama31_8b_training_5.1_gfx942 "
 
 
 # Training Configs: 

small_llm_pretraining/nemo/config_H200_1x8x1_8b.sh

@@ -71,9 +71,6 @@ export FROM_HF=1
 # Model: Whether we want to save a checkpoint. Must be 1 if NPAR > 1. If 1, then we save a checkpoint at the end.
 export SAVE_CKPT=0
 
-# export SRC_PATH="/data/training/Llama-3.1-8B/original/"
-# export DST_PATH="/data/training/Llama-3.1-8B/nemo/"
-export CONT_IMAGE_URL="rocm/mlperf:llama31_8b_training_5.1_gfx942 "
 
 
 # Training Configs: 

small_llm_pretraining/nemo/config_MI325X_1x8x1_8b.sh

@@ -38,21 +38,21 @@ export IMAGE="DUMMY"
 # export ORIGINAL_C4_PATH="/data/data/C4"
 
 # This corresponds to the PREPROCESSED_PATH in README section 3's dataset download part
-export PREPROCESSED_PATH="/data/llama3_8b/data/C4_processed"
-export MERGED_C4_PATH="/data/llama3_8b/data/C4_merged"
+export PREPROCESSED_PATH="/data/llama31_8b/data/C4_processed/"
+export MERGED_C4_PATH="/data/llama31_8b/data/C4_merged"
 # Dataset: Numpy index working directory, contains shuffled dataset
 # This path must be able to hold >400GB data
 export TMP_NPY_INDEX="/data/npy_indices"
 # Dataset: Tokenizer path
 # This corresponds to the TOKENIZER_PATH in README section 3's tokenizer download part
-export TOKENIZER_PATH="/data/llama3_8b/model/Llama-3.1-8B"
+export TOKENIZER_PATH="/data/llama31_8b/model/Llama-3.1-8B-ref/"
 # export TOKENIZER_PATH="/data/llama3_405b_ref/tokenizer"
 
 # Model: checkpoint and tokenizer path
 #     This is the checkpoint that we want to start with. 
 #     Each checkpoint should be a folder containing two sub-folders: context and weights. 
 #     And we need to pass this folder's path (the folder containing context and weights) here.  
-export MODEL_CKPT="/data/llama3_8b/model/Llama-3.1-8B_nemo"
+export MODEL_CKPT="/data/llama31_8b/model/Llama-3.1-8B-ref/"
 # export MODEL_CKPT="None"
 # Model: Continual checkpoint directory to write and resume
 #     This is the directory to hold all intermediate checkpoints. 
@@ -71,9 +71,6 @@ export FROM_HF=1
 # Model: Whether we want to save a checkpoint. Must be 1 if NPAR > 1. If 1, then we save a checkpoint at the end.
 export SAVE_CKPT=0
 
-# export SRC_PATH="/data/training/Llama-3.1-8B/original/"
-# export DST_PATH="/data/training/Llama-3.1-8B/nemo/"
-export CONT_IMAGE_URL="rocm/mlperf:llama31_8b_training_5.1_gfx942 "
 
 
 # Training Configs: 
@@ -88,8 +85,11 @@ export MAX_LR="5e-4"
 #     If an empty string is provided (""), then the training will continue until time limit
 #     If we want to save a checkpoint, then this value must be set
 # export MAX_STEPS=1200000 # Fixed max_steps=1200000 in pretrain_llama31.py  
+export WARMUP_STEPS=512 # 16384 // GBS
 export EVAL_EVERY=12288
 export START_EVAL_AT=0
+
+export TENSOR_PARALLEL_SIZE=1
 # Experiment: starting steps
 #     This is the starting "offset" step from the checkpoint. 
 #     For instance, if you are resuming from a checkpoint folder `checkpoint-par-0-20-steps/checkpoint`, 

small_llm_pretraining/nemo/dev/run_llama31.sh

@@ -144,7 +144,7 @@ python3 pretrain_llama31.py \
 --num_pars $NPAR \
 --initial_ckpt_path $MODEL_CKPT \
 --continual_ckpt_path $CONTINUAL_CKPT \
---tokenizer_path "/tokenizer" \
+--tokenizer_path $TOKENIZER_PATH \
 --target_log_ppl $TARGET \
 --step_time_atol $STEP_TIME_ATOL \
 --ckpt_start_step $START_STEPS \

small_llm_pretraining/nemo/pretrain_llama31.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import os 
 import math
 import argparse
 from typing import Optional
@@ -207,7 +207,7 @@ def get_pretrain(
     pretrain.trainer.max_steps = 1200000 # Hardcoded to fix max_steps for this benchmark 
 
     pretrain.data = data_module
-    pretrain.trainer.val_check_interval = start_eval_at
+    pretrain.trainer.val_check_interval = eval_every / int (os.getenv ("GBS"))
     pretrain.trainer.limit_val_batches = eval_batches
     pretrain.trainer.limit_test_batches = eval_batches
 
@@ -239,7 +239,7 @@ def get_data(
 
     train_datasets = None
 
-    dataset_path = "/preproc_data"
+    dataset_path = dataset_path = os.getenv("PREPROCESSED_PATH")
 
     if use_full_dataset:
         train_datasets = sum([["12.5", f"{dataset_path}/c4-train.en_{idx}_text_document"] for idx in range(8)], [])