dellemc-mlperf-training-v6.0-preservation/start.sh at XE9780LAPB300-SXM-270GB-MLPerf-training-v6.0-logs · mrcawood-dell/dellemc-mlperf-training-v6.0-preservation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#!/bin/bash

# Validate input argument
if [ $# -ne 1 ]; then
    echo "Usage: $0 <benchmark_type>"
    echo "  1 = llama31_8b"
    echo "  2 = llama2_70b_lora"
    echo "  3 = dlrm_dcnv2"
    echo "  4 = gpt_oss_20b"
    exit 1
fi

BENCHMARK_TYPE=$1

# Generate timestamp for unique LOGDIR
TIMESTAMP=$(date +'%Y%m%d_%H%M%S')

# Set benchmark-specific variables
case $BENCHMARK_TYPE in
    1)
        BENCHMARK_NAME="llama31_8b"
        BENCHMARK_DIR="/mnt/data/dellemc-mlperf-training-v6.0/benchmarks/llama31_8b/implementations/nemo"
        CONT="/mnt/data/training_images/mlperf-nvidia+llama31_8b-pyt.sqsh"
        DATADIR="/mnt/data/training_dataset_v6.0"
        CONFIG_FILE="config_XE9780LAPB300_1x8x3xtp1pp1cp1_8b_fp4.sh"
        MODEL_SIZE="8b"
        BENCHMARK_ARG="llama3_1-8b"
        ;;
    2)
        BENCHMARK_NAME="llama2_70b_lora"
        BENCHMARK_DIR="/mnt/data/dellemc-mlperf-training-v6.0/benchmarks/llama2_70b_lora/implementations/nemo"
        CONT="/mnt/data/training_images/mlperf-nvidia+llama2_70b_lora-pyt.sqsh"
        DATADIR="/mnt/data/training_dataset_v6.0/gov_report"
        MODEL="/mnt/data/training_dataset_v6.0/model"
        CONFIG_FILE="config_XE9780LAPB300_1x8x2xtp1pp1cp1_fp4.sh"
        MODEL_SIZE="70b"
        BENCHMARK_ARG="llama2-70b"
        ;;
    3)
        BENCHMARK_NAME="dlrm_dcnv2"
        BENCHMARK_DIR="/mnt/data/dellemc-mlperf-training-v6.0/benchmarks/dlrm_dcnv2"
        CONT="/mnt/data/training_images/mlperf-nvidia+recommendation-hugectr.sqsh"
        DATADIR="/mnt/data/training_dataset_v6.0/data/preprocessed_criteo_click_logs"
        CONFIG_FILE="config_DGXB200_1x8x6912.sh"
        MODEL_SIZE=""
        BENCHMARK_ARG="dlrmv2"
        ;;
    4)
        BENCHMARK_NAME="gpt_oss_20b"
        BENCHMARK_DIR="/mnt/data/dellemc-mlperf-training-v6.0/benchmarks/gpt_oss_20b"
        CONT="/mnt/data/training_images/mlperf-nvidia+gpt_oss_20b-pyt.sqsh"
        DATADIR="/mnt/data/training_dataset_v6.0"
        CONFIG_FILE="config_XE9780LAPB300_1x8x3xtp1pp1cp1ep1_mxfp8.sh"
        MODEL_SIZE="20b"
        BENCHMARK_ARG="gptj"
        ;;
    *)
        echo "Invalid benchmark type: $BENCHMARK_TYPE"
        echo "Valid options: 1, 2, 3, 4"
        exit 1
        ;;
esac

# Create unique LOGDIR
export LOGDIR="/mnt/data/dellemc-mlperf-training-v6.0/results/${BENCHMARK_NAME}_${TIMESTAMP}"
mkdir -p "${LOGDIR}"

# Export MODEL variable
export MODEL

# Copy telemetry config to LOGDIR
cp /mnt/data/dellemc-mlperf-training-v6.0/single_node_ai_telemetry/hwsys_monitor.cfg "${LOGDIR}/"

# Change to benchmark directory
cd "${BENCHMARK_DIR}"

# Set common environment variables
export NEXP=3
export IDLE_TIME=60
export MLPERF_SUBMITTER="Dell"
export MLPERF_SYSTEM_NAME="XE9780LAPB300"
export SLURM_MPI_TYPE=pmix

# Source config file
source "${CONFIG_FILE}"

# Set MODEL_SIZE if applicable
if [ -n "${MODEL_SIZE}" ]; then
    export MODEL_SIZE
fi

# Submit SLURM job with environment variables
echo "Submitting ${BENCHMARK_NAME} benchmark..."
echo "LOGDIR: ${LOGDIR}"
echo "IDLE_TIME: ${IDLE_TIME}s"
echo "NEXP: ${NEXP}"
JOB_ID=$(env CONT="${CONT}" DATADIR="${DATADIR}" LOGDIR="${LOGDIR}" NEXP="${NEXP}" IDLE_TIME="${IDLE_TIME}" MLPERF_SUBMITTER="${MLPERF_SUBMITTER}" MLPERF_SYSTEM_NAME="${MLPERF_SYSTEM_NAME}" MODEL_SIZE="${MODEL_SIZE}" MODEL="${MODEL}" sbatch -A root -N ${DGXNNODES} --time=${WALLTIME} run.sub | awk '{print $4}')
echo "Job ID: ${JOB_ID}"

echo "Job submitted. Monitor with: squeue -j ${JOB_ID}"
echo "After job completes, collect results with:"
echo "  cd /mnt/data/dellemc-mlperf-training-v6.0/mlperf_tools/training"
echo "  python3 find_runtime '${LOGDIR}' > '${LOGDIR}/runtime_results.txt'"