Skip to content

Commit b46737c

Browse files
elvingerpaulelvingerpaul
authored
Merge pull request #3 from eth-easl/benchmarking_inter
Green contexts and inter_sm scripts
2 parents 4599a45 + 0bc38f4 commit b46737c

28 files changed

Lines changed: 706 additions & 323 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# results
2+
*.txt
3+
14
# version file generated by setuptools-scm
25
/vllm/_version.py
36

‎inter_sm/benchmark_comp.sh‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,13 @@ NUM_TOKENS=10
66
PROMPT_SIZE=1000
77
PROFILE_TYPE=decode
88

9-
MPS_PERC=50
9+
NUM_REQUESTS=10
1010
INTER_FILE="inter_sm/comp_kernel_config.json"
1111
RESULTS="inter_sm/results_comp"
12+
NSYS_FILE="output"
13+
PLOTS="inter_sm/plots"
1214

13-
NSYS_FILE="output_comp_1_512_3"
14-
15-
echo "------------------------------------"
16-
echo "Starting MPS"
17-
export CUDA_VISIBLE_DEVICES=0
18-
export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps # Select a location that’s accessible to the given $UID
19-
export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log # Select a location that’s accessible to the given $UID
20-
nvidia-cuda-mps-control -d
21-
22-
function cleanup() {
23-
echo "Shutting down MPS control daemon..."
24-
echo quit | nvidia-cuda-mps-control
25-
echo "MPS control daemon shut down."
26-
}
15+
TB_SIZES=(0 34 68 102 136)
2716

2817
# nsys profile \
2918
# -w true \
@@ -32,12 +21,23 @@ function cleanup() {
3221
# --cuda-graph-trace node \
3322
# --force-overwrite true \
3423
# -o "$NSYS_FILE".nsys-rep \
35-
CUDA_MPS_ACTIVE_THREAD_PERCENTAGE=50 python3 inter_sm/mps_test.py \
36-
--model $MODEL \
37-
--profile $PROFILE_TYPE \
38-
--prompt_size $PROMPT_SIZE \
39-
--num_tokens $NUM_TOKENS \
40-
--batch_size $BATCH_SIZE \
41-
--mps $MPS_PERC \
42-
--interference_config_json $INTER_FILE \
43-
--inter_tb 1
24+
25+
mkdir -p $RESULTS
26+
mkdir -p $PLOTS
27+
28+
# for tb in "${TB_SIZES[@]}"
29+
# do
30+
# echo "$tb"
31+
# python3 inter_sm/gcontext_test.py \
32+
# --model $MODEL \
33+
# --profile $PROFILE_TYPE \
34+
# --prompt_size $PROMPT_SIZE \
35+
# --num_tokens $NUM_TOKENS \
36+
# --batch_size $BATCH_SIZE \
37+
# --num_requests $NUM_REQUESTS \
38+
# --interference_config_json $INTER_FILE \
39+
# --inter_tb $tb \
40+
# --inter_decode > "$RESULTS/output_comp_$tb.txt"
41+
# done
42+
43+
python inter_sm/process_results.py --output_dir $RESULTS --fig_path $PLOTS/comp.png --num_tokens $NUM_TOKENS --vary_grid

‎inter_sm/benchmark_l2.sh‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,34 @@ NUM_TOKENS=10
66
PROMPT_SIZE=1000
77
PROFILE_TYPE=decode
88

9-
MPS_PERC=50
9+
NUM_REQUESTS=10
1010
INTER_FILE="inter_sm/l2_kernel_config.json"
1111
RESULTS="inter_sm/results_l2"
12+
NSYS_FILE="output_l2"
13+
TB=68
14+
NUM_FLOATS=(4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608)
1215

13-
echo "------------------------------------"
14-
echo "Starting MPS"
15-
export CUDA_VISIBLE_DEVICES=0
16-
export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps # Select a location that’s accessible to the given $UID
17-
export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log # Select a location that’s accessible to the given $UID
18-
nvidia-cuda-mps-control -d
16+
# nsys profile \
17+
# -w true \
18+
# --trace cuda \
19+
# --trace-fork-before-exec=true \
20+
# --cuda-graph-trace node \
21+
# --force-overwrite true \
22+
# -o "$NSYS_FILE".nsys-rep \
1923

20-
function cleanup() {
21-
echo "Shutting down MPS control daemon..."
22-
echo quit | nvidia-cuda-mps-control
23-
echo "MPS control daemon shut down."
24-
}
25-
26-
27-
python3 inter_sm/mps_test.py \
28-
--model $MODEL \
29-
--profile $PROFILE_TYPE \
30-
--prompt_size $PROMPT_SIZE \
31-
--num_tokens $NUM_TOKENS \
32-
--batch_size $BATCH_SIZE \
33-
--mps $MPS_PERC \
34-
--interference_config_json $INTER_FILE \
35-
--inter_tb 66
24+
mkdir -p $RESULTS
25+
for nf in "${NUM_FLOATS[@]}"
26+
do
27+
echo "$nf"
28+
python3 inter_sm/gcontext_test.py \
29+
--model $MODEL \
30+
--profile $PROFILE_TYPE \
31+
--prompt_size $PROMPT_SIZE \
32+
--num_tokens $NUM_TOKENS \
33+
--batch_size $BATCH_SIZE \
34+
--num_requests $NUM_REQUESTS \
35+
--interference_config_json $INTER_FILE \
36+
--inter_tb $TB \
37+
--inter_num_floats $nf \
38+
--inter_decode > "$RESULTS/output_l2_$nf.txt"
39+
done

‎inter_sm/benchmark_mem.sh‎

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,38 +6,36 @@ NUM_TOKENS=10
66
PROMPT_SIZE=1000
77
PROFILE_TYPE=decode
88

9-
MPS_PERC=50
9+
NUM_REQUESTS=10
1010
INTER_FILE="inter_sm/copy_kernel_config.json"
1111
RESULTS="inter_sm/results_mem"
12+
NSYS_FILE="output"
13+
PLOTS="inter_sm/plots"
1214

13-
TB_SIZES=(0 33 66 99 132)
15+
TB_SIZES=(0 34 68 102 136)
1416

17+
# nsys profile \
18+
# -w true \
19+
# --trace cuda \
20+
# --trace-fork-before-exec=true \
21+
# --cuda-graph-trace node \
22+
# --force-overwrite true \
23+
# -o "$NSYS_FILE".nsys-rep \
1524

16-
echo "------------------------------------"
17-
echo "Starting MPS"
18-
export CUDA_VISIBLE_DEVICES=0
19-
export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps # Select a location that’s accessible to the given $UID
20-
export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log # Select a location that’s accessible to the given $UID
21-
nvidia-cuda-mps-control -d
22-
23-
function cleanup() {
24-
echo "Shutting down MPS control daemon..."
25-
echo quit | nvidia-cuda-mps-control
26-
echo "MPS control daemon shut down."
27-
}
28-
25+
mkdir -p $RESULTS
26+
mkdir -p $PLOTS
2927
for tb in "${TB_SIZES[@]}"
3028
do
3129
echo "$tb"
32-
python3 inter_sm/mps_test.py \
30+
python3 inter_sm/gcontext_test.py \
3331
--model $MODEL \
3432
--profile $PROFILE_TYPE \
3533
--prompt_size $PROMPT_SIZE \
3634
--num_tokens $NUM_TOKENS \
3735
--batch_size $BATCH_SIZE \
38-
--mps $MPS_PERC \
36+
--num_requests $NUM_REQUESTS \
3937
--interference_config_json $INTER_FILE \
40-
--inter_tb $tb > "$RESULTS/output_mem_$tb"
38+
--inter_tb $tb \
39+
--inter_decode > "$RESULTS/output_mem_$tb.txt"
4140
done
42-
43-
python inter_sm/process_results.py $RESULTS inter_sm/mem_tb_nums.png
41+
python inter_sm/process_results.py --output_dir $RESULTS --fig_path $PLOTS/mem.png --num_tokens $NUM_TOKENS --vary_grid

‎inter_sm/benchmark_softmax.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
NSYS_FILE="output_softmax_gcontexts"
4+
INTER_FILE="inter_sm/comp_kernel_config.json"
5+
6+
# echo "------------------------------------"
7+
# echo "Starting MPS"
8+
# export CUDA_VISIBLE_DEVICES=0
9+
# export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps # Select a location that’s accessible to the given $UID
10+
# export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log # Select a location that’s accessible to the given $UID
11+
# nvidia-cuda-mps-control -d
12+
13+
# function cleanup() {
14+
# echo "Shutting down MPS control daemon..."
15+
# echo quit | nvidia-cuda-mps-control
16+
# echo "MPS control daemon shut down."
17+
# }
18+
19+
nsys profile \
20+
-w true \
21+
--trace cuda \
22+
--trace-fork-before-exec=true \
23+
--cuda-graph-trace node \
24+
--force-overwrite true \
25+
-o "$NSYS_FILE".nsys-rep \
26+
python3 inter_sm/test_softmax.py \
27+
--interference_config_json $INTER_FILE \
28+
--inter_tb 64

‎inter_sm/comp_kernel_config.json‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
{
2-
"kernel_name": "ComputeKernel",
3-
"init_args": {
4-
"shared_lib": "/home/fot/vllm_profile/vllm/interference/build/libinterference_kernels.so",
5-
"num_floats": 128
6-
},
7-
"run_args": {
8-
"num_tb": 66,
9-
"num_threads_per_block": 128,
10-
"num_itrs": 1000000000
11-
}
2+
"kernel_name": "ComputeKernel",
3+
"init_args": {
4+
"shared_lib": "/scratch/fstrati/vllm_profile/vllm/interference/build/libinterference_kernels.so",
5+
"use_gcontexts": true,
6+
"num_floats": 1024
7+
},
8+
"run_args": {
9+
"num_tb": 136,
10+
"num_threads_per_block": 1024,
11+
"num_itrs": 3000000,
12+
"kernel_name": "fma_fp32_ilp4"
13+
},
14+
"num_requests": 10,
15+
"inter_sm": true,
16+
"intra_sm": false
1217
}

‎inter_sm/copy_kernel_config.json‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
{
2-
"kernel_name": "CopyKernel",
3-
"init_args": {
4-
"shared_lib": "/home/fot/vllm_profile/vllm/interference/build/libinterference_kernels.so",
5-
"num_floats": 429496729
6-
},
7-
"run_args": {
8-
"num_tb": 33,
9-
"num_threads_per_block": 1024,
10-
"num_floats": 429496729,
11-
"num_itrs": 10000
12-
}
2+
"kernel_name": "CopyKernel",
3+
"init_args": {
4+
"shared_lib": "/scratch/fstrati/vllm_profile/vllm/interference/build/libinterference_kernels.so",
5+
"use_gcontexts": true,
6+
"num_floats": 1073741824
7+
},
8+
"run_args": {
9+
"num_tb": 136,
10+
"num_threads_per_block": 1024,
11+
"num_floats": 1073741824,
12+
"num_itrs": 100
13+
},
14+
"num_requests": 10,
15+
"inter_sm": true,
16+
"intra_sm": false
1317
}

0 commit comments

Comments
 (0)