Feedsim Min tuning

Summary:
This diff reduce the execution time of Feedsim mini by eliminating fixed sleeps, replacing them with checking for QPS, and tunning configuration parameters.

### Key Changes:

**1. Reduced Test Durations:**

*   `fixed_qps_duration`
*   `warmup_time`

**2. Enhanced Warmup Control:**

*   Added QPS threshold-based steady state detection (`-r` parameter)
*   Added maximum warmup iterations parameter (`-x` parameter) with default of 60 iterations (configurable to 10 in jobs config)
*   Dynamic warmup that stops when QPS increase falls below threshold (0.1) instead of relying solely on fixed time

**3. Improved Monitoring & Debugging:**

*   Enhanced logging throughout the warmup process with iteration-by-iteration QPS and latency reporting
*   Better state reporting for debugging benchmark execution

Rollback Plan:

Differential Revision: D81509229

Author: marziehlenjaniMeta

benchpress/config/jobs.yml

@@ -472,14 +472,19 @@
     - '-q {fixed_qps}'
     - '-d {fixed_qps_duration}'
     - '-w {warmup_time}'
+    - '-r {qps_percentage_threshold}'
+    - '-x {max_warmup_iterations}'
     - '-S {graph_store_path}'
     - '-L {graph_load_path}'
+    - '-N'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
-    - 'fixed_qps=100'
-    - 'fixed_qps_duration=300'
-    - 'warmup_time=120'
+    - 'fixed_qps=100000'
+    - 'fixed_qps_duration=10'
+    - 'warmup_time=4'
+    - 'qps_percentage_threshold=-1'
+    - 'max_warmup_iterations=-1'
     - 'graph_store_path=default_do_not_store'
     - 'graph_load_path=default_do_not_load'
     - 'extra_args='
@@ -535,13 +540,20 @@
     - '-q {fixed_qps}'
     - '-d {fixed_qps_duration}'
     - '-w {warmup_time}'
+    - '-r {qps_percentage_threshold}'
+    - '-x {max_warmup_iterations}'
+    - '-S {graph_store_path}'
+    - '-L {graph_load_path}'
+    - '-N'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
     - 'icache_iterations=400000'
-    - 'fixed_qps=100'
-    - 'fixed_qps_duration=300'
-    - 'warmup_time=120'
+    - 'fixed_qps=100000'
+    - 'fixed_qps_duration=10'
+    - 'warmup_time=4'
+    - 'qps_percentage_threshold=-1'
+    - 'max_warmup_iterations=-1'
     - 'graph_store_path=default_do_not_store'
     - 'graph_load_path=default_do_not_load'
     - 'extra_args='

packages/feedsim/run-feedsim-multi.sh

@@ -3,7 +3,6 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
-
 BREPS_LFILE=/tmp/feedsim_log.txt
 IS_FIXED_QPS=0
 FIXQPS_SUFFIX=""
@@ -28,9 +27,6 @@ Usage: ${0##*/} [OPTION]...
     -h Display this help and exit
     -n Number of parallel instances to run. Default: $(( ( NCPU + 99 ) / 100 ))
     -i Number of icache iterations to use. Default: 1600000
-    -S Store the generated graph to a file (requires a file path)
-    -L Load a graph from a file instead of generating one (requires a file path)
-    -I Enable timing instrumentation for graph operations (build, store, load)
 
 Any remaining arguments are passed to run.sh
 
@@ -40,10 +36,6 @@ EOF
 SCRIPT_NAME="$(basename "$0")"
 echo "${SCRIPT_NAME}: DCPERF_PERF_RECORD=${DCPERF_PERF_RECORD}"
 
-# Initialize variables for graph storage and loading
-STORE_GRAPH=""
-LOAD_GRAPH=""
-INSTRUMENT_GRAPH=""
 
 while [ $# -ne 0 ]; do
     case $1 in
@@ -55,15 +47,7 @@ while [ $# -ne 0 ]; do
         -i)
             NUM_ICACHE_ITERATIONS="$2"
             ;;
-        -S)
-            STORE_GRAPH="-S $2"
-            ;;
-        -L)
-            LOAD_GRAPH="-L $2"
-            ;;
-        -I)
-            INSTRUMENT_GRAPH="-I"
-            ;;
+
         -h|--help)
             show_help >&2
             exit 1
@@ -73,7 +57,7 @@ while [ $# -ne 0 ]; do
     esac
 
     case $1 in
-        -n|-i|-S|-L)
+        -n|-i)
             if [ -z "$2" ]; then
                 echo "Invalid option: '$1' requires an argument" 1>&2
                 exit 1
@@ -129,10 +113,10 @@ echo > $BREPS_LFILE
 # shellcheck disable=SC2086
 for i in $(seq 1 ${NUM_INSTANCES}); do
     CORE_RANGE="$(get_cpu_range "${NUM_INSTANCES}" "$((i - 1))")"
-    CMD="IS_AUTOSCALE_RUN=${NUM_INSTANCES} taskset --cpu-list ${CORE_RANGE} ${FEEDSIM_ROOT}/run.sh -p ${PORT} -i ${NUM_ICACHE_ITERATIONS} -o feedsim_results_${FIXQPS_SUFFIX}${i}.txt ${STORE_GRAPH} ${LOAD_GRAPH} ${INSTRUMENT_GRAPH} $*"
+    CMD="IS_AUTOSCALE_RUN=${NUM_INSTANCES} taskset --cpu-list ${CORE_RANGE} ${FEEDSIM_ROOT}/run.sh -p ${PORT} -i ${NUM_ICACHE_ITERATIONS} -o feedsim_results_${FIXQPS_SUFFIX}${i}.txt  $*"
     echo "$CMD" > "${FEEDSIM_LOG_PREFIX}${i}.log"
     # shellcheck disable=SC2068,SC2069
-    IS_AUTOSCALE_RUN=${NUM_INSTANCES} stdbuf -i0 -o0 -e0 taskset --cpu-list "${CORE_RANGE}" "${FEEDSIM_ROOT}"/run.sh -p "${PORT}" -i "${NUM_ICACHE_ITERATIONS}" -o "feedsim_results_${FIXQPS_SUFFIX}${i}.txt" ${STORE_GRAPH} ${LOAD_GRAPH} ${INSTRUMENT_GRAPH} $@ 2>&1 > "${FEEDSIM_LOG_PREFIX}${i}.log" &
+    IS_AUTOSCALE_RUN=${NUM_INSTANCES} stdbuf -i0 -o0 -e0 taskset --cpu-list "${CORE_RANGE}" "${FEEDSIM_ROOT}"/run.sh -p "${PORT}" -i "${NUM_ICACHE_ITERATIONS}" -o "feedsim_results_${FIXQPS_SUFFIX}${i}.txt" $@ 2>&1 > "${FEEDSIM_LOG_PREFIX}${i}.log" &
     PIDS+=("$!")
     PHY_CORE_ID=$((PHY_CORE_ID + CORES_PER_INST))
     SMT_ID=$((SMT_ID + CORES_PER_INST))

packages/feedsim/run.sh

@@ -68,6 +68,10 @@ Usage: ${0##*/} [OPTION]...
     -S Store the generated graph to a file (requires a file path)
     -L Load a graph from a file instead of generating one (requires a file path)
     -I Enable timing instrumentation for graph operations (build, store, load)
+    -r QPS increase threshold for steady state detection during warmup (in percentage). If specified and greater than zero, warmup continues until
+       QPS increase is less than this threshold percentage of the previous QPS.
+    -x Maximum number of warmup iterations when using QPS threshold. Default: 10
+    -N No retry mode. Skip sleep and PID checking in load test startup, break immediately without retrying.
 EOF
 }
 
@@ -135,6 +139,14 @@ main() {
     local instrument_graph
     instrument_graph=""
 
+    local qps_threshold
+    qps_threshold=""
+
+    local max_warmup_iterations
+    max_warmup_iterations="10"
+
+    local no_retry_mode
+    no_retry_mode=""
 
     if [ -z "$IS_AUTOSCALE_RUN" ]; then
        echo > $BREPS_LFILE
@@ -189,6 +201,19 @@ main() {
             -I)
                 instrument_graph="--instrument_graph"
                 ;;
+            -r)
+                if [[ "$2" -gt 0 ]]; then
+                    qps_threshold="$2"
+                fi
+                ;;
+            -x)
+                if [[ "$2" -gt 0 ]]; then
+                    max_warmup_iterations="$2"
+                fi
+                ;;
+            -N)
+                no_retry_mode="1"
+                ;;
             -h|--help)
                 show_help >&2
                 exit 1
@@ -199,7 +224,7 @@ main() {
         esac
 
         case $1 in
-            -t|-c|-s|-d|-p|-q|-o|-w|-i|-l|-S|-L)
+            -t|-c|-s|-d|-p|-q|-o|-w|-i|-l|-S|-L|-r|-x)
                 if [ -z "$2" ]; then
                     echo "Invalid option: '$1' requires an argument" 1>&2
                     exit 1
@@ -268,9 +293,23 @@ main() {
 
     # Start DriverNode
     client_monitor_port="$((monitor_port-1000))"
+
+    # Construct QPS threshold and max iterations parameters if specified
+    qps_threshold_args=""
+    if [ -n "$qps_threshold" ]; then
+        qps_threshold_args="-r $qps_threshold -x $max_warmup_iterations"
+    fi
+
+    # Construct no retry mode parameter if specified
+    no_retry_args=""
+    if [ -n "$no_retry_mode" ]; then
+        no_retry_args="-N"
+    fi
+
     if [ -z "$fixed_qps" ] && [ "$auto_driver_threads" != "1" ]; then
         benchreps_tell_state "before search_qps"
-        scripts/search_qps.sh -w 15 -f 300 -s 95p:500 -o "${FEEDSIM_ROOT}/${result_filename}" -- \
+        # shellcheck disable=SC2086
+        scripts/search_qps.sh -w 15 -f 300 -s 95p:500 $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \
             build/workloads/ranking/DriverNodeRank \
                 --server "0.0.0.0:$port" \
                 --monitor_port "$client_monitor_port" \
@@ -279,7 +318,8 @@ main() {
         benchreps_tell_state "after search_qps"
     elif [ -z "$fixed_qps" ] && [ "$auto_driver_threads" = "1" ]; then
         benchreps_tell_state "before search_qps"
-        scripts/search_qps.sh -a -w 15 -f 300 -s 95p:500 -o "${FEEDSIM_ROOT}/${result_filename}" -- \
+        # shellcheck disable=SC2086
+        scripts/search_qps.sh -a -w 15 -f 300 -s 95p:500 $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \
             build/workloads/ranking/DriverNodeRank \
                 --monitor_port "$client_monitor_port" \
                 --server "0.0.0.0:$port"
@@ -296,9 +336,11 @@ main() {
             num_workers=$driver_threads
         fi
         benchreps_tell_state "before fixed_qps_exp"
+        # shellcheck disable=SC2086
         scripts/search_qps.sh -s 95p -t "$fixed_qps_duration" \
            -m "$warmup_time" \
            -q "$fixed_qps" \
+           $qps_threshold_args $no_retry_args \
            -o "${FEEDSIM_ROOT}/${result_filename}" \
            -- build/workloads/ranking/DriverNodeRank \
                 --server "0.0.0.0:$port" \

packages/feedsim/third_party/src/scripts/search_qps.sh

@@ -77,6 +77,12 @@ mutilate (EuroSys \'14) [https://github.com/leverich/mutilate]
                 appending '--threads=T --connections=4' to the driver command during load
                 tests. T will be the lesser of requested_qps / 4 or $(nproc) / 5.
     -o          output filename to record samples as csv. Optional
+    -r          QPS increase threshold for steady state detection during warmup (in percentage).
+                When specified, warmup continues until QPS increase is less than this threshold
+                percentage of the previous QPS. Optional
+    -x          Maximum number of warmup iterations when using QPS threshold. Default: 10
+    -N          No retry mode. Skip sleep and PID checking in load test startup, break immediately
+                without retrying. Optional
 EOF
 }
 
@@ -122,19 +128,28 @@ run_loadtest() {
     local tmp_file=$(mktemp)
     $command $threads_arg $qps_arg &>$tmp_file &
     LOADTEST_PID=$!
-    sleep 7
-    ps -p $LOADTEST_PID -o pid= > /dev/null && break
-    echo "Retrying $r of 3 to start load test..."
+
+    if [ "$no_retry_mode" = "1" ]; then
+      # No retry mode: skip sleep and PID checking, break immediately
+      break
+    else
+      # Normal mode: sleep and check PID
+      sleep 7
+      ps -p $LOADTEST_PID -o pid= > /dev/null && break
+      benchreps_tell_state "Retrying $r of 3 to start load test..."
+    fi
   done
 
   # wait for time
   sleep $experiment_time
+  benchreps_tell_state "after sleeping for experiment_time=${experiment_time} seconds"
 
   # send SIGINT to the command
   kill -SIGINT $LOADTEST_PID
 
   # wait for results to show up and queries to drain
   sleep $wait_time
+  benchreps_tell_state "after sleeping for wait_time=${wait_time} seconds"
 
   # check file for QPS
   if grep -q "#: [0-9]\+.\([0-9]\+\)\? QPS" $tmp_file; then
@@ -216,9 +231,12 @@ load_test_retries=3
 output_csv_file=""
 fixed_qps=""
 auto_driver_threads=""
+qps_threshold=""
+max_warmup_iterations=10
+no_retry_mode=""
 
 OPTIND=1 # Reset is necessary if getopts was used previously in the script.  It is a good idea to make this local in a function.
-while getopts "ht:f:w:m:s:q:ao:" opt; do
+while getopts "ht:f:w:m:s:q:ao:r:x:N" opt; do
   case "$opt" in
     h)
       show_help
@@ -249,6 +267,15 @@ while getopts "ht:f:w:m:s:q:ao:" opt; do
     o)
       output_csv_file=$OPTARG
       ;;
+    r)
+      qps_threshold=$OPTARG
+      ;;
+    x)
+      max_warmup_iterations=$OPTARG
+      ;;
+    N)
+      no_retry_mode=1
+      ;;
     '?')
       show_help >&2
       exit 1
@@ -322,8 +349,54 @@ if [ "$warmup_time" -gt 0 ]; then
   benchreps_tell_state "before warmup"
   saved_experiment_time="$experiment_time"
   experiment_time="$warmup_time"
-  run_loadtest peak_qps measured_latency
+
+  if [ -n "$qps_threshold" ]; then
+    # Dynamic QPS-based warmup
+    previous_qps=0
+    warmup_iteration=0
+
+    while [ $warmup_iteration -lt $max_warmup_iterations ]; do
+      run_loadtest current_qps measured_latency
+      benchreps_tell_state "after iteration $warmup_iteration"
+      printf "warmup iteration %d: qps = %.2f, latency = %.2f\n" $warmup_iteration $current_qps $measured_latency >> $BREPS_LFILE
+
+      # Skip first warmup iteration, always do the second round
+      if [ $warmup_iteration -gt 1 ]; then
+        # Calculate percentage increase: (current - previous) / previous * 100
+        if [ $(echo "$previous_qps > 0" | bc) -eq 1 ]; then
+          qps_increase_percentage=$(echo "scale=5; ($current_qps - $previous_qps) / $previous_qps * 100" | bc)
+          # Convert to absolute value for comparison
+          #qps_increase_abs=$(echo "scale=5; ($qps_increase_percentage^2)^0.5" | bc)
+          #qps_increase_check=$(echo "$qps_increase_abs <= $qps_threshold" | bc)
+          qps_increase_check=$(echo "$qps_increase_percentage <= $qps_threshold" | bc)
+
+          if [ $qps_increase_check -eq 1 ]; then
+            printf "QPS steady state reached. QPS increase percentage (%.2f%%) is less than threshold (%.2f%%)\n" $qps_increase_percentage $qps_threshold >> $BREPS_LFILE
+            break
+          fi
+        else
+          # If previous_qps is 0, skip the check for this iteration
+          printf "Previous QPS is 0, skipping threshold check for iteration %d\n" $warmup_iteration >> $BREPS_LFILE
+        fi
+      fi
+
+      previous_qps=$current_qps
+      warmup_iteration=$((warmup_iteration + 1))
+
+      # Wait between warmup iterations (except for the last one)
+      if [ $warmup_iteration -lt $max_warmup_iterations ]; then
+        sleep $wait_time
+      fi
+    done
+
+    peak_qps=$current_qps
+  else
+    # Original fixed warmup time
+    run_loadtest peak_qps measured_latency
+  fi
+
   printf "warmup qps = %.2f, latency = %.2f\n" $peak_qps $measured_latency
+  echo "warmup qps = $peak_qps, latency = $measured_latency" >> $BREPS_LFILE
   benchreps_tell_state "after warmup"
   experiment_time="$saved_experiment_time"
 fi
@@ -338,13 +411,16 @@ if [[ -n "$fixed_qps" ]]; then
         collect_perf_record &
     fi
     run_loadtest measured_qps measured_latency $fixed_qps
+
     printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $fixed_qps $measured_qps $measured_latency
+    echo "final requested_qps = $fixed_qps, measured_qps = $measured_qps, latency = $measured_latency" >> $BREPS_LFILE
     benchreps_tell_state "after fixed_qps_single"
   else
     for fixed_qps_el in $fixed_qps_array; do
       benchreps_tell_state "before fixed_qps_iter $fixed_qps_el"
       run_loadtest measured_qps measured_latency $fixed_qps_el
       printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $fixed_qps_el $measured_qps $measured_latency
+      echo "final requested_qps = $fixed_qps_el, measured_qps = $measured_qps, latency = $measured_latency" >> $BREPS_LFILE
       benchreps_tell_state "after fixed_qps_iter $fixed_qps_el"
       sleep 7 # wait between iterations
     done