Feedsim: removing randomness

Summary:
This diff reduce score variations in feedsim by controlling random number generation. This change is critical for ensuring fair and accurate hardware performance comparisons.

This is crucial for:

*   **Vendor Hardware Comparisons**: Eliminating noise from random variations when comparing different hardware platforms
*   **DCPerf Mini Reliability**: Short-duration benchmarks are especially sensitive to random variations that can skew results
*   **Fair Performance Evaluation**: Ensuring benchmark scores reflect true hardware performance characteristics
*   **Scientific Rigor**: Providing reproducible results for performance analysis and decision-making

This implementation provides **granular control** over all three sources of randomness in feedsim:

#### **1. NodeRank Random Number Generation (`--node_rank_seed`)**

*   Controls thread-local random number generators used in LeafNodeRank
*   Affects latency distribution sampling and general RNG operations
*   **Usage**: `-R {seed_value}` or `--node_rank_seed={seed_value}`

#### **2. PageRank Algorithm Randomness (`--page_rank_seed`)**

*   Controls PageRank graph traversal and subset selection randomness
*   Affects graph partitioning and algorithm initialization
*   **Usage**: `-P {seed_value}` or `--page_rank_seed={seed_value}`

#### **3. PointerChase Memory Access Patterns (`--pointer_chase_seed`)** _(New Feature)_

*   Controls memory access pattern randomization in PointerChase workload
*   Affects cache behavior simulation and memory stress patterns
*   **Usage**: `-C {seed_value}` or `--pointer_chase_seed={seed_value}`

Differential Revision: D83871667

Author: marziehlenjaniMeta

benchpress/config/jobs.yml

@@ -477,6 +477,9 @@
     - '-S {graph_store_path}'
     - '-L {graph_load_path}'
     - '-D {queue_drain_time}'
+    - '-R {node_rank_seed}'
+    - '-P {page_rank_seed}'
+    - '-C {pointer_chase_seed}'
     - '-N'
     - '{extra_args}'
   vars:
@@ -489,6 +492,9 @@
     - 'graph_store_path=default_do_not_store'
     - 'graph_load_path=default_do_not_load'
     - 'queue_drain_time=1'
+    - 'node_rank_seed=54321'
+    - 'page_rank_seed=12345'
+    - 'pointer_chase_seed=98765'
     - 'extra_args='
   hooks:
     - hook: copymove

packages/feedsim/run.sh

@@ -73,6 +73,9 @@ Usage: ${0##*/} [OPTION]...
     -x Maximum number of warmup iterations when using QPS threshold. Default: 10
     -N No retry mode. Skip sleep and PID checking in load test startup, break immediately without retrying.
     -D Drain time in seconds. Time to wait for queue to drain after experiments. Default: 5
+    -R Seed for LeafNodeRank random number generator. If not provided, current time will be used.
+    -P Seed for PageRank random number generator. If not provided, current time will be used.
+    -C Seed for PointerChase random number generator. If not provided, current time will be used.
 EOF
 }
 
@@ -152,6 +155,15 @@ main() {
     local queue_drain_time
     queue_drain_time="5"
 
+    local leafnoderank_seed
+    leafnoderank_seed=""
+
+    local pagerank_seed
+    pagerank_seed=""
+
+    local pointerchase_seed
+    pointerchase_seed=""
+
     if [ -z "$IS_AUTOSCALE_RUN" ]; then
        echo > $BREPS_LFILE
     fi
@@ -221,6 +233,15 @@ main() {
             -D)
                 queue_drain_time="$2"
                 ;;
+            -R)
+                leafnoderank_seed="--node_rank_seed=$2"
+                ;;
+            -P)
+                pagerank_seed="--page_rank_seed=$2"
+                ;;
+            -C)
+                pointerchase_seed="--pointer_chase_seed=$2"
+                ;;
             -h|--help)
                 show_help >&2
                 exit 1
@@ -231,7 +252,7 @@ main() {
         esac
 
         case $1 in
-            -t|-c|-s|-d|-p|-q|-o|-w|-i|-l|-S|-L|-r|-x|-D)
+            -t|-c|-s|-d|-p|-q|-o|-w|-i|-l|-S|-L|-r|-x|-D|-R|-P|-C)
                 if [ -z "$2" ]; then
                     echo "Invalid option: '$1' requires an argument" 1>&2
                     exit 1
@@ -270,7 +291,10 @@ main() {
         --min_icache_iterations="$icache_iterations" \
         "$store_graph" \
         "$load_graph" \
-        "$instrument_graph" >> $BREPS_LFILE 2>&1 &
+        "$instrument_graph" \
+        "$leafnoderank_seed" \
+        "$pagerank_seed" \
+        "$pointerchase_seed" >> $BREPS_LFILE 2>&1 &
 
     LEAF_PID=$!
 

packages/feedsim/third_party/src/workloads/ranking/LeafNodeRank.cc

@@ -95,15 +95,35 @@ void ThreadStartup(
   this_thread.srvIOThreadPool = srvIOThreadPool;
   this_thread.ioThreadPool = ioThreadPool;
   this_thread.timekeeperPool = timekeeperPool;
+  unsigned noderank_seed;
+  if (args.node_rank_seed_given) {
+    noderank_seed = static_cast<unsigned>(args.node_rank_seed_arg);
+  } else {
+    noderank_seed = std::chrono::system_clock::now().time_since_epoch().count();
+  }
+
+  unsigned page_rank_seed;
+  if (args.page_rank_seed_given) {
+    page_rank_seed = static_cast<unsigned>(args.page_rank_seed_arg);
+  } else {
+    page_rank_seed = std::chrono::system_clock::now().time_since_epoch().count();
+  }
+
+  unsigned pointer_chase_seed;
+  if (args.pointer_chase_seed_given) {
+    pointer_chase_seed = static_cast<unsigned>(args.pointer_chase_seed_arg);
+  } else {
+    pointer_chase_seed =
+        std::chrono::system_clock::now().time_since_epoch().count();
+  }
+
   this_thread.page_ranker = std::make_unique<ranking::dwarfs::PageRank>(
-      std::move(graph), args.cpu_threads_arg);
+      std::move(graph), args.cpu_threads_arg, page_rank_seed);
   this_thread.icache_buster =
       std::make_unique<ICacheBuster>(kNumICacheBusterMethods);
-  this_thread.pointer_chaser =
-      std::make_unique<search::PointerChase>(kPointerChaseSize);
-
-  unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
-  this_thread.rng.seed(seed);
+  this_thread.pointer_chaser = std::make_unique<search::PointerChase>(
+      kPointerChaseSize, pointer_chase_seed);
+  this_thread.rng.seed(noderank_seed);
 
   const double alpha = 0.7;
   const double beta = 20000;
@@ -319,8 +339,12 @@ int main(int argc, char** argv) {
       auto start_load = std::chrono::steady_clock::now();
       g_shared_graph = params.loadGraphFromFile(args.load_graph_arg);
       auto end_load = std::chrono::steady_clock::now();
-      auto load_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_load - start_load).count();
-      std::cout << "Graph loading time: " << load_duration << " ms" << std::endl;
+      auto load_duration =
+          std::chrono::duration_cast<std::chrono::milliseconds>(
+              end_load - start_load)
+              .count();
+      std::cout << "Graph loading time: " << load_duration << " ms"
+                << std::endl;
     } else {
       g_shared_graph = params.loadGraphFromFile(args.load_graph_arg);
     }
@@ -329,15 +353,23 @@ int main(int argc, char** argv) {
       auto start_build = std::chrono::steady_clock::now();
       g_shared_graph = params.buildGraph();
       auto end_build = std::chrono::steady_clock::now();
-      auto build_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_build - start_build).count();
-      std::cout << "Graph building time: " << build_duration << " ms" << std::endl;
+      auto build_duration =
+          std::chrono::duration_cast<std::chrono::milliseconds>(
+              end_build - start_build)
+              .count();
+      std::cout << "Graph building time: " << build_duration << " ms"
+                << std::endl;
 
       if (args.store_graph_given) {
         auto start_store = std::chrono::steady_clock::now();
         params.storeGraphToFile(g_shared_graph, args.store_graph_arg);
         auto end_store = std::chrono::steady_clock::now();
-        auto store_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_store - start_store).count();
-        std::cout << "Graph storing time: " << store_duration << " ms" << std::endl;
+        auto store_duration =
+            std::chrono::duration_cast<std::chrono::milliseconds>(
+                end_store - start_store)
+                .count();
+        std::cout << "Graph storing time: " << store_duration << " ms"
+                  << std::endl;
       }
     } else {
       g_shared_graph = params.buildGraph();

packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo

@@ -35,6 +35,9 @@ option "max_response_size" - "Maximum response size in bytes returned by the lea
 option "compression_data_size" - "Number of bytes to compress per request." int default="131072"
 option "rank_trials_per_thread" - "Number of iterations each CPU thread executes of rank work." int default="1"
 option "min_icache_iterations" - "At least this number of icache busting iteration will be executed." int default="0"
+option "node_rank_seed" - "Seed for random number generator. If not provided, current time will be used." long optional
+option "page_rank_seed" - "Seed for PageRank random number generator. If not provided, current time will be used." long optional
+option "pointer_chase_seed" - "Seed for PointerChase random number generator. If not provided, current time will be used." long optional
 option "chase_iterations" - "Number of chases to execute on handler thread." int default="5120"
 option "io_chase_iterations" - "Number of chases to execute on IO threads." int default="5120"
 option "io_time_ms" - "Milliseconds to sleep emualting I/O offcpu." int default="200"

packages/feedsim/third_party/src/workloads/ranking/dwarfs/pagerank.cpp

@@ -91,11 +91,11 @@ CSRGraph<int32_t> PageRankParams::makeGraphCopy(
       original.out_neigh(0).begin(),
       original.out_neigh(num_nodes - 1).end(),
       out_neighbors);
- // Set up index pointers for each node
- #pragma omp parallel for
+// Set up index pointers for each node
+#pragma omp parallel for
   for (int64_t n = 0; n < num_nodes; n++) {
-    //TODO: check this is correct
-    out_index[n] = out_neighbors  +
+    // TODO: check this is correct
+    out_index[n] = out_neighbors +
         (original.out_neigh(n).begin() - original.out_neigh(0).begin());
   }
   // Set the last index pointer
@@ -112,8 +112,8 @@ CSRGraph<int32_t> PageRankParams::makeGraphCopy(
         original.in_neigh(num_nodes - 1).end(),
         in_neighbors);
 
-    // Set up index pointers for each node
-    #pragma omp parallel for
+// Set up index pointers for each node
+#pragma omp parallel for
     for (int64_t n = 0; n < num_nodes; n++) {
       in_index[n] = in_neighbors +
           (original.in_neigh(n).begin() - original.in_neigh(0).begin());
@@ -177,7 +177,8 @@ void PageRankParams::storeGraphToFile(
       } else if (n == num_nodes) {
         in_offsets[n] = num_edges;
       } else {
-        in_offsets[n] = original.in_neigh(n).begin() - original.in_neigh(0).begin();
+        in_offsets[n] =
+            original.in_neigh(n).begin() - original.in_neigh(0).begin();
       }
     }
     outFile.write(
@@ -222,7 +223,7 @@ CSRGraph<int32_t> PageRankParams::loadGraphFromFile(
 
   // Create out_index pointer array
   int32_t** out_index = new int32_t*[num_nodes + 1];
-  #pragma omp parallel for
+#pragma omp parallel for
   for (int64_t n = 0; n <= num_nodes; n++) {
     out_index[n] = out_neighbors + out_offsets[n];
   }
@@ -241,7 +242,7 @@ CSRGraph<int32_t> PageRankParams::loadGraphFromFile(
 
     // Create in_index pointer array
     int32_t** in_index = new int32_t*[num_nodes + 1];
-    #pragma omp parallel for
+#pragma omp parallel for
     for (int64_t n = 0; n <= num_nodes; n++) {
       in_index[n] = in_neighbors + in_offsets[n];
     }
@@ -255,8 +256,13 @@ CSRGraph<int32_t> PageRankParams::loadGraphFromFile(
   }
 }
 
-PageRank::PageRank(CSRGraph<int32_t> graph, int num_pvectors_entries)
-    : graph_(std::move(graph)), num_pvectors_entries_(num_pvectors_entries) {
+PageRank::PageRank(
+    CSRGraph<int32_t> graph,
+    int num_pvectors_entries,
+    unsigned seed)
+    : graph_(std::move(graph)),
+      num_pvectors_entries_(num_pvectors_entries),
+      seed_(seed) {
   const float init_score = 1.0f / graph_.num_nodes();
   for (int i = 0; i < num_pvectors_entries; i++) {
     pvector<float> scores{graph_.num_nodes(), init_score};
@@ -285,14 +291,17 @@ int PageRank::rank(
       0,
       num_nodes < graph_.num_nodes() ? graph_.num_nodes() - num_nodes
                                      : graph_.num_nodes()};
-  std::random_device rd;
-  std::mt19937 gen(rd());
+  // Use the seed passed to the constructor
+  unsigned local_seed = seed_ != 0
+      ? seed_ + thread_id
+      : std::chrono::system_clock::now().time_since_epoch().count();
+  std::mt19937 gen(local_seed);
   NodeID start = u_dist(gen);
 
   const auto split_size = std::max(num_pvectors_entries_, 1);
   std::uniform_int_distribution<int64_t> split_dist{
       0, graph_.num_nodes() / split_size - 1};
-  std::mt19937 split_gen(rd());
+  std::mt19937 split_gen(local_seed + 1);
   NodeID split_start = split_dist(split_gen);
   NodeID split_end = split_start + (graph_.num_nodes() / split_size) - 1;
 

packages/feedsim/third_party/src/workloads/ranking/dwarfs/pagerank.h

@@ -45,11 +45,10 @@ class PageRankParams {
   CSRGraph<int32_t> makeGraphCopy(const CSRGraph<int32_t>& original);
 
   void storeGraphToFile(
-    const CSRGraph<int32_t>& original,
-    const std::string& filePath);
+      const CSRGraph<int32_t>& original,
+      const std::string& filePath);
 
-  CSRGraph<int32_t> loadGraphFromFile(
-    const std::string& filePath);
+  CSRGraph<int32_t> loadGraphFromFile(const std::string& filePath);
 
  private:
   struct Impl;
@@ -63,7 +62,10 @@ class PageRank {
  public:
   constexpr static const float kDamp = 0.85;
 
-  explicit PageRank(CSRGraph<int32_t> graph, int num_pvectors_entries);
+  explicit PageRank(
+      CSRGraph<int32_t> graph,
+      int num_pvectors_entries,
+      unsigned seed = 0);
 
   int rank(
       int thread_id,
@@ -75,6 +77,7 @@ class PageRank {
  private:
   CSRGraph<int32_t> graph_;
   int num_pvectors_entries_;
+  unsigned seed_;
   folly::F14FastMap<int, pvector<float>> scores_pvectors_map_;
   folly::F14FastMap<int, pvector<float>> outgoing_pvectors_map_;
 };

packages/feedsim/third_party/src/workloads/search/LeafNode.cc

@@ -26,9 +26,9 @@
 #include "oldisim/QueryContext.h"
 #include "oldisim/Util.h"
 
-#include "PointerChase.h"
 #include "ICacheBuster.h"
 #include "LeafNodeCmdline.h"
+#include "PointerChase.h"
 #include "RequestTypes.h"
 
 // Shared configuration flags
@@ -50,8 +50,9 @@ struct ThreadData {
   std::string random_string;
 };
 
-void ThreadStartup(oldisim::NodeThread& thread,
-                   std::vector<ThreadData>& thread_data) {
+void ThreadStartup(
+    oldisim::NodeThread& thread,
+    std::vector<ThreadData>& thread_data) {
   ThreadData& this_thread = thread_data[thread.get_thread_num()];
 
   // Initialize of I$Buster
@@ -61,8 +62,13 @@ void ThreadStartup(oldisim::NodeThread& thread,
   this_thread.icache_buster.reset(new ICacheBuster(kICacheBusterSize));
 
   // Initialize RNG and latency sampler
-  unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
-  this_thread.rng.seed(seed);
+  unsigned node_seed;
+  if (args.node_seed_given) {
+    node_seed = static_cast<unsigned>(args.node_seed_arg);
+  } else {
+    node_seed = std::chrono::system_clock::now().time_since_epoch().count();
+  }
+  this_thread.rng.seed(node_seed);
 
   const double alpha = 0.7;
   const double beta = 20000;
@@ -73,9 +79,10 @@ void ThreadStartup(oldisim::NodeThread& thread,
   this_thread.random_string = RandomString(kMaxResponseSize);
 }
 
-void SearchRequestHandler(oldisim::NodeThread& thread,
-                          oldisim::QueryContext& context,
-                          std::vector<ThreadData>& thread_data) {
+void SearchRequestHandler(
+    oldisim::NodeThread& thread,
+    oldisim::QueryContext& context,
+    std::vector<ThreadData>& thread_data) {
   ThreadData& this_thread = thread_data[thread.get_thread_num()];
   search::PointerChase& chaser = *this_thread.pointer_chaser;
   ICacheBuster& buster = *this_thread.icache_buster;
@@ -123,9 +130,11 @@ int main(int argc, char** argv) {
       std::bind(ThreadStartup, std::placeholders::_1, std::ref(thread_data)));
   server.RegisterQueryCallback(
       search::kSearchRequestType,
-      std::bind(SearchRequestHandler, std::placeholders::_1,
-                std::placeholders::_2,
-                std::ref(thread_data)));
+      std::bind(
+          SearchRequestHandler,
+          std::placeholders::_1,
+          std::placeholders::_2,
+          std::ref(thread_data)));
 
   server.SetNumThreads(args.threads_arg);
   server.SetThreadPinning(!args.noaffinity_given);

packages/feedsim/third_party/src/workloads/search/LeafNodeCmdline.ggo

@@ -25,5 +25,6 @@ option "quiet" - "Disable log messages."
 option "threads" - "Number of threads to use for serving." int default="1"
 option "port" - "Port to run server on." int default="11222"
 option "monitor_port" - "Port to run monitoring server on." int default="8888"
+option "node_seed" - "Seed for random number generator. If not provided, current time will be used." long optional
 option "noaffinity" - "Specify to disable thread pinning"
 option "noloadbalance" - "Specify to disable thread load balancing"