Sample response size from rpc_dist_v2 + bump keepalive default to 200ms (#745)

benchpress/config/jobs.yml

@@ -646,6 +646,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors={feature_extractors}'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
   vars:
     - 'port=11222'
     - 'output=feedsim_results.txt'
@@ -655,13 +659,17 @@
     - 'dlrm_model=models/dlrm_small.pt'
     - 'dlrm_batch_size=256'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    - 'dlrm_inferences=1'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_extractors=0'
+    - 'feature_complexity=5'
+    - 'num_stories=100'
+    - 'extractors_per_story=50'
   hooks:
     - hook: cpu-mpstat
       options:
@@ -731,22 +739,47 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
+    - '--story-processors-per-story={story_processors_per_story}'
+    - '--stories-per-processor-pass={stories_per_processor_pass}'
+    - '--silesia-dir={silesia_dir}'
+    - '--stories-per-request={stories_per_request}'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
     - 'io_dist=fixed'
     - 'io_mean=200'
     - 'workload=dlrm'
     - 'dlrm_model=models/dlrm_small.pt'
-    - 'dlrm_batch_size=256'
+    # 32: half of t29 baseline 64. t32 BGM still showed Ranking-Prediction
+    # at 17.8% vs prod 5.9% with batch=64 — halving the batch ≈ halves the
+    # per-inference DLRM compute and brings it closer to prod's share.
+    - 'dlrm_batch_size=32'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    - 'dlrm_inferences=1'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_complexity=5'
+    # 400 x 280 = 112K extractor calls/req. Combined with the scaled-up
+    # story-processor pipeline (below) this matches prod's FE:story CPU
+    # ratio of ~1.3:1 (multifeed: FE 29%, story 22% on BGM Strobelight).
+    - 'num_stories=400'
+    - 'extractors_per_story=280'
+    # Story-processor pipeline scaled 8x from t32 first attempt (was
+    # 1 pass/story * 25 stories/pass = 25 cmpns/story → 1.1% Ranking-Story
+    # on BGM, vs prod 9.3%). Bumped to 2 passes * 100 stories = 200
+    # comparisons/story, target ~9% Ranking-Story share.
+    - 'story_processors_per_story=2'
+    - 'stories_per_processor_pass=100'
+    - 'silesia_dir=silesia'
+    - 'stories_per_request=10'
     - 'extra_args='
   hooks:
     - hook: cpu-mpstat
@@ -761,6 +794,7 @@
           - 'benchmarks/feedsim/feedsim_results*.txt'
           - 'benchmarks/feedsim/feedsim-multi-inst-*.log'
           - 'benchmarks/feedsim/src/perf.data'
+          - 'benchmarks/feedsim/breakdown.csv'
           - '/tmp/feedsim_log.txt'
 
 - name: feedsim_autoscale_dlrm_mini
@@ -785,6 +819,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
     - '-q {fixed_qps}'
     - '-d {fixed_qps_duration}'
     - '-w {warmup_time}'
@@ -804,13 +842,21 @@
     - 'dlrm_model=models/dlrm_small.pt'
     - 'dlrm_batch_size=256'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    # 8 inferences/req brings DLRM-Inference CPU share from 55-62% down
+    # toward prod multifeed_aggregator's Ranking-Prediction share of 7-13%.
+    - 'dlrm_inferences=8'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_complexity=5'
+    # 1800 stories x 50 extractors = 90K extractor calls/req. Tuned to
+    # push FeatureExtraction CPU share into prod's 30-35% band; 5K
+    # calls/req previously gave only 1.7-1.9%.
+    - 'num_stories=1800'
+    - 'extractors_per_story=50'
     - 'fixed_qps=100000'
     - 'fixed_qps_duration=10'
     - 'warmup_time=5'

benchpress/plugins/hooks/perf_monitors/__init__.py

@@ -98,17 +98,45 @@ def run(self):
 
     def terminate(self):
         """
-        Kill the monitoring process using SIGINT signal and join the stdout
-        and stderr catcher threads.
+        Kill the monitoring process and join its stdout/stderr catcher threads.
+
+        The monitor's `proc` is typically a bash wrapper script (e.g.
+        `perfutils/collect_amd_perf_counters.sh`) that itself spawns `perf
+        stat`. Signaling only the bash PID is unreliable because bash does
+        not synchronously forward signals to its `wait`-blocked foreground
+        child — leaving `perf stat` running and `proc.wait()` hanging
+        forever. We send SIGINT to the entire process group (which requires
+        the Popen to have used `start_new_session=True`) and bound the wait
+        with a timeout, escalating to SIGKILL if SIGINT isn't honored.
         """
         exitcode = -1
         if hasattr(self, "proc") and isinstance(self.proc, subprocess.Popen):
-            os.kill(self.proc.pid, signal.SIGINT)
-            exitcode = self.proc.wait()
+            try:
+                os.killpg(os.getpgid(self.proc.pid), signal.SIGINT)
+            except (ProcessLookupError, PermissionError):
+                pass
+            try:
+                exitcode = self.proc.wait(timeout=15)
+            except subprocess.TimeoutExpired:
+                logger.warning(
+                    f"{getattr(self, 'name', 'Monitor')}: SIGINT did not "
+                    "terminate within 15s, escalating to SIGKILL"
+                )
+                try:
+                    os.killpg(os.getpgid(self.proc.pid), signal.SIGKILL)
+                except (ProcessLookupError, PermissionError):
+                    pass
+                try:
+                    exitcode = self.proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    logger.error(
+                        f"{getattr(self, 'name', 'Monitor')}: SIGKILL did "
+                        "not reap process within 5s; leaving as orphan"
+                    )
         if hasattr(self, "oc") and isinstance(self.oc, threading.Thread):
-            self.oc.join()
+            self.oc.join(timeout=5)
         if hasattr(self, "ec") and isinstance(self.ec, threading.Thread):
-            self.ec.join()
+            self.ec.join(timeout=5)
         return exitcode
 
     def get_result(self):

benchpress/plugins/hooks/perf_monitors/mpstat.py

@@ -18,7 +18,17 @@ def __init__(self, interval, job_uuid):
 
     def run(self):
         args = ["mpstat", "-u", f"{self.interval}"]
-        self.proc = subprocess.Popen(args, stdout=subprocess.PIPE, encoding="utf-8")
+        self.proc = subprocess.Popen(
+            args,
+            stdout=subprocess.PIPE,
+            encoding="utf-8",
+            # CRITICAL: Run in its own process group so Monitor.terminate()
+            # can killpg() the bash wrapper + mpstat cleanly without
+            # signalling benchpress itself (which would raise
+            # KeyboardInterrupt and abort cleanup of all subsequent
+            # monitors).
+            start_new_session=True,
+        )
         super(MPStat, self).run()
 
     def process_output(self, line):

benchpress/plugins/hooks/perf_monitors/perfstat.py

@@ -93,5 +93,11 @@ def run(self):
             "--log-fd",
             "1",
         ]
-        self.proc = subprocess.Popen(args, stdout=subprocess.PIPE, encoding="utf-8")
+        self.proc = subprocess.Popen(
+            args,
+            stdout=subprocess.PIPE,
+            encoding="utf-8",
+            # Process-group isolation for clean teardown.
+            start_new_session=True,
+        )
         super(PerfStat, self).run()

benchpress/plugins/hooks/perf_monitors/topdown.py

@@ -185,7 +185,16 @@ def run(self):
             self.collect_output_path,
         ]
         self.proc = subprocess.Popen(
-            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"
+            args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding="utf-8",
+            # Run in its own process group so Monitor.terminate() can killpg()
+            # both the bash wrapper and its perf-stat child cleanly. Without
+            # this, SIGINT to the bash PID alone leaves perf-stat orphaned
+            # and parent .wait() hangs (root cause of the t29/t32 teardown
+            # hang on fb_chef_off_turbo_on + perf hook combo).
+            start_new_session=True,
         )
         super(IntelPerfSpect, self).run()
 
@@ -266,7 +275,12 @@ def run(self):
             self.collect_output_path,
         ]
         self.proc = subprocess.Popen(
-            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"
+            args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding="utf-8",
+            # Process-group isolation for clean teardown (see IntelPerfSpect).
+            start_new_session=True,
         )
         super(IntelPerfSpect3, self).run()
 
@@ -329,7 +343,14 @@ def run(self):
         cmd = [perf_collect_script]
         if self.interval is not None:
             cmd.append(str(self.interval))
-        self.proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf-8")
+        self.proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            encoding="utf-8",
+            # Process-group isolation for clean teardown — Monitor.terminate
+            # killpg()'s this so perf stat + bash wrapper both die together.
+            start_new_session=True,
+        )
         super(BasePerfUtil, self).run()
 
     def gen_csv(self):
@@ -494,6 +515,8 @@ def run(self):
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             encoding="utf-8",
+            # Process-group isolation for clean teardown.
+            start_new_session=True,
         )
         super(ARMPerfUtil, self).run()
 

packages/common/runtime_breakdown_utils.sh

@@ -25,6 +25,16 @@ create_breakdown_csv() {
 
     local csv_file="${folder_path}/${breakdown_file_name}"
 
+    # Idempotent: if the file already exists, leave it alone. Multi-instance
+    # workloads (e.g. feedsim run-feedsim-multi.sh) have several processes
+    # racing to call create_breakdown_csv concurrently — truncating after the
+    # first writer would silently drop entries already logged by the earlier
+    # instance. benchpress's copymove hook (is_move: true) clears the file
+    # between iterations, so stale data is not a risk.
+    if [ -f "$csv_file" ]; then
+        return 0
+    fi
+
     # Create CSV file with headers
     if echo "operation_name,PID,timestamp_type,timestamp,sub_operation_name" > "$csv_file"; then
         echo "Created breakdown CSV file: $csv_file"

packages/feedsim/feed_aggregator_req_sizes.json

@@ -0,0 +1 @@
+[{"req_size_min":9,"req_size_p05":55,"req_size_p10":56,"req_size_p15":87,"req_size_p20":353,"req_size_p25":354,"req_size_p30":361,"req_size_p35":368,"req_size_p40":35168,"req_size_p45":42544,"req_size_p50":47874,"req_size_p55":55587,"req_size_p60":388656,"req_size_p65":1084512,"req_size_p70":1522044,"req_size_p75":1921331,"req_size_p80":2239067,"req_size_p85":2584693,"req_size_p90":3025469,"req_size_p95":3679075,"req_size_p99":4075462,"req_size_max":18882521}]

packages/feedsim/feed_aggregator_resp_sizes.json

@@ -0,0 +1 @@
+[{"resp_size_min":4,"resp_size_p05":4,"resp_size_p10":4,"resp_size_p15":4,"resp_size_p20":4,"resp_size_p25":4,"resp_size_p30":4,"resp_size_p35":4,"resp_size_p40":4,"resp_size_p45":42,"resp_size_p50":44,"resp_size_p55":44,"resp_size_p60":46,"resp_size_p65":30473,"resp_size_p70":128474,"resp_size_p75":148553,"resp_size_p80":217715,"resp_size_p85":873710,"resp_size_p90":1303070,"resp_size_p95":2100241,"resp_size_p99":2594795,"resp_size_max":10319524}]