Match prod multifeed RPC profile: ZSTD, smaller response, DLRM=1, feature extractors (#744)

benchpress/config/jobs.yml

@@ -646,6 +646,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors={feature_extractors}'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
   vars:
     - 'port=11222'
     - 'output=feedsim_results.txt'
@@ -655,13 +659,17 @@
     - 'dlrm_model=models/dlrm_small.pt'
     - 'dlrm_batch_size=256'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    - 'dlrm_inferences=1'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_extractors=0'
+    - 'feature_complexity=5'
+    - 'num_stories=100'
+    - 'extractors_per_story=50'
   hooks:
     - hook: cpu-mpstat
       options:
@@ -731,22 +739,41 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
+    - '--silesia-dir={silesia_dir}'
+    - '--stories-per-request={stories_per_request}'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
     - 'io_dist=fixed'
     - 'io_mean=200'
     - 'workload=dlrm'
     - 'dlrm_model=models/dlrm_small.pt'
-    - 'dlrm_batch_size=256'
+    # 64 matches t29 baseline (was 256, which is 4x oversized vs t29). Combined
+    # with dlrm_inferences=1 the per-request DLRM CPU drops further toward
+    # prod multifeed's Ranking-Prediction share (~6%).
+    - 'dlrm_batch_size=64'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    - 'dlrm_inferences=1'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_complexity=5'
+    # 400 x 400 = 160K extractor calls/req. Matches the t29 BGM/CPL/GRC
+    # search_qps validation that converged cleanly at 700ms p95 SLA. The
+    # earlier 1800x50 tuning came from feedsim_autoscale_dlrm_mini (fixed
+    # QPS, no search) and produces queueing collapse under search_qps.
+    # 5K calls/req (100x50) is too few to surface FeatureExtraction CPU.
+    - 'num_stories=400'
+    - 'extractors_per_story=400'
+    - 'silesia_dir=silesia'
+    - 'stories_per_request=10'
     - 'extra_args='
   hooks:
     - hook: cpu-mpstat
@@ -761,6 +788,7 @@
           - 'benchmarks/feedsim/feedsim_results*.txt'
           - 'benchmarks/feedsim/feedsim-multi-inst-*.log'
           - 'benchmarks/feedsim/src/perf.data'
+          - 'benchmarks/feedsim/breakdown.csv'
           - '/tmp/feedsim_log.txt'
 
 - name: feedsim_autoscale_dlrm_mini
@@ -785,6 +813,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
     - '-q {fixed_qps}'
     - '-d {fixed_qps_duration}'
     - '-w {warmup_time}'
@@ -804,13 +836,21 @@
     - 'dlrm_model=models/dlrm_small.pt'
     - 'dlrm_batch_size=256'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    # 8 inferences/req brings DLRM-Inference CPU share from 55-62% down
+    # toward prod multifeed_aggregator's Ranking-Prediction share of 7-13%.
+    - 'dlrm_inferences=8'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_complexity=5'
+    # 1800 stories x 50 extractors = 90K extractor calls/req. Tuned to
+    # push FeatureExtraction CPU share into prod's 30-35% band; 5K
+    # calls/req previously gave only 1.7-1.9%.
+    - 'num_stories=1800'
+    - 'extractors_per_story=50'
     - 'fixed_qps=100000'
     - 'fixed_qps_duration=10'
     - 'warmup_time=5'

packages/common/runtime_breakdown_utils.sh

@@ -25,6 +25,16 @@ create_breakdown_csv() {
 
     local csv_file="${folder_path}/${breakdown_file_name}"
 
+    # Idempotent: if the file already exists, leave it alone. Multi-instance
+    # workloads (e.g. feedsim run-feedsim-multi.sh) have several processes
+    # racing to call create_breakdown_csv concurrently — truncating after the
+    # first writer would silently drop entries already logged by the earlier
+    # instance. benchpress's copymove hook (is_move: true) clears the file
+    # between iterations, so stale data is not a risk.
+    if [ -f "$csv_file" ]; then
+        return 0
+    fi
+
     # Create CSV file with headers
     if echo "operation_name,PID,timestamp_type,timestamp,sub_operation_name" > "$csv_file"; then
         echo "Created breakdown CSV file: $csv_file"

packages/feedsim/feed_aggregator_req_sizes.json

@@ -0,0 +1 @@
+[{"req_size_min":9,"req_size_p05":55,"req_size_p10":56,"req_size_p15":87,"req_size_p20":353,"req_size_p25":354,"req_size_p30":361,"req_size_p35":368,"req_size_p40":35168,"req_size_p45":42544,"req_size_p50":47874,"req_size_p55":55587,"req_size_p60":388656,"req_size_p65":1084512,"req_size_p70":1522044,"req_size_p75":1921331,"req_size_p80":2239067,"req_size_p85":2584693,"req_size_p90":3025469,"req_size_p95":3679075,"req_size_p99":4075462,"req_size_max":18882521}]

packages/feedsim/feed_aggregator_resp_sizes.json

@@ -0,0 +1 @@
+[{"resp_size_min":4,"resp_size_p05":4,"resp_size_p10":4,"resp_size_p15":4,"resp_size_p20":4,"resp_size_p25":4,"resp_size_p30":4,"resp_size_p35":4,"resp_size_p40":4,"resp_size_p45":42,"resp_size_p50":44,"resp_size_p55":44,"resp_size_p60":46,"resp_size_p65":30473,"resp_size_p70":128474,"resp_size_p75":148553,"resp_size_p80":217715,"resp_size_p85":873710,"resp_size_p90":1303070,"resp_size_p95":2100241,"resp_size_p99":2594795,"resp_size_max":10319524}]

packages/feedsim/generate_dlrm_models.py

@@ -0,0 +1,171 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Generate DLRM medium and large TorchScript models with random weights.
+
+For benchmarking, trained weights aren't needed — the compute pattern
+(embedding lookups, MLP forward passes, feature interactions) is identical
+regardless of weight values. These models are architecture-compatible with
+the existing dlrm_small.pt model used by FeedSim's DLRM inference path.
+
+Usage:
+    python3 generate_dlrm_models.py <output_dir>
+"""
+
+import os
+import sys
+
+import torch
+from torch import nn
+
+
+class DLRM(nn.Module):
+    """Simplified DLRM model matching production inference patterns."""
+
+    def __init__(
+        self,
+        emb_dim: int = 64,
+        num_dense: int = 13,
+        num_sparse: int = 26,
+        max_emb_rows: int = 250000,
+        bottom_mlp_dims: list = None,
+        top_mlp_dims: list = None,
+    ):
+        super().__init__()
+        if bottom_mlp_dims is None:
+            bottom_mlp_dims = [256, 128, emb_dim]
+        if top_mlp_dims is None:
+            top_mlp_dims = [256, 128, 1]
+
+        # Bottom MLP: dense features -> embedding dimension
+        layers = []
+        in_dim = num_dense
+        for out_dim in bottom_mlp_dims:
+            layers.append(nn.Linear(in_dim, out_dim))
+            layers.append(nn.ReLU())
+            in_dim = out_dim
+        self.bottom_mlp = nn.Sequential(*layers)
+
+        # Embedding tables for sparse features
+        emb_sizes = [
+            40000000,
+            39060,
+            17295,
+            7424,
+            20265,
+            3,
+            7122,
+            1543,
+            63,
+            40000000,
+            3067956,
+            405282,
+            10,
+            2209,
+            11938,
+            155,
+            4,
+            976,
+            14,
+            40000000,
+            40000000,
+            40000000,
+            590152,
+            12973,
+            108,
+            36,
+        ]
+        # Use min(actual_size, max_emb_rows) to control model size
+        self.embeddings = nn.ModuleList(
+            [
+                nn.EmbeddingBag(min(s, max_emb_rows), emb_dim, mode="sum")
+                for s in emb_sizes[:num_sparse]
+            ]
+        )
+
+        # Top MLP: interaction output -> prediction. ReLU only on hidden
+        # layers; the final Linear feeds raw logits into sigmoid in forward().
+        n = 1 + num_sparse  # bottom_mlp output + embedding outputs
+        interaction_size = emb_dim + (n * (n - 1)) // 2
+        top_layers = []
+        in_dim = interaction_size
+        for i, out_dim in enumerate(top_mlp_dims):
+            top_layers.append(nn.Linear(in_dim, out_dim))
+            if i < len(top_mlp_dims) - 1:
+                top_layers.append(nn.ReLU())
+            in_dim = out_dim
+        self.top_mlp = nn.Sequential(*top_layers)
+
+    def forward(self, dense: torch.Tensor, sparse: torch.Tensor) -> torch.Tensor:
+        # Bottom MLP
+        d = self.bottom_mlp(dense)
+
+        # Embedding lookups
+        embs = [emb(sparse[:, i].unsqueeze(1)) for i, emb in enumerate(self.embeddings)]
+
+        # Feature interaction (dot product)
+        combined = torch.cat([d.unsqueeze(1)] + [e.unsqueeze(1) for e in embs], dim=1)
+        interact = torch.bmm(combined, combined.transpose(1, 2))
+        n = combined.size(1)
+        idx = torch.triu_indices(n, n, offset=1)
+        flat = interact[:, idx[0], idx[1]]
+
+        # Top MLP
+        x = torch.cat([d, flat], dim=1)
+        return torch.sigmoid(self.top_mlp(x))
+
+
+def generate_model(output_path: str, **kwargs):
+    """Generate and save a TorchScript DLRM model."""
+    model = DLRM(**kwargs)
+    param_bytes = sum(p.numel() * p.element_size() for p in model.parameters())
+    print(f"  Parameters: {sum(p.numel() for p in model.parameters()):,}")
+    print(f"  Model size: {param_bytes / 1e6:.0f} MB")
+
+    scripted = torch.jit.script(model)
+    scripted.save(output_path)
+    file_size = os.path.getsize(output_path)
+    print(f"  Saved to: {output_path} ({file_size / 1e6:.0f} MB on disk)")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} <output_dir>")
+        sys.exit(1)
+
+    output_dir = sys.argv[1]
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Medium model: larger embeddings (~500MB)
+    medium_path = os.path.join(output_dir, "dlrm_medium.pt")
+    if not os.path.exists(medium_path):
+        print("Generating DLRM medium model...")
+        generate_model(
+            medium_path,
+            emb_dim=64,
+            max_emb_rows=500000,
+            bottom_mlp_dims=[256, 128, 64],
+            top_mlp_dims=[256, 128, 1],
+        )
+    else:
+        print(f"[SKIPPED] {medium_path} already exists")
+
+    # Large model: even larger embeddings (~1GB)
+    large_path = os.path.join(output_dir, "dlrm_large.pt")
+    if not os.path.exists(large_path):
+        print("Generating DLRM large model...")
+        generate_model(
+            large_path,
+            emb_dim=128,
+            max_emb_rows=500000,
+            bottom_mlp_dims=[512, 256, 128],
+            top_mlp_dims=[512, 256, 1],
+        )
+    else:
+        print(f"[SKIPPED] {large_path} already exists")
+
+
+if __name__ == "__main__":
+    main()