Make prod-aligned knobs the defaults (#712)

Summary:
Pull Request resolved: #712

Bake in four knobs that were previously env-overrides during the t29-t34 calibration sweeps. These are the configurations that bring the FeedSim profile closest to prod multifeed_aggregator on CPL/BGM/Grace (validated in t34, see T269255604):

- `--rpc_fanout_scale` default 0.025 → 0.10 (LeafNodeRankCmdline.ggo)
  - ~94 → ~376 outbound RPCs/session
  - Raises Encryption / RPC-Compression / RPC-AsyncIO shares (which all flow through the TLS+ZSTD mock_services channel) toward prod's 3.3% / 2.9% / 14.4% on BGM
- `FEEDSIM_SLA_P95_MS` default 500 → 700 (run.sh)
  - Matches prod multifeed_aggregator's own end-to-end p95 budget
- `FEEDSIM_TLS` default 0 → 1 (run.sh, run-feedsim-multi.sh)
  - TLS on mock_services channel by default (Rocket-over-TLS, ALPN "rs")
- `MOCK_KEEPALIVE_INTERVAL_MS` default unset → 100 (run.sh)
  - 100 ms getStatus() pings on every channel to defeat the cold-channel
    anti-pattern (t25 showed BGM 14x p95 cliff at q=5 without keepalive)

All four remain env-overridable for experiments (set to 0 / different value to opt out).

Reviewed By: YifanYuan3

Differential Revision: D107579495

Author: excelle08

packages/feedsim/run-feedsim-multi.sh

@@ -164,11 +164,12 @@ function start_mock_services() {
     local offset_us="${MOCK_LATENCY_OFFSET_US:-0}"
     local skip_us="${MOCK_LATENCY_SKIP_THRESHOLD_US:-100}"
 
-    # TLS opt-in: FEEDSIM_TLS=1 enables TLS on both server and client.
-    # Server reads --tls_cert / --tls_key; client picks up MOCK_TLS env var
-    # (LeafNodeRank uses gengetopt and rejects unknown CLI flags).
+    # TLS on by default (matches prod's Rocket-over-TLS); set FEEDSIM_TLS=0
+    # to disable. Server reads --tls_cert / --tls_key; client picks up
+    # MOCK_TLS env var set by run.sh (LeafNodeRank uses gengetopt and rejects
+    # unknown CLI flags).
     local tls_opts=""
-    if [ "${FEEDSIM_TLS:-0}" = "1" ]; then
+    if [ "${FEEDSIM_TLS:-1}" = "1" ]; then
         local cert_dir="${FEEDSIM_ROOT}/certs"
         if [ ! -r "${cert_dir}/example.crt" ] || [ ! -r "${cert_dir}/example.key" ]; then
             echo "ERROR: FEEDSIM_TLS=1 but ${cert_dir}/example.{crt,key} not found" >&2
@@ -177,7 +178,7 @@ function start_mock_services() {
         tls_opts="--tls_cert=${cert_dir}/example.crt --tls_key=${cert_dir}/example.key"
     fi
 
-    echo "Starting mock_services on port ${port} (cores=${core_range}, io_threads=${io_threads}, cap_us=${cap_us}, offset_us=${offset_us}, skip_us=${skip_us}, tls=${FEEDSIM_TLS:-0}, silesia=${SILESIA_DIR_ABS})"
+    echo "Starting mock_services on port ${port} (cores=${core_range}, io_threads=${io_threads}, cap_us=${cap_us}, offset_us=${offset_us}, skip_us=${skip_us}, tls=${FEEDSIM_TLS:-1}, silesia=${SILESIA_DIR_ABS})"
     # shellcheck disable=SC2086
     taskset --cpu-list "$core_range" \
         "$MOCK_SERVICES_BIN" \

packages/feedsim/run.sh

@@ -735,8 +735,8 @@ main() {
     local mock_port="${MOCK_SERVICES_PORT:-21222}"
     local mock_services_opts="--rpc_dist_path=$rpc_dist_json --mock_services_host=localhost --mock_services_port=${mock_port}"
     # Optional fanout-scale override (defaults to LeafNodeRank's
-    # --rpc_fanout_scale=0.025 when the env var is unset). Lets sweep
-    # scripts A/B test heavier outbound load without rebuilding.
+    # --rpc_fanout_scale=0.10 when the env var is unset). Lets sweep
+    # scripts A/B test different outbound load levels without rebuilding.
     if [ -n "${RPC_FANOUT_SCALE:-}" ]; then
         mock_services_opts="$mock_services_opts --rpc_fanout_scale=${RPC_FANOUT_SCALE}"
         echo "RPC fanout scale override: $RPC_FANOUT_SCALE"
@@ -752,23 +752,24 @@ main() {
         mock_services_opts="$mock_services_opts --use_legacy_sleep"
         echo "RPC fanout: forced OFF via LEAFNODE_USE_LEGACY_SLEEP=1 (legacy sleep path)"
     fi
-    # t25 mitigation knob: per-MockServicesClient keepalive ping. When
-    # MOCK_KEEPALIVE_INTERVAL_MS is set and > 0, each channel issues a
-    # 1-byte getStatus() probe every N ms to defeat the cold-channel
-    # anti-pattern observed at low QPS (BGM saw 14x p95 cliff at q=5).
-    # Recommended starting value: 150-500 ms. 0 / unset = disabled
-    # (anti-pattern stays observable).
-    if [ -n "${MOCK_KEEPALIVE_INTERVAL_MS:-}" ] && [ "${MOCK_KEEPALIVE_INTERVAL_MS}" != "0" ]; then
-        mock_services_opts="$mock_services_opts --mock_keepalive_interval_ms=${MOCK_KEEPALIVE_INTERVAL_MS}"
-        echo "MockServicesClient keepalive: ENABLED (interval=${MOCK_KEEPALIVE_INTERVAL_MS} ms)"
+    # Per-MockServicesClient keepalive ping. Each channel issues a tiny
+    # getStatus() probe every N ms to defeat the cold-channel anti-pattern
+    # observed at low QPS (BGM saw 14x p95 cliff at q=5 without keepalive).
+    # Default 100 ms. Set MOCK_KEEPALIVE_INTERVAL_MS=0 to disable.
+    mock_keepalive_ms="${MOCK_KEEPALIVE_INTERVAL_MS:-100}"
+    if [ "${mock_keepalive_ms}" != "0" ]; then
+        mock_services_opts="$mock_services_opts --mock_keepalive_interval_ms=${mock_keepalive_ms}"
+        echo "MockServicesClient keepalive: ENABLED (interval=${mock_keepalive_ms} ms)"
     fi
     # TLS + wire compression on the outbound MockServicesClient channel.
     # LeafNodeRank uses gengetopt (rejects unknown CLI flags), so these knobs
     # are plumbed via env vars MOCK_TLS / MOCK_COMPRESS_ZSTD read inside
-    # MockServicesClient.cc. FEEDSIM_TLS=1 must match the server-side
-    # --tls_cert/--tls_key wiring in run-feedsim-multi.sh. FEEDSIM_NO_RPC_ZSTD=1
-    # disables ZSTD (binary default is on); leave unset for prod-parity.
-    if [ "${FEEDSIM_TLS:-0}" = "1" ]; then
+    # MockServicesClient.cc. FEEDSIM_TLS defaults to 1 (matches prod's
+    # Rocket-over-TLS); set FEEDSIM_TLS=0 to disable. Server-side
+    # --tls_cert/--tls_key wiring is in run-feedsim-multi.sh.
+    # FEEDSIM_NO_RPC_ZSTD=1 disables ZSTD (binary default is on); leave
+    # unset for prod-parity.
+    if [ "${FEEDSIM_TLS:-1}" = "1" ]; then
         export MOCK_TLS=1
         echo "MockServicesClient TLS: ENABLED (via MOCK_TLS env)"
     fi
@@ -889,11 +890,10 @@ main() {
     # Preprocessing complete; search_qps.sh will own the main_benchmark phase.
     log_preprocessing_end "$BREAKDOWN_FOLDER" "$$"
 
-    # SLA target for search_qps (95p latency in milliseconds). Default 500ms.
-    # Override via FEEDSIM_SLA_P95_MS env var. Prior experiments (t29, t30)
-    # ran at 700ms to give the system more headroom past the prod-aggregator's
-    # own end-to-end budget.
-    sla_p95_ms="${FEEDSIM_SLA_P95_MS:-500}"
+    # SLA target for search_qps (95p latency in milliseconds). Default 700ms
+    # matches the prod multifeed aggregator's own end-to-end budget at p95.
+    # Override via FEEDSIM_SLA_P95_MS env var.
+    sla_p95_ms="${FEEDSIM_SLA_P95_MS:-700}"
     sla_arg="95p:${sla_p95_ms}"
 
     if [ -z "$fixed_qps" ] && [ "$auto_driver_threads" != "1" ]; then

packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo

@@ -91,7 +91,7 @@ option "silesia_dir" - "Path to Silesia corpus directory. When set, server-side
 option "rpc_dist_path" - "Path to rpc_dist.json. When set, request handlers fan out real RPCs to mock_services." string typestr="filename" default=""
 option "mock_services_host" - "Hostname for mock_services Thrift server." string default="127.0.0.1"
 option "mock_services_port" - "Port for mock_services Thrift server (matches mock_services --port)." int default="21222"
-option "rpc_fanout_scale" - "Scale factor applied to per-session fanout counts. Default 0.025 yields ~94 RPCs/session (vs ~3742 at scale=1.0)." double default="0.025"
+option "rpc_fanout_scale" - "Scale factor applied to per-session fanout counts. Default 0.10 yields ~376 RPCs/session (vs ~3742 at scale=1.0); calibrated to raise outbound RPC volume so Encryption/RPC-Compression/RPC-AsyncIO shares match prod's multifeed_aggregator profile." double default="0.10"
 # Diagnostic / isolation knob. When set, request handlers take the legacy
 # folly::futures::sleep path even if --rpc_dist_path is supplied, and the
 # per-thread MockServicesClient is not constructed (no connection to