perf(agentic): add GB200 DEP8 prefill candidate

Author: cquil11

.github/configs/nvidia-master.yaml

@@ -12830,6 +12830,36 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-1p2d:
           ep: 1
           dp-attn: false
 
+dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8:
+  image: vllm/vllm-openai:v0.23.0
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: gb200
+  precision: fp4
+  framework: dynamo-vllm
+  multinode: true
+  disagg: true
+  scenarios:
+    agentic-coding:
+    - duration: 1800
+      search-space:
+      - spec-decoding: none
+        conc-list: [64, 96, 128, 160]
+        prefill:
+          num-worker: 1
+          # Matrix TP records physical GPUs per worker for normalization;
+          # the recipe implements them as attention DP8 / expert EP8.
+          tp: 8
+          ep: 8
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-tp8-agentic.yaml"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+
 dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d:
   image: vllm/vllm-openai:v0.23.0
   model: deepseek-ai/DeepSeek-V4-Pro

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/GB200_VLLM_AGENTIC_SWEEP_NOTES.md

@@ -753,3 +753,24 @@ while the first production 2P/1D batch is the end-to-end validation.
 The B200 database baseline was re-queried before the high-throughput launch.
 Workflow `2022` c196 with offload remains the normalized target at 113,671.86
 tok/s total and 14,208.98 tok/s/GPU (15.526s mean TTFT, 279.09ms mean TPOT).
+
+## High-Concurrency Server-Reuse Run
+
+Official run `27922709353` uses one 2P/1D allocation for c128, c160, c192,
+and c256. The first point completed successfully but established that c128 is
+already deep overload for this topology: 59,345.29 tok/s total, 2,472.72
+tok/s/GPU, 337.97s mean TTFT, and 21.82% aggregate server GPU cache hit rate.
+The theoretical trace reuse remained 96.65%. Decode generally had only 6--28
+running requests and roughly 15--30% KV usage; both prefills stayed near 20%
+KV usage. There were no NIXL errors, and all decode ranks passed compatibility
+checks. This rules out decode saturation and KV-capacity thrashing at c128;
+the fixed-duration run admitted too many distinct long trajectories to reach
+the high-reuse steady state.
+
+Added a separate 1P/1D DEP8-prefill experiment using 16 inference GPUs. It
+keeps Dynamo KV routing, prefix caching, the 32K retention interval, KV event
+publication, and the validated TP8 decode path. Its prefill follows the repo's
+existing vLLM DEP pattern (`TP1 x DP8`, EP8, `deep_gemm_mega_moe`) and raises
+the prefill batch-token ceiling from 16K to 32K. This tests whether eight-way
+attention parallelism can improve raw prefill throughput enough to offset the
+expected per-rank load-balance and cache-affinity penalty.

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-tp8-agentic.yaml

@@ -0,0 +1,147 @@
+name: "svf-vllm-disagg-gb200-1p1d-dep8-tp8-agentic"
+
+# High-throughput experiment: one DEP8 prefill worker and one TP8 decode
+# worker. DEP exposes eight independent attention ranks behind one vLLM worker
+# while EP8 retains full-system MoE execution. This trades cache-affinity and
+# rank-balance simplicity for higher raw prefill parallelism.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:v0.23.0"
+  precision: "fp4"
+
+dynamo:
+  install: true
+  wheel: "1.3.0.dev20260618"
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb200"
+  gpus_per_node: 4
+  prefill_nodes: 2
+  decode_nodes: 2
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_prefill: 8
+  gpus_per_decode: 8
+
+infra:
+  etcd_nats_dedicated_node: true
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  env:
+    DYN_REQUEST_PLANE: "tcp"
+  args:
+    router-mode: "kv"
+    router-reset-states: true
+
+backend:
+  type: vllm
+  connector: null
+  prefill_environment:
+    DYN_REQUEST_PLANE: "tcp"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    TORCH_SYMMMEM: "NVSHMEM"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
+    VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_RCACHE_MAX_UNRELEASED: "1024"
+    UCX_TLS: "cuda_copy,rc"
+    NCCL_P2P_LEVEL: NVL
+  decode_environment:
+    DYN_REQUEST_PLANE: "tcp"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    TORCH_SYMMMEM: "NVSHMEM"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_RCACHE_MAX_UNRELEASED: "1024"
+    UCX_TLS: "cuda_copy,rc"
+    NCCL_P2P_LEVEL: NVL
+
+  vllm_config:
+    prefill:
+      kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
+      kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 8
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-ep-weight-filter: true
+      attention-config: '{"use_fp4_indexer_cache": true}'
+      moe-backend: "deep_gemm_mega_moe"
+      enforce-eager: true
+      max-num-seqs: 256
+      max-num-batched-tokens: 32768
+      trust-remote-code: true
+      no-enable-flashinfer-autotune: true
+      no-async-scheduling: true
+      block-size: 256
+      gpu-memory-utilization: 0.9
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+    decode:
+      kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 8
+      pipeline-parallel-size: 1
+      attention-config: '{"use_fp4_indexer_cache": true}'
+      max-num-seqs: 512
+      max-cudagraph-capture-size: 512
+      max-num-batched-tokens: 512
+      trust-remote-code: true
+      no-enable-flashinfer-autotune: true
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+      gpu-memory-utilization: 0.9
+      stream-interval: 50
+      all2all-backend: "flashinfer_nvlink_one_sided"
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+
+sbatch_directives:
+  cpus-per-task: "72"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  aiperf_server_metrics: true
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_061526"