config.example.yaml

# Copyright 2026 FlagOS Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Agent Benchmark Configuration

# Agent configuration
agent:
  type: claude           # claude / cursor / custom
  bin: claude            # path to executable
  timeout: 1800          # per-operator timeout (seconds)
  max_retries: 3         # max retries on failure
  budget: 50.0           # per-operator budget (USD, claude only)

# Device configuration
# Device type is auto-detected (CUDA, NPU, MUSA, Iluvatar, Hygon DCU, MetaX).
# Override with GEMS_VENDOR env var: nvidia, ascend, mthreads, iluvatar, hygon, muxi
device:
  gpu_ids: null          # null=auto-detect, or [0,1,2,3]
  lock_dir: /tmp/agent_bench_gpu_locks

# Path configuration (relative to agent_bench/ directory)
paths:
  prompts: prompts
  runs: runs
  templates: templates
  python: python         # Python interpreter path - set to your conda env, e.g. /path/to/conda/envs/myenv/bin/python

# Dataset to test module mapping (relative to project root)
# Sub-datasets (KernelGenBench-aten, etc.) are useful for non-NVIDIA chips that
# don't support cuBLAS operators.
test_modules:
  KernelGenBench:
    - src/kernelgenbench/accuracy/test_ops_with_benchmark.py
    - src/kernelgenbench/accuracy/vllm13/
    - src/kernelgenbench/accuracy/cublas/
  KernelGenBench-aten:
    - src/kernelgenbench/accuracy/test_ops_with_benchmark.py
  KernelGenBench-vllm:
    - src/kernelgenbench/accuracy/vllm13/
  KernelGenBench-cublas:
    - src/kernelgenbench/accuracy/cublas/
  KernelGenBench-nocublas:
    - src/kernelgenbench/accuracy/test_ops_with_benchmark.py
    - src/kernelgenbench/accuracy/vllm13/

# Poll interval (seconds)
poll_interval: 10