perf.yml

name: Perf Benchmark

on:
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      concurrency:
        description: "aiperf --concurrency"
        default: "10"
      request_count:
        description: "aiperf --request-count"
        default: "200"

# Cancel superseded PR runs; keep main builds.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

jobs:
  proxy-perf:
    name: Proxy overhead (local stub backend)
    runs-on: ubuntu-latest

    env:
      PROXY_PORT: 4000
      STUB_PORT: 9000
      # A local zero-latency stub replaces the removed noop route; the proxy
      # forwards to it over loopback, which adds a small constant overhead.
      PERF_MODEL: mock-model
      # gpt2 tokenizer is ~500 KB and downloads in < 2 s on Actions runners.
      # It is only used for dataset-manager token counting; actual inference
      # token counts come from the server (--use-server-token-count).
      PERF_TOKENIZER: gpt2
      PERF_CONCURRENCY: ${{ inputs.concurrency || '10' }}
      PERF_REQUEST_COUNT: ${{ inputs.request_count || '200' }}

    steps:
      - uses: actions/checkout@v4

      - uses: astral-sh/setup-uv@v6
        with:
          python-version: "3.12"
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - name: Install switchyard (default extras only)
        run: uv sync

      - name: Install aiperf
        run: uv pip install aiperf

      # ------------------------------------------------------------------
      # Start a local zero-latency mock OpenAI upstream. This replaces the
      # removed noop route: the proxy serves a real `type: model` chain that
      # forwards to this loopback stub, which returns a fixed completion
      # instantly. The extra loopback hop adds a small constant overhead.
      # ------------------------------------------------------------------
      - name: Start local mock OpenAI upstream
        run: |
          cat > perf_stub.py <<'PY'
          """Zero-latency OpenAI-compatible chat.completions stub (loopback only)."""
          import json
          import os
          from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

          _COMPLETION = {
              "id": "chatcmpl-perf-stub",
              "object": "chat.completion",
              "created": 1700000000,
              "model": "mock-model",
              "choices": [
                  {
                      "index": 0,
                      "message": {"role": "assistant", "content": "4"},
                      "finish_reason": "stop",
                  }
              ],
              "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
          }

          _CHUNK = {
              "id": "chatcmpl-perf-stub",
              "object": "chat.completion.chunk",
              "created": 1700000000,
              "model": "mock-model",
              "choices": [
                  {"index": 0, "delta": {"content": "4"}, "finish_reason": "stop"}
              ],
              "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
          }


          class Handler(BaseHTTPRequestHandler):
              protocol_version = "HTTP/1.1"

              def do_POST(self):
                  length = int(self.headers.get("content-length", "0"))
                  body = json.loads(self.rfile.read(length) or b"{}")
                  if body.get("stream"):
                      payload = (
                          f"data: {json.dumps(_CHUNK)}\n\n".encode()
                          + b"data: [DONE]\n\n"
                      )
                      content_type = "text/event-stream"
                  else:
                      payload = json.dumps(_COMPLETION).encode()
                      content_type = "application/json"
                  self.send_response(200)
                  self.send_header("content-type", content_type)
                  self.send_header("content-length", str(len(payload)))
                  self.end_headers()
                  self.wfile.write(payload)

              def log_message(self, *_args):
                  return None


          port = int(os.environ["STUB_PORT"])
          ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()
          PY
          python3 perf_stub.py &
          echo "STUB_PID=$!" >> "$GITHUB_ENV"

      # ------------------------------------------------------------------
      # Start the proxy: a `type: model` route pointed at the local stub.
      # ------------------------------------------------------------------
      - name: Start switchyard proxy
        run: |
          cat > bench.yaml <<YAML
          defaults:
            api_key: dummy
            base_url: http://localhost:$STUB_PORT/v1
            format: openai
          routes:
            mock-model:
              type: model
              model: mock-model
          YAML
          uv run switchyard --routing-profiles bench.yaml -- serve --port $PROXY_PORT &
          echo "PROXY_PID=$!" >> "$GITHUB_ENV"

      - name: Wait for proxy to be ready
        run: |
          for i in $(seq 1 30); do
            if curl -sf http://localhost:$PROXY_PORT/health > /dev/null 2>&1; then
              echo "Proxy is up after ${i}s"
              exit 0
            fi
            sleep 1
          done
          echo "Proxy did not become ready in 30s"
          exit 1

      # ------------------------------------------------------------------
      # Pre-build a minimal input dataset so aiperf doesn't need to
      # download a HuggingFace tokenizer for synthetic data generation.
      # Combined with --use-server-token-count this makes the benchmark
      # completely self-contained (no network calls beyond the proxy).
      # ------------------------------------------------------------------
      - name: Generate perf input dataset
        run: |
          python3 - <<'PY'
          import json
          # 300 entries — enough to cycle through for any request-count
          with open("perf-input.jsonl", "w") as f:
              for _ in range(300):
                  f.write(json.dumps({"text": "What is 2 + 2?"}) + "\n")
          PY

      # ------------------------------------------------------------------
      # Run the benchmark
      # ------------------------------------------------------------------
      - name: Run aiperf benchmark (non-streaming)
        run: |
          mkdir -p perf-results
          uv run aiperf profile \
            --model "$PERF_MODEL" \
            --tokenizer "$PERF_TOKENIZER" \
            --url "http://localhost:$PROXY_PORT" \
            --endpoint-type chat \
            --concurrency "$PERF_CONCURRENCY" \
            --request-count "$PERF_REQUEST_COUNT" \
            --ui none \
            --custom-dataset-type single-turn \
            --input-file perf-input.jsonl \
            --use-server-token-count \
            --output-artifact-dir perf-results/non-streaming

      - name: Run aiperf benchmark (streaming)
        run: |
          uv run aiperf profile \
            --model "$PERF_MODEL" \
            --tokenizer "$PERF_TOKENIZER" \
            --url "http://localhost:$PROXY_PORT" \
            --endpoint-type chat \
            --streaming \
            --concurrency "$PERF_CONCURRENCY" \
            --request-count "$PERF_REQUEST_COUNT" \
            --ui none \
            --custom-dataset-type single-turn \
            --input-file perf-input.jsonl \
            --use-server-token-count \
            --output-artifact-dir perf-results/streaming

      # ------------------------------------------------------------------
      # Stop proxy and local stub
      # ------------------------------------------------------------------
      - name: Stop proxy and stub
        if: always()
        run: |
          kill "$PROXY_PID" || true
          kill "$STUB_PID" || true

      # ------------------------------------------------------------------
      # Upload results as a build artifact for comparison over time
      # ------------------------------------------------------------------
      - name: Upload perf results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: perf-results-${{ github.sha }}
          path: perf-results/
          retention-days: 90