fix(server): render Anthropic error envelopes #1229
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Perf Benchmark | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| concurrency: | |
| description: "aiperf --concurrency" | |
| default: "10" | |
| request_count: | |
| description: "aiperf --request-count" | |
| default: "200" | |
| # Cancel superseded PR runs; keep main builds. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| proxy-perf: | |
| name: Proxy overhead (local stub backend) | |
| runs-on: ubuntu-latest | |
| env: | |
| PROXY_PORT: 4000 | |
| STUB_PORT: 9000 | |
| # A local zero-latency stub replaces the removed noop route; the proxy | |
| # forwards to it over loopback, which adds a small constant overhead. | |
| PERF_MODEL: mock-model | |
| # gpt2 tokenizer is ~500 KB and downloads in < 2 s on Actions runners. | |
| # It is only used for dataset-manager token counting; actual inference | |
| # token counts come from the server (--use-server-token-count). | |
| PERF_TOKENIZER: gpt2 | |
| PERF_CONCURRENCY: ${{ inputs.concurrency || '10' }} | |
| PERF_REQUEST_COUNT: ${{ inputs.request_count || '200' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Install switchyard (default extras only) | |
| run: uv sync | |
| - name: Install aiperf | |
| run: uv pip install aiperf | |
| # ------------------------------------------------------------------ | |
| # Start a local zero-latency mock OpenAI upstream. This replaces the | |
| # removed noop route: the proxy serves a real `type: model` chain that | |
| # forwards to this loopback stub, which returns a fixed completion | |
| # instantly. The extra loopback hop adds a small constant overhead. | |
| # ------------------------------------------------------------------ | |
| - name: Start local mock OpenAI upstream | |
| run: | | |
| cat > perf_stub.py <<'PY' | |
| """Zero-latency OpenAI-compatible chat.completions stub (loopback only).""" | |
| import json | |
| import os | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| _COMPLETION = { | |
| "id": "chatcmpl-perf-stub", | |
| "object": "chat.completion", | |
| "created": 1700000000, | |
| "model": "mock-model", | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "message": {"role": "assistant", "content": "4"}, | |
| "finish_reason": "stop", | |
| } | |
| ], | |
| "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, | |
| } | |
| _CHUNK = { | |
| "id": "chatcmpl-perf-stub", | |
| "object": "chat.completion.chunk", | |
| "created": 1700000000, | |
| "model": "mock-model", | |
| "choices": [ | |
| {"index": 0, "delta": {"content": "4"}, "finish_reason": "stop"} | |
| ], | |
| "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, | |
| } | |
| class Handler(BaseHTTPRequestHandler): | |
| protocol_version = "HTTP/1.1" | |
| def do_POST(self): | |
| length = int(self.headers.get("content-length", "0")) | |
| body = json.loads(self.rfile.read(length) or b"{}") | |
| if body.get("stream"): | |
| payload = ( | |
| f"data: {json.dumps(_CHUNK)}\n\n".encode() | |
| + b"data: [DONE]\n\n" | |
| ) | |
| content_type = "text/event-stream" | |
| else: | |
| payload = json.dumps(_COMPLETION).encode() | |
| content_type = "application/json" | |
| self.send_response(200) | |
| self.send_header("content-type", content_type) | |
| self.send_header("content-length", str(len(payload))) | |
| self.end_headers() | |
| self.wfile.write(payload) | |
| def log_message(self, *_args): | |
| return None | |
| port = int(os.environ["STUB_PORT"]) | |
| ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() | |
| PY | |
| python3 perf_stub.py & | |
| echo "STUB_PID=$!" >> "$GITHUB_ENV" | |
| # ------------------------------------------------------------------ | |
| # Start the proxy: a `type: model` route pointed at the local stub. | |
| # ------------------------------------------------------------------ | |
| - name: Start switchyard proxy | |
| run: | | |
| cat > bench.yaml <<YAML | |
| defaults: | |
| api_key: dummy | |
| base_url: http://localhost:$STUB_PORT/v1 | |
| format: openai | |
| routes: | |
| mock-model: | |
| type: model | |
| model: mock-model | |
| YAML | |
| uv run switchyard serve --routing-profiles bench.yaml --port $PROXY_PORT & | |
| echo "PROXY_PID=$!" >> "$GITHUB_ENV" | |
| - name: Wait for proxy to be ready | |
| run: | | |
| for i in $(seq 1 30); do | |
| if curl -sf http://localhost:$PROXY_PORT/health > /dev/null 2>&1; then | |
| echo "Proxy is up after ${i}s" | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "Proxy did not become ready in 30s" | |
| exit 1 | |
| # ------------------------------------------------------------------ | |
| # Pre-build a minimal input dataset so aiperf doesn't need to | |
| # download a HuggingFace tokenizer for synthetic data generation. | |
| # Combined with --use-server-token-count this makes the benchmark | |
| # completely self-contained (no network calls beyond the proxy). | |
| # ------------------------------------------------------------------ | |
| - name: Generate perf input dataset | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| # 300 entries — enough to cycle through for any request-count | |
| with open("perf-input.jsonl", "w") as f: | |
| for _ in range(300): | |
| f.write(json.dumps({"text": "What is 2 + 2?"}) + "\n") | |
| PY | |
| # ------------------------------------------------------------------ | |
| # Run the benchmark | |
| # ------------------------------------------------------------------ | |
| - name: Run aiperf benchmark (non-streaming) | |
| run: | | |
| mkdir -p perf-results | |
| uv run aiperf profile \ | |
| --model "$PERF_MODEL" \ | |
| --tokenizer "$PERF_TOKENIZER" \ | |
| --url "http://localhost:$PROXY_PORT" \ | |
| --endpoint-type chat \ | |
| --concurrency "$PERF_CONCURRENCY" \ | |
| --request-count "$PERF_REQUEST_COUNT" \ | |
| --ui none \ | |
| --custom-dataset-type single-turn \ | |
| --input-file perf-input.jsonl \ | |
| --use-server-token-count \ | |
| --output-artifact-dir perf-results/non-streaming | |
| - name: Run aiperf benchmark (streaming) | |
| run: | | |
| uv run aiperf profile \ | |
| --model "$PERF_MODEL" \ | |
| --tokenizer "$PERF_TOKENIZER" \ | |
| --url "http://localhost:$PROXY_PORT" \ | |
| --endpoint-type chat \ | |
| --streaming \ | |
| --concurrency "$PERF_CONCURRENCY" \ | |
| --request-count "$PERF_REQUEST_COUNT" \ | |
| --ui none \ | |
| --custom-dataset-type single-turn \ | |
| --input-file perf-input.jsonl \ | |
| --use-server-token-count \ | |
| --output-artifact-dir perf-results/streaming | |
| # ------------------------------------------------------------------ | |
| # Stop proxy and local stub | |
| # ------------------------------------------------------------------ | |
| - name: Stop proxy and stub | |
| if: always() | |
| run: | | |
| kill "$PROXY_PID" || true | |
| kill "$STUB_PID" || true | |
| # ------------------------------------------------------------------ | |
| # Upload results as a build artifact for comparison over time | |
| # ------------------------------------------------------------------ | |
| - name: Upload perf results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: perf-results-${{ github.sha }} | |
| path: perf-results/ | |
| retention-days: 90 |