feat(routing): add modality-aware target selection #1552

Workflow file for this run

	name: Perf Benchmark

	on:
	pull_request:
	push:
	branches: [main]
	workflow_dispatch:
	inputs:
	concurrency:
	description: "aiperf --concurrency"
	default: "10"
	request_count:
	description: "aiperf --request-count"
	default: "200"

	# Cancel superseded PR runs; keep main builds.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	permissions:
	contents: read

	jobs:
	proxy-perf:
	name: Proxy overhead (local stub backend)
	runs-on: ubuntu-latest

	env:
	PROXY_PORT: 4000
	STUB_PORT: 9000
	# A local zero-latency stub replaces the removed noop route; the proxy
	# forwards to it over loopback, which adds a small constant overhead.
	PERF_MODEL: mock-model
	# gpt2 tokenizer is ~500 KB and downloads in < 2 s on Actions runners.
	# It is only used for dataset-manager token counting; actual inference
	# token counts come from the server (--use-server-token-count).
	PERF_TOKENIZER: gpt2
	PERF_CONCURRENCY: ${{ inputs.concurrency \|\| '10' }}
	PERF_REQUEST_COUNT: ${{ inputs.request_count \|\| '200' }}

	steps:
	- uses: actions/checkout@v4

	- uses: astral-sh/setup-uv@v6
	with:
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: "uv.lock"

	- name: Install switchyard (default extras only)
	run: uv sync

	- name: Install aiperf
	run: uv pip install aiperf

	# ------------------------------------------------------------------
	# Start a local zero-latency mock OpenAI upstream. The proxy serves a
	# real `type: passthrough` route that
	# forwards to this loopback stub, which returns a fixed completion
	# instantly. The extra loopback hop adds a small constant overhead.
	# ------------------------------------------------------------------
	- name: Start local mock OpenAI upstream
	run: \|
	cat > perf_stub.py <<'PY'
	"""Zero-latency OpenAI-compatible chat.completions stub (loopback only)."""
	import json
	import os
	from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

	_COMPLETION = {
	"id": "chatcmpl-perf-stub",
	"object": "chat.completion",
	"created": 1700000000,
	"model": "mock-model",
	"choices": [
	{
	"index": 0,
	"message": {"role": "assistant", "content": "4"},
	"finish_reason": "stop",
	}
	],
	"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
	}

	_CHUNK = {
	"id": "chatcmpl-perf-stub",
	"object": "chat.completion.chunk",
	"created": 1700000000,
	"model": "mock-model",
	"choices": [
	{"index": 0, "delta": {"content": "4"}, "finish_reason": "stop"}
	],
	"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
	}


	class Handler(BaseHTTPRequestHandler):
	protocol_version = "HTTP/1.1"

	def do_POST(self):
	length = int(self.headers.get("content-length", "0"))
	body = json.loads(self.rfile.read(length) or b"{}")
	if body.get("stream"):
	payload = (
	f"data: {json.dumps(_CHUNK)}\n\n".encode()
	+ b"data: [DONE]\n\n"
	)
	content_type = "text/event-stream"
	else:
	payload = json.dumps(_COMPLETION).encode()
	content_type = "application/json"
	self.send_response(200)
	self.send_header("content-type", content_type)
	self.send_header("content-length", str(len(payload)))
	self.end_headers()
	self.wfile.write(payload)

	def log_message(self, *_args):
	return None


	port = int(os.environ["STUB_PORT"])
	ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()
	PY
	python3 perf_stub.py &
	echo "STUB_PID=$!" >> "$GITHUB_ENV"

	# ------------------------------------------------------------------
	# Build and start the native proxy against the local stub.
	# ------------------------------------------------------------------
	- name: Build switchyard server
	run: cargo build --locked --release -p switchyard-server

	- name: Start switchyard proxy
	run: \|
	cat > bench.toml <<TOML
	schema_version = 1

	[llm_clients.stub]
	format = "openai_chat"
	base_url = "http://localhost:$STUB_PORT/v1"

	[targets.mock]
	id = "mock-model"
	llm_client = "stub"

	[routes.mock]
	id = "mock-model"
	type = "passthrough"
	target = "mock"
	TOML
	target/release/switchyard-server --config bench.toml --port $PROXY_PORT &
	echo "PROXY_PID=$!" >> "$GITHUB_ENV"

	- name: Wait for proxy to be ready
	run: \|
	for i in $(seq 1 30); do
	if curl -sf http://localhost:$PROXY_PORT/health > /dev/null 2>&1; then
	echo "Proxy is up after ${i}s"
	exit 0
	fi
	sleep 1
	done
	echo "Proxy did not become ready in 30s"
	exit 1

	# ------------------------------------------------------------------
	# Pre-build a minimal input dataset so aiperf doesn't need to
	# download a HuggingFace tokenizer for synthetic data generation.
	# Combined with --use-server-token-count this makes the benchmark
	# completely self-contained (no network calls beyond the proxy).
	# ------------------------------------------------------------------
	- name: Generate perf input dataset
	run: \|
	python3 - <<'PY'
	import json
	# 300 entries — enough to cycle through for any request-count
	with open("perf-input.jsonl", "w") as f:
	for _ in range(300):
	f.write(json.dumps({"text": "What is 2 + 2?"}) + "\n")
	PY

	# ------------------------------------------------------------------
	# Run the benchmark
	# ------------------------------------------------------------------
	- name: Run aiperf benchmark (non-streaming)
	run: \|
	mkdir -p perf-results
	uv run aiperf profile \
	--model "$PERF_MODEL" \
	--tokenizer "$PERF_TOKENIZER" \
	--url "http://localhost:$PROXY_PORT" \
	--endpoint-type chat \
	--concurrency "$PERF_CONCURRENCY" \
	--request-count "$PERF_REQUEST_COUNT" \
	--ui none \
	--custom-dataset-type single-turn \
	--input-file perf-input.jsonl \
	--use-server-token-count \
	--output-artifact-dir perf-results/non-streaming

	- name: Run aiperf benchmark (streaming)
	run: \|
	uv run aiperf profile \
	--model "$PERF_MODEL" \
	--tokenizer "$PERF_TOKENIZER" \
	--url "http://localhost:$PROXY_PORT" \
	--endpoint-type chat \
	--streaming \
	--concurrency "$PERF_CONCURRENCY" \
	--request-count "$PERF_REQUEST_COUNT" \
	--ui none \
	--custom-dataset-type single-turn \
	--input-file perf-input.jsonl \
	--use-server-token-count \
	--output-artifact-dir perf-results/streaming

	# ------------------------------------------------------------------
	# Stop proxy and local stub
	# ------------------------------------------------------------------
	- name: Stop proxy and stub
	if: always()
	run: \|
	kill "$PROXY_PID" \|\| true
	kill "$STUB_PID" \|\| true

	# ------------------------------------------------------------------
	# Upload results as a build artifact for comparison over time
	# ------------------------------------------------------------------
	- name: Upload perf results
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: perf-results-${{ github.sha }}
	path: perf-results/
	retention-days: 90

Navigation Menu

Uh oh!

feat(routing): add modality-aware target selection #1552

Workflow file

feat(routing): add modality-aware target selection #1552

Uh oh!

Workflow file for this run