Run OpenResponses conformance tests against ogx Responses API #1572

Workflow file for this run

.github/workflows/openresponses-conformance.yml at cac8b20

	name: OpenResponses Conformance Tests

	run-name: Run OpenResponses conformance tests against ogx Responses API

	# This job is REQUIRED — it enforces conformance with the OpenResponses spec
	# (https://openresponses.org). Any failing conformance test fails the job.
	# See: https://github.com/ogx-ai/ogx/issues/4818
	#
	# Inference calls are replayed from checked-in recordings under
	# tests/integration/openresponses/recordings/. To add or update recordings,
	# run the server locally with OGX_TEST_INFERENCE_MODE=record-if-missing
	# pointed at that directory, run the compliance tests, and commit the results.

	on:
	push:
	branches:
	- main
	- 'release-[0-9]+.[0-9]+.x'
	paths:
	- 'src/ogx/providers/inline/responses/**'
	- 'src/ogx_api/responses/**'
	- 'src/ogx_api/openai_responses.py'
	- 'src/ogx/core/server/auth.py'
	- 'tests/integration/openresponses/**'
	- '.github/workflows/openresponses-conformance.yml'
	pull_request:
	branches:
	- main
	- 'release-[0-9]+.[0-9]+.x'
	paths:
	- 'src/ogx/providers/inline/responses/**'
	- 'src/ogx_api/responses/**'
	- 'src/ogx_api/openai_responses.py'
	- 'src/ogx/core/server/auth.py'
	- 'tests/integration/openresponses/**'
	- '.github/workflows/openresponses-conformance.yml'
	merge_group:
	branches:
	- main
	- 'release-[0-9]+.[0-9]+.x'
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	openresponses-conformance:
	name: OpenResponses Conformance
	runs-on: ubuntu-latest

	env:
	# Dummy key satisfies config parsing; actual inference is replayed from recordings
	OPENAI_API_KEY: "dummy-key-for-replay-mode"
	INFERENCE_MODEL: "openai/gpt-4o-mini"
	OGX_PORT: 8321
	OGX_TEST_INFERENCE_MODE: "replay"
	OGX_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/openresponses

	steps:
	- name: Checkout repository
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Check for recordings
	id: check-recordings
	run: \|
	RECORDINGS_DIR="${{ github.workspace }}/tests/integration/openresponses/recordings"
	COUNT=$(find "$RECORDINGS_DIR" -name "*.json" 2>/dev/null \| wc -l)
	echo "recordings_count=$COUNT" >> "$GITHUB_OUTPUT"
	if [ "$COUNT" -eq 0 ]; then
	echo "::warning::No recordings found in tests/integration/openresponses/recordings/."
	echo "::warning::Run the server locally with OGX_TEST_INFERENCE_MODE=record-if-missing"
	echo "::warning::and commit the resulting recordings to enable conformance testing in CI."
	else
	echo "Found $COUNT recording(s)"
	fi

	- name: Install dependencies
	uses: ./.github/actions/setup-runner
	with:
	python-version: '3.12'

	- name: Install ci-tests distro provider dependencies
	run: uv run ogx stack list-deps ci-tests --format uv \| sh

	- name: Start OGX server
	run: \|
	mkdir -p /tmp/ogx-conformance
	export OGX_LOG_WIDTH=200
	nohup uv run ogx stack run ci-tests --port "$OGX_PORT" --insecure \
	> /tmp/ogx-conformance/server.log 2>&1 &
	echo "Server PID: $!"

	- name: Wait for OGX server to be ready
	run: \|
	echo "Waiting for OGX server..."
	for _ in {1..60}; do
	if curl -s "http://localhost:$OGX_PORT/v1/health" \| grep -q "OK"; then
	echo "OGX server is ready!"
	exit 0
	fi
	sleep 2
	done
	echo "OGX server failed to start in 120 seconds"
	cat /tmp/ogx-conformance/server.log
	exit 1

	- name: Setup Bun
	uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 #v2.0.0

	- name: Clone OpenResponses repository
	run: \|
	git clone --depth=1 https://github.com/openresponses/openresponses.git /tmp/openresponses

	- name: Install OpenResponses dependencies
	working-directory: /tmp/openresponses
	run: bun install

	- name: Run OpenResponses conformance tests
	id: run-tests
	run: \|
	cd /tmp/openresponses
	bun run bin/compliance-test.ts \
	--base-url "http://localhost:$OGX_PORT/v1" \
	--api-key "ogx" \
	--model "$INFERENCE_MODEL" \
	--json > /tmp/openresponses-results.json 2>/dev/null \|\| true

	echo "=== OpenResponses Conformance Test Output ==="
	bun run bin/compliance-test.ts \
	--base-url "http://localhost:$OGX_PORT/v1" \
	--api-key "ogx" \
	--model "$INFERENCE_MODEL" \
	--verbose 2>&1 \| tee /tmp/openresponses-output.txt \|\| true

	- name: Generate conformance report
	if: always()
	run: \|
	{
	echo "## OpenResponses Conformance Test Results"
	echo ""
	echo "> These tests enforce [OpenResponses](https://www.openresponses.org) API"
	echo "> conformance. Any failing test fails this job."
	echo "> See [#4818](https://github.com/ogx-ai/ogx/issues/4818)."
	echo ""
	echo "Model: \`$INFERENCE_MODEL\`"
	echo "Recordings: ${{ steps.check-recordings.outputs.recordings_count }} file(s) in \`tests/integration/openresponses/recordings/\`"
	echo ""
	} >> "$GITHUB_STEP_SUMMARY"

	if [ "${{ steps.check-recordings.outputs.recordings_count }}" = "0" ]; then
	{
	echo "### No Recordings Found"
	echo ""
	echo "To enable conformance testing, generate recordings locally:"
	echo '```bash'
	echo "OPENAI_API_KEY=\$YOUR_KEY bash scripts/record-openresponses-conformance.sh"
	echo ""
	echo "# Then commit the recordings"
	echo "git add tests/integration/openresponses/recordings/"
	echo "git commit -m 'chore: add OpenResponses conformance recordings'"
	echo '```'
	echo "Commit the resulting \`tests/integration/openresponses/recordings/*.json\` files."
	} >> "$GITHUB_STEP_SUMMARY"
	elif [ -f /tmp/openresponses-results.json ] && jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
	PASSED=$(jq -r '.summary.passed' /tmp/openresponses-results.json)
	FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
	TOTAL=$(jq -r '.summary.total' /tmp/openresponses-results.json)

	{
	echo "### Summary"
	echo ""
	echo "\| Metric \| Count \|"
	echo "\|--------\|-------\|"
	echo "\| ✅ Passed \| $PASSED \|"
	echo "\| ❌ Failed \| $FAILED \|"
	echo "\| Total \| $TOTAL \|"
	echo ""
	echo "### Test Details"
	echo ""
	echo "\| Test \| Status \| Duration \|"
	echo "\|------\|--------\|----------\|"
	jq -r '.results[] \| "\| \(.name) \| \(if .status == "passed" then "✅ Pass" else "❌ Fail" end) \| \(.duration // "—")ms \|"' \
	/tmp/openresponses-results.json
	echo ""
	} >> "$GITHUB_STEP_SUMMARY"

	if [ "$FAILED" -gt 0 ]; then
	{
	echo "### Failure Details"
	echo ""
	jq -r '.results[] \| select(.status == "failed") \| "\(.name)\n" + ((.errors // []) \| map("- \(.)") \| join("\n")) + "\n"' \
	/tmp/openresponses-results.json
	} >> "$GITHUB_STEP_SUMMARY"
	fi
	else
	{
	echo "### Results"
	echo ""
	echo "Could not parse structured results."
	if [ -f /tmp/openresponses-output.txt ]; then
	echo ""
	echo "<details><summary>Raw output</summary>"
	echo ""
	echo '```'
	cat /tmp/openresponses-output.txt
	echo '```'
	echo ""
	echo "</details>"
	fi
	} >> "$GITHUB_STEP_SUMMARY"
	fi

	- name: Enforce conformance
	run: \|
	if [ ! -f /tmp/openresponses-results.json ] \|\| ! jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
	echo "::error::No parseable OpenResponses conformance results were produced."
	exit 1
	fi
	FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
	if [ "$FAILED" -gt 0 ]; then
	echo "::error::$FAILED OpenResponses conformance test(s) failed."
	exit 1
	fi
	echo "All OpenResponses conformance tests passed."

	- name: Upload conformance test results
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: openresponses-conformance-results-${{ github.run_id }}
	path: \|
	/tmp/openresponses-results.json
	/tmp/openresponses-output.txt
	retention-days: 30
	if-no-files-found: warn

	- name: Print server log on failure
	if: failure()
	run: \|
	echo "=== OGX server log (last 200 lines) ==="
	tail -200 /tmp/ogx-conformance/server.log \|\| true

Navigation Menu

Uh oh!

Run OpenResponses conformance tests against ogx Responses API #1572

Workflow file

Run OpenResponses conformance tests against ogx Responses API #1572

Uh oh!

Workflow file for this run