Skip to content

Commit 2177c6e

Browse files
cdoerncdoern
authored
feat: add OpenResponses conformance CI job with replay recordings (#4981)
1 parent 2494453 commit 2177c6e

12 files changed

Lines changed: 4537 additions & 0 deletions

‎.github/workflows/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
1515
| Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
1616
| Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
1717
| OpenAPI Generator SDK Validation | [openapi-generator-validation.yml](openapi-generator-validation.yml) | Validate OpenAPI Generator SDK generation |
18+
| OpenResponses Conformance Tests | [openresponses-conformance.yml](openresponses-conformance.yml) | Run OpenResponses conformance tests against llama-stack Responses API |
1819
| Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
1920
| Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
2021
| Test llama stack list-deps | [providers-list-deps.yml](providers-list-deps.yml) | Test llama stack list-deps |

‎.github/workflows/openresponses-conformance.yml‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
name: OpenResponses Conformance Tests
2+
3+
run-name: Run OpenResponses conformance tests against llama-stack Responses API
4+
5+
# This job is OPTIONAL and informational — it tracks progress toward full
6+
# conformance with the OpenResponses spec (https://openresponses.org).
7+
# Failures are expected while gaps remain in the Responses API implementation.
8+
# See: https://github.com/llamastack/llama-stack/issues/4818
9+
#
10+
# Inference calls are replayed from checked-in recordings under
11+
# tests/integration/openresponses/recordings/. To add or update recordings,
12+
# run the server locally with LLAMA_STACK_TEST_INFERENCE_MODE=record-if-missing
13+
# pointed at that directory, run the compliance tests, and commit the results.
14+
15+
on:
16+
push:
17+
branches:
18+
- main
19+
- 'release-[0-9]+.[0-9]+.x'
20+
paths:
21+
- 'src/llama_stack/providers/inline/agents/**'
22+
- 'src/llama_stack/apis/agents/**'
23+
- 'tests/integration/openresponses/**'
24+
- '.github/workflows/openresponses-conformance.yml'
25+
pull_request:
26+
branches:
27+
- main
28+
- 'release-[0-9]+.[0-9]+.x'
29+
paths:
30+
- 'src/llama_stack/providers/inline/agents/**'
31+
- 'src/llama_stack/apis/agents/**'
32+
- 'tests/integration/openresponses/**'
33+
- '.github/workflows/openresponses-conformance.yml'
34+
workflow_dispatch:
35+
36+
concurrency:
37+
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
38+
cancel-in-progress: true
39+
40+
permissions:
41+
contents: read
42+
43+
jobs:
44+
openresponses-conformance:
45+
name: OpenResponses Conformance (Informational)
46+
runs-on: ubuntu-latest
47+
# Failures are expected — this job does NOT block PRs or merges.
48+
# It exists solely to track progress toward OpenResponses API conformance.
49+
continue-on-error: true
50+
51+
env:
52+
# Dummy key satisfies config parsing; actual inference is replayed from recordings
53+
OPENAI_API_KEY: "dummy-key-for-replay-mode"
54+
INFERENCE_MODEL: "openai/gpt-4o-mini"
55+
LLAMA_STACK_PORT: 8321
56+
LLAMA_STACK_TEST_INFERENCE_MODE: "replay"
57+
LLAMA_STACK_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/openresponses
58+
59+
steps:
60+
- name: Checkout repository
61+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
62+
63+
- name: Check for recordings
64+
id: check-recordings
65+
run: |
66+
RECORDINGS_DIR="${{ github.workspace }}/tests/integration/openresponses/recordings"
67+
COUNT=$(find "$RECORDINGS_DIR" -name "*.json" 2>/dev/null | wc -l)
68+
echo "recordings_count=$COUNT" >> $GITHUB_OUTPUT
69+
if [ "$COUNT" -eq 0 ]; then
70+
echo "::warning::No recordings found in tests/integration/openresponses/recordings/."
71+
echo "::warning::Run the server locally with LLAMA_STACK_TEST_INFERENCE_MODE=record-if-missing"
72+
echo "::warning::and commit the resulting recordings to enable conformance testing in CI."
73+
else
74+
echo "Found $COUNT recording(s)"
75+
fi
76+
77+
- name: Install dependencies
78+
uses: ./.github/actions/setup-runner
79+
with:
80+
python-version: '3.12'
81+
82+
- name: Install ci-tests distro provider dependencies
83+
run: uv run llama stack list-deps ci-tests --format uv | sh
84+
85+
- name: Start Llama Stack server
86+
run: |
87+
mkdir -p /tmp/llama-stack-conformance
88+
export LLAMA_STACK_LOG_WIDTH=200
89+
nohup uv run llama stack run ci-tests --port $LLAMA_STACK_PORT \
90+
> /tmp/llama-stack-conformance/server.log 2>&1 &
91+
echo "Server PID: $!"
92+
93+
- name: Wait for Llama Stack server to be ready
94+
run: |
95+
echo "Waiting for Llama Stack server..."
96+
for i in {1..60}; do
97+
if curl -s http://localhost:$LLAMA_STACK_PORT/v1/health | grep -q "OK"; then
98+
echo "Llama Stack server is ready!"
99+
exit 0
100+
fi
101+
sleep 2
102+
done
103+
echo "Llama Stack server failed to start in 120 seconds"
104+
cat /tmp/llama-stack-conformance/server.log
105+
exit 1
106+
107+
- name: Setup Bun
108+
uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 #v2.0.0
109+
110+
- name: Clone OpenResponses repository
111+
run: |
112+
git clone --depth=1 https://github.com/openresponses/openresponses.git /tmp/openresponses
113+
114+
- name: Install OpenResponses dependencies
115+
working-directory: /tmp/openresponses
116+
run: bun install
117+
118+
- name: Run OpenResponses conformance tests
119+
id: run-tests
120+
run: |
121+
cd /tmp/openresponses
122+
bun run bin/compliance-test.ts \
123+
--base-url "http://localhost:$LLAMA_STACK_PORT/v1" \
124+
--api-key "llama-stack" \
125+
--model "$INFERENCE_MODEL" \
126+
--json > /tmp/openresponses-results.json 2>/dev/null || true
127+
128+
echo "=== OpenResponses Conformance Test Output ==="
129+
bun run bin/compliance-test.ts \
130+
--base-url "http://localhost:$LLAMA_STACK_PORT/v1" \
131+
--api-key "llama-stack" \
132+
--model "$INFERENCE_MODEL" \
133+
--verbose 2>&1 | tee /tmp/openresponses-output.txt || true
134+
135+
- name: Generate conformance report
136+
if: always()
137+
run: |
138+
{
139+
echo "## OpenResponses Conformance Test Results"
140+
echo ""
141+
echo "> **Note:** These tests are **informational only** and track progress toward full"
142+
echo "> [OpenResponses](https://www.openresponses.org) API conformance."
143+
echo "> Failures are expected while gaps remain in the Responses API implementation."
144+
echo "> See [#4818](https://github.com/llamastack/llama-stack/issues/4818)."
145+
echo ""
146+
echo "**Model:** \`$INFERENCE_MODEL\`"
147+
echo "**Recordings:** ${{ steps.check-recordings.outputs.recordings_count }} file(s) in \`tests/integration/openresponses/recordings/\`"
148+
echo ""
149+
} >> $GITHUB_STEP_SUMMARY
150+
151+
if [ "${{ steps.check-recordings.outputs.recordings_count }}" -eq 0 ]; then
152+
{
153+
echo "### No Recordings Found"
154+
echo ""
155+
echo "To enable conformance testing, generate recordings locally:"
156+
echo '```bash'
157+
echo "OPENAI_API_KEY=\$YOUR_KEY bash scripts/record-openresponses-conformance.sh"
158+
echo ""
159+
echo "# Then commit the recordings"
160+
echo "git add tests/integration/openresponses/recordings/"
161+
echo "git commit -m 'chore: add OpenResponses conformance recordings'"
162+
echo '```'
163+
echo "Commit the resulting \`tests/integration/openresponses/recordings/*.json\` files."
164+
} >> $GITHUB_STEP_SUMMARY
165+
elif [ -f /tmp/openresponses-results.json ] && jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
166+
PASSED=$(jq -r '.summary.passed' /tmp/openresponses-results.json)
167+
FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
168+
TOTAL=$(jq -r '.summary.total' /tmp/openresponses-results.json)
169+
170+
{
171+
echo "### Summary"
172+
echo ""
173+
echo "| Metric | Count |"
174+
echo "|--------|-------|"
175+
echo "| ✅ Passed | $PASSED |"
176+
echo "| ❌ Failed | $FAILED |"
177+
echo "| **Total** | **$TOTAL** |"
178+
echo ""
179+
echo "### Test Details"
180+
echo ""
181+
echo "| Test | Status | Duration |"
182+
echo "|------|--------|----------|"
183+
jq -r '.results[] | "| \(.name) | \(if .status == "passed" then "✅ Pass" else "❌ Fail" end) | \(.duration // "—")ms |"' \
184+
/tmp/openresponses-results.json
185+
echo ""
186+
} >> $GITHUB_STEP_SUMMARY
187+
188+
if [ "$FAILED" -gt 0 ]; then
189+
{
190+
echo "### Failure Details"
191+
echo ""
192+
jq -r '.results[] | select(.status == "failed") | "**\(.name)**\n" + ((.errors // []) | map("- \(.)") | join("\n")) + "\n"' \
193+
/tmp/openresponses-results.json
194+
} >> $GITHUB_STEP_SUMMARY
195+
fi
196+
else
197+
{
198+
echo "### Results"
199+
echo ""
200+
echo "Could not parse structured results."
201+
if [ -f /tmp/openresponses-output.txt ]; then
202+
echo ""
203+
echo "<details><summary>Raw output</summary>"
204+
echo ""
205+
echo '```'
206+
cat /tmp/openresponses-output.txt
207+
echo '```'
208+
echo ""
209+
echo "</details>"
210+
fi
211+
} >> $GITHUB_STEP_SUMMARY
212+
fi
213+
214+
- name: Upload conformance test results
215+
if: always()
216+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
217+
with:
218+
name: openresponses-conformance-results-${{ github.run_id }}
219+
path: |
220+
/tmp/openresponses-results.json
221+
/tmp/openresponses-output.txt
222+
retention-days: 30
223+
if-no-files-found: warn
224+
225+
- name: Print server log on failure
226+
if: failure()
227+
run: |
228+
echo "=== Llama Stack server log (last 200 lines) ==="
229+
tail -200 /tmp/llama-stack-conformance/server.log || true

‎scripts/record-openresponses-conformance.sh‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the terms described in the LICENSE file in
6+
# the root directory of this source tree.
7+
8+
set -euo pipefail
9+
10+
# Records OpenResponses conformance test interactions against a local llama-stack
11+
# server so that CI can replay them without a live API key.
12+
#
13+
# Run this script whenever you add new compliance tests or the openresponses
14+
# test suite changes, then commit the resulting recordings:
15+
#
16+
# git add tests/integration/openresponses/recordings/
17+
# git commit -m "chore: update OpenResponses conformance recordings"
18+
#
19+
# Requirements:
20+
# - OPENAI_API_KEY must be set
21+
# - uv must be available (https://github.com/astral-sh/uv)
22+
# - bun will be installed automatically if missing
23+
24+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
25+
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
26+
RECORDING_DIR="$REPO_ROOT/tests/integration/openresponses"
27+
PORT="${PORT:-8321}"
28+
INFERENCE_MODEL="${INFERENCE_MODEL:-openai/gpt-4o-mini}"
29+
OPENRESPONSES_DIR="${OPENRESPONSES_DIR:-/tmp/openresponses}"
30+
LOG_FILE="/tmp/openresponses-server.log"
31+
32+
# ── Cleanup ────────────────────────────────────────────────────────────────────
33+
SERVER_PID=""
34+
cleanup() {
35+
if [[ -n "$SERVER_PID" ]]; then
36+
echo ""
37+
echo "Stopping llama-stack server (PID $SERVER_PID)..."
38+
kill "$SERVER_PID" 2>/dev/null || true
39+
fi
40+
}
41+
trap cleanup EXIT
42+
43+
# ── Preflight checks ───────────────────────────────────────────────────────────
44+
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
45+
echo "Error: OPENAI_API_KEY must be set to record conformance test interactions."
46+
exit 1
47+
fi
48+
49+
cd "$REPO_ROOT"
50+
51+
# ── Bun ────────────────────────────────────────────────────────────────────────
52+
if ! command -v bun &>/dev/null; then
53+
echo "=== Installing bun ==="
54+
curl -fsSL https://bun.sh/install | bash
55+
export PATH="$HOME/.bun/bin:$PATH"
56+
fi
57+
58+
# ── OpenResponses CLI ──────────────────────────────────────────────────────────
59+
if [[ -d "$OPENRESPONSES_DIR/.git" ]]; then
60+
echo "=== Updating openresponses ==="
61+
git -C "$OPENRESPONSES_DIR" pull --ff-only
62+
else
63+
echo "=== Cloning openresponses ==="
64+
git clone --depth=1 https://github.com/openresponses/openresponses.git "$OPENRESPONSES_DIR"
65+
fi
66+
echo "=== Installing openresponses dependencies ==="
67+
(cd "$OPENRESPONSES_DIR" && bun install)
68+
69+
# ── llama-stack provider dependencies ─────────────────────────────────────────
70+
echo "=== Installing ci-tests distro dependencies ==="
71+
llama stack list-deps ci-tests --format uv | sh
72+
73+
# ── Start server ───────────────────────────────────────────────────────────────
74+
echo "=== Starting llama-stack server (record-if-missing) ==="
75+
mkdir -p "$(dirname "$LOG_FILE")"
76+
77+
LLAMA_STACK_TEST_INFERENCE_MODE=record-if-missing \
78+
LLAMA_STACK_TEST_RECORDING_DIR="$RECORDING_DIR" \
79+
LLAMA_STACK_LOG_WIDTH=200 \
80+
nohup llama stack run ci-tests --port "$PORT" \
81+
> "$LOG_FILE" 2>&1 &
82+
SERVER_PID=$!
83+
echo "Server PID: $SERVER_PID"
84+
85+
# ── Wait for health ────────────────────────────────────────────────────────────
86+
echo "Waiting for llama-stack server to be ready..."
87+
for i in {1..60}; do
88+
if curl -sf "http://localhost:$PORT/v1/health" 2>/dev/null | grep -q "OK"; then
89+
echo "Server is ready!"
90+
break
91+
fi
92+
if [[ $i -eq 60 ]]; then
93+
echo "Server failed to start within 120 seconds. Log:"
94+
cat "$LOG_FILE"
95+
exit 1
96+
fi
97+
sleep 2
98+
done
99+
100+
# ── Run compliance tests ───────────────────────────────────────────────────────
101+
echo ""
102+
echo "=== Running OpenResponses compliance tests ==="
103+
echo ""
104+
(
105+
cd "$OPENRESPONSES_DIR"
106+
bun run bin/compliance-test.ts \
107+
--base-url "http://localhost:$PORT/v1" \
108+
--api-key "llama-stack" \
109+
--model "$INFERENCE_MODEL" \
110+
--verbose
111+
) || true # continue-on-error: failures here are expected while the implementation has gaps
112+
113+
# ── Summary ────────────────────────────────────────────────────────────────────
114+
echo ""
115+
echo "=== Recordings written to: $RECORDING_DIR/recordings/ ==="
116+
RECORDING_COUNT=$(find "$RECORDING_DIR/recordings" -name "*.json" 2>/dev/null | wc -l | tr -d ' ')
117+
echo "Total recording files: $RECORDING_COUNT"
118+
echo ""
119+
if [[ "$RECORDING_COUNT" -gt 0 ]]; then
120+
echo "Commit the recordings to include them in CI:"
121+
echo ""
122+
echo " git add tests/integration/openresponses/recordings/"
123+
echo " git commit -m 'chore: add OpenResponses conformance recordings'"
124+
else
125+
echo "No recordings were created. Check $LOG_FILE for server errors."
126+
fi

0 commit comments

Comments
 (0)