openresponses-conformance.yml

name: OpenResponses Conformance Tests

run-name: Run OpenResponses conformance tests against ogx Responses API

# This job is REQUIRED — it enforces conformance with the OpenResponses spec
# (https://openresponses.org). Any failing conformance test fails the job.
# See: https://github.com/ogx-ai/ogx/issues/4818
#
# Inference calls are replayed from checked-in recordings under
# tests/integration/openresponses/recordings/. To add or update recordings,
# run the server locally with OGX_TEST_INFERENCE_MODE=record-if-missing
# pointed at that directory, run the compliance tests, and commit the results.

on:
  push:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'
    paths:
      - 'src/ogx/providers/inline/responses/**'
      - 'src/ogx_api/responses/**'
      - 'src/ogx_api/openai_responses.py'
      - 'src/ogx/core/server/auth.py'
      - 'tests/integration/openresponses/**'
      - '.github/workflows/openresponses-conformance.yml'
  pull_request:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'
    paths:
      - 'src/ogx/providers/inline/responses/**'
      - 'src/ogx_api/responses/**'
      - 'src/ogx_api/openai_responses.py'
      - 'src/ogx/core/server/auth.py'
      - 'tests/integration/openresponses/**'
      - '.github/workflows/openresponses-conformance.yml'
  merge_group:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  openresponses-conformance:
    name: OpenResponses Conformance
    runs-on: ubuntu-latest

    env:
      # Dummy key satisfies config parsing; actual inference is replayed from recordings
      OPENAI_API_KEY: "dummy-key-for-replay-mode"
      INFERENCE_MODEL: "openai/gpt-4o-mini"
      OGX_PORT: 8321
      OGX_TEST_INFERENCE_MODE: "replay"
      OGX_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/openresponses

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Check for recordings
        id: check-recordings
        run: |
          RECORDINGS_DIR="${{ github.workspace }}/tests/integration/openresponses/recordings"
          COUNT=$(find "$RECORDINGS_DIR" -name "*.json" 2>/dev/null | wc -l)
          echo "recordings_count=$COUNT" >> "$GITHUB_OUTPUT"
          if [ "$COUNT" -eq 0 ]; then
            echo "::warning::No recordings found in tests/integration/openresponses/recordings/."
            echo "::warning::Run the server locally with OGX_TEST_INFERENCE_MODE=record-if-missing"
            echo "::warning::and commit the resulting recordings to enable conformance testing in CI."
          else
            echo "Found $COUNT recording(s)"
          fi

      - name: Install dependencies
        uses: ./.github/actions/setup-runner
        with:
          python-version: '3.12'

      - name: Install ci-tests distro provider dependencies
        run: uv run ogx stack list-deps ci-tests --format uv | sh

      - name: Start OGX server
        run: |
          mkdir -p /tmp/ogx-conformance
          export OGX_LOG_WIDTH=200
          nohup uv run ogx stack run ci-tests --port "$OGX_PORT" --insecure \
            > /tmp/ogx-conformance/server.log 2>&1 &
          echo "Server PID: $!"

      - name: Wait for OGX server to be ready
        run: |
          echo "Waiting for OGX server..."
          for _ in {1..60}; do
            if curl -s "http://localhost:$OGX_PORT/v1/health" | grep -q "OK"; then
              echo "OGX server is ready!"
              exit 0
            fi
            sleep 2
          done
          echo "OGX server failed to start in 120 seconds"
          cat /tmp/ogx-conformance/server.log
          exit 1

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 #v2.0.0

      - name: Clone OpenResponses repository
        run: |
          git clone --depth=1 https://github.com/openresponses/openresponses.git /tmp/openresponses

      - name: Install OpenResponses dependencies
        working-directory: /tmp/openresponses
        run: bun install

      - name: Run OpenResponses conformance tests
        id: run-tests
        run: |
          cd /tmp/openresponses
          bun run bin/compliance-test.ts \
            --base-url "http://localhost:$OGX_PORT/v1" \
            --api-key "ogx" \
            --model "$INFERENCE_MODEL" \
            --json > /tmp/openresponses-results.json 2>/dev/null || true

          echo "=== OpenResponses Conformance Test Output ==="
          bun run bin/compliance-test.ts \
            --base-url "http://localhost:$OGX_PORT/v1" \
            --api-key "ogx" \
            --model "$INFERENCE_MODEL" \
            --verbose 2>&1 | tee /tmp/openresponses-output.txt || true

      - name: Generate conformance report
        if: always()
        run: |
          {
            echo "## OpenResponses Conformance Test Results"
            echo ""
            echo "> These tests enforce [OpenResponses](https://www.openresponses.org) API"
            echo "> conformance. Any failing test fails this job."
            echo "> See [#4818](https://github.com/ogx-ai/ogx/issues/4818)."
            echo ""
            echo "**Model:** \`$INFERENCE_MODEL\`"
            echo "**Recordings:** ${{ steps.check-recordings.outputs.recordings_count }} file(s) in \`tests/integration/openresponses/recordings/\`"
            echo ""
          } >> "$GITHUB_STEP_SUMMARY"

          if [ "${{ steps.check-recordings.outputs.recordings_count }}" = "0" ]; then
            {
              echo "### No Recordings Found"
              echo ""
              echo "To enable conformance testing, generate recordings locally:"
              echo '```bash'
              echo "OPENAI_API_KEY=\$YOUR_KEY bash scripts/record-openresponses-conformance.sh"
              echo ""
              echo "# Then commit the recordings"
              echo "git add tests/integration/openresponses/recordings/"
              echo "git commit -m 'chore: add OpenResponses conformance recordings'"
              echo '```'
              echo "Commit the resulting \`tests/integration/openresponses/recordings/*.json\` files."
            } >> "$GITHUB_STEP_SUMMARY"
          elif [ -f /tmp/openresponses-results.json ] && jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
            PASSED=$(jq -r '.summary.passed' /tmp/openresponses-results.json)
            FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
            TOTAL=$(jq -r '.summary.total' /tmp/openresponses-results.json)

            {
              echo "### Summary"
              echo ""
              echo "| Metric | Count |"
              echo "|--------|-------|"
              echo "| ✅ Passed | $PASSED |"
              echo "| ❌ Failed | $FAILED |"
              echo "| **Total** | **$TOTAL** |"
              echo ""
              echo "### Test Details"
              echo ""
              echo "| Test | Status | Duration |"
              echo "|------|--------|----------|"
              jq -r '.results[] | "| \(.name) | \(if .status == "passed" then "✅ Pass" else "❌ Fail" end) | \(.duration // "—")ms |"' \
                /tmp/openresponses-results.json
              echo ""
            } >> "$GITHUB_STEP_SUMMARY"

            if [ "$FAILED" -gt 0 ]; then
              {
                echo "### Failure Details"
                echo ""
                jq -r '.results[] | select(.status == "failed") | "**\(.name)**\n" + ((.errors // []) | map("- \(.)") | join("\n")) + "\n"' \
                  /tmp/openresponses-results.json
              } >> "$GITHUB_STEP_SUMMARY"
            fi
          else
            {
              echo "### Results"
              echo ""
              echo "Could not parse structured results."
              if [ -f /tmp/openresponses-output.txt ]; then
                echo ""
                echo "<details><summary>Raw output</summary>"
                echo ""
                echo '```'
                cat /tmp/openresponses-output.txt
                echo '```'
                echo ""
                echo "</details>"
              fi
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Enforce conformance
        run: |
          if [ ! -f /tmp/openresponses-results.json ] || ! jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
            echo "::error::No parseable OpenResponses conformance results were produced."
            exit 1
          fi
          FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
          if [ "$FAILED" -gt 0 ]; then
            echo "::error::$FAILED OpenResponses conformance test(s) failed."
            exit 1
          fi
          echo "All OpenResponses conformance tests passed."

      - name: Upload conformance test results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: openresponses-conformance-results-${{ github.run_id }}
          path: |
            /tmp/openresponses-results.json
            /tmp/openresponses-output.txt
          retention-days: 30
          if-no-files-found: warn

      - name: Print server log on failure
        if: failure()
        run: |
          echo "=== OGX server log (last 200 lines) ==="
          tail -200 /tmp/ogx-conformance/server.log || true