Auto-record missing test recordings for PR #3294
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Auto-records missing integration test recordings when PRs are opened or updated. | |
| # Records for multiple providers (ollama, gpt, azure, etc.) and uploads recordings as artifacts. | |
| # A companion workflow (commit-recordings.yml) commits the artifacts back to the PR. | |
| # Expandable: add new providers by adding entries to the providers matrix. | |
| # | |
| # SECURITY NOTE: This workflow uses pull_request (not pull_request_target) for security. | |
| # Security measures: | |
| # 1. Runs with read-only permissions (no write access to repo or secrets exposure) | |
| # 2. Only ollama variants run automatically on pull_request (no secrets needed) | |
| # 3. API key providers (gpt, azure, bedrock) only run via manual workflow_dispatch | |
| # 4. Fork PRs with API key providers are blocked (prevents secret theft) | |
| # 5. Recordings uploaded as artifacts; companion workflow handles commits | |
| name: Integration Tests (Record) | |
| run-name: Auto-record missing test recordings for PR #${{ github.event.pull_request.number || inputs.pr_number || github.run_number }} | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - 'tests/integration/**' | |
| - 'src/ogx/**' | |
| - '!src/ogx_ui/**' | |
| # Exclude recordings to prevent a feedback loop with commit-recordings.yml: | |
| # that workflow pushes recordings back to the PR branch, which would | |
| # otherwise re-trigger this workflow via pull_request: synchronize. | |
| - '!tests/integration/**/recordings/**' | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number to record for (optional, uses current branch if not provided)' | |
| type: number | |
| required: false | |
| providers: | |
| description: 'Comma-separated list of providers to record' | |
| type: string | |
| required: false | |
| default: 'gpt,azure,watsonx,vertexai' | |
| suite: | |
| description: 'Test suite override (default: per-provider from matrix)' | |
| type: string | |
| required: false | |
| default: '' | |
| subdirs: | |
| description: 'Comma-separated list of test subdirectories to run; overrides suite' | |
| type: string | |
| required: false | |
| default: '' | |
| pattern: | |
| description: 'Regex pattern to pass to pytest -k' | |
| type: string | |
| required: false | |
| default: '' | |
| commit_mode: | |
| description: 'How to land recordings: pr (commit to PR branch), branch (open a new PR against the dispatched branch), none (artifacts only). Leave blank to auto-select.' | |
| type: string | |
| required: false | |
| default: '' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.run_number }} | |
| cancel-in-progress: true | |
| # Read-only permissions - no write access | |
| # id-token: write is required for GCP Workload Identity Federation (OIDC token exchange) | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| id-token: write | |
| jobs: | |
| # Compute PR information for both pull_request and workflow_dispatch | |
| compute-pr-info: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pr_number: ${{ steps.compute.outputs.pr_number }} | |
| pr_head_ref: ${{ steps.compute.outputs.pr_head_ref }} | |
| pr_head_sha: ${{ steps.compute.outputs.pr_head_sha }} | |
| pr_head_repo: ${{ steps.compute.outputs.pr_head_repo }} | |
| is_fork_pr: ${{ steps.compute.outputs.is_fork_pr }} | |
| providers_to_run: ${{ steps.compute.outputs.providers_to_run }} | |
| commit_mode: ${{ steps.compute.outputs.commit_mode }} | |
| steps: | |
| - name: Compute PR metadata | |
| id: compute | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| # Pass attacker-controllable values via env vars to prevent code injection. | |
| # Never interpolate these directly in shell with ${{ }} syntax. | |
| EVENT_NAME: ${{ github.event_name }} | |
| INPUT_PR_NUMBER: ${{ inputs.pr_number }} | |
| INPUT_PROVIDERS: ${{ inputs.providers }} | |
| INPUT_COMMIT_MODE: ${{ inputs.commit_mode }} | |
| REPO: ${{ github.repository }} | |
| REF_NAME: ${{ github.ref_name }} | |
| SHA: ${{ github.sha }} | |
| PR_NUMBER_EVENT: ${{ github.event.pull_request.number }} | |
| PR_HEAD_REF_EVENT: ${{ github.event.pull_request.head.ref }} | |
| PR_HEAD_SHA_EVENT: ${{ github.event.pull_request.head.sha }} | |
| PR_HEAD_REPO_EVENT: ${{ github.event.pull_request.head.repo.full_name }} | |
| run: | | |
| if [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| if [ -n "$INPUT_PR_NUMBER" ]; then | |
| # Fetch PR info via API | |
| PR_DATA=$(gh pr view "$INPUT_PR_NUMBER" --repo "$REPO" --json number,headRefName,headRefOid,headRepository,headRepositoryOwner) | |
| PR_NUMBER="$INPUT_PR_NUMBER" | |
| HEAD_REF=$(echo "$PR_DATA" | jq -r '.headRefName') | |
| HEAD_SHA=$(echo "$PR_DATA" | jq -r '.headRefOid') | |
| # headRepository.nameWithOwner can be empty for fork PRs, so construct it manually | |
| HEAD_REPO=$(echo "$PR_DATA" | jq -r '"\(.headRepositoryOwner.login)/\(.headRepository.name)"') | |
| else | |
| # Use current branch | |
| PR_NUMBER="manual" | |
| HEAD_REF="$REF_NAME" | |
| HEAD_SHA="$SHA" | |
| HEAD_REPO="$REPO" | |
| fi | |
| else | |
| # pull_request event | |
| PR_NUMBER="$PR_NUMBER_EVENT" | |
| HEAD_REF="$PR_HEAD_REF_EVENT" | |
| HEAD_SHA="$PR_HEAD_SHA_EVENT" | |
| HEAD_REPO="$PR_HEAD_REPO_EVENT" | |
| fi | |
| BASE_REPO="$REPO" | |
| if [ "$HEAD_REPO" != "$BASE_REPO" ]; then | |
| IS_FORK_PR="true" | |
| else | |
| IS_FORK_PR="false" | |
| fi | |
| # Determine how recordings should land. | |
| # - PR-targeted runs (pull_request event, or dispatch with a pr_number) commit to the PR branch. | |
| # - Dispatch on a branch with no PR (e.g. main) opens a new PR with the refreshed recordings. | |
| # An explicit commit_mode input overrides the auto-selection. | |
| if [ -n "$INPUT_COMMIT_MODE" ]; then | |
| COMMIT_MODE="$INPUT_COMMIT_MODE" | |
| elif [ "$PR_NUMBER" = "manual" ]; then | |
| COMMIT_MODE="branch" | |
| else | |
| COMMIT_MODE="pr" | |
| fi | |
| # Determine which providers to run | |
| # Security: For pull_request, only run ollama variants (no secrets needed) | |
| # Manual workflow_dispatch uses input providers (defaults to API providers) | |
| if [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| PROVIDERS="$INPUT_PROVIDERS" | |
| else | |
| # Auto-trigger on PR: only ollama (no secrets exposed to any PR) | |
| PROVIDERS="ollama,ollama-reasoning" | |
| fi | |
| { | |
| echo "pr_number=${PR_NUMBER}" | |
| echo "pr_head_ref=${HEAD_REF}" | |
| echo "pr_head_sha=${HEAD_SHA}" | |
| echo "pr_head_repo=${HEAD_REPO}" | |
| echo "is_fork_pr=${IS_FORK_PR}" | |
| echo "providers_to_run=${PROVIDERS}" | |
| echo "commit_mode=${COMMIT_MODE}" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "Recording for PR #${PR_NUMBER}" | |
| echo " Branch: ${HEAD_REF}" | |
| echo " SHA: ${HEAD_SHA}" | |
| echo " Repo: ${HEAD_REPO}" | |
| echo " Fork PR: ${IS_FORK_PR}" | |
| echo " Providers: ${PROVIDERS}" | |
| echo " Commit mode: ${COMMIT_MODE}" | |
| # Upload PR metadata for companion workflow | |
| upload-pr-metadata: | |
| needs: compute-pr-info | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Create PR metadata artifact | |
| env: | |
| # Pass via env vars to prevent code injection (same pattern as compute-pr-info step) | |
| PR_NUMBER: ${{ needs.compute-pr-info.outputs.pr_number }} | |
| PR_HEAD_REF: ${{ needs.compute-pr-info.outputs.pr_head_ref }} | |
| PR_HEAD_SHA: ${{ needs.compute-pr-info.outputs.pr_head_sha }} | |
| PR_HEAD_REPO: ${{ needs.compute-pr-info.outputs.pr_head_repo }} | |
| IS_FORK_PR: ${{ needs.compute-pr-info.outputs.is_fork_pr }} | |
| COMMIT_MODE: ${{ needs.compute-pr-info.outputs.commit_mode }} | |
| run: | | |
| mkdir -p pr-metadata | |
| cat > pr-metadata/pr-info.json <<EOF | |
| { | |
| "pr_number": "${PR_NUMBER}", | |
| "pr_head_ref": "${PR_HEAD_REF}", | |
| "pr_head_sha": "${PR_HEAD_SHA}", | |
| "pr_head_repo": "${PR_HEAD_REPO}", | |
| "is_fork_pr": "${IS_FORK_PR}", | |
| "commit_mode": "${COMMIT_MODE}" | |
| } | |
| EOF | |
| cat pr-metadata/pr-info.json | |
| - name: Upload PR metadata | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pr-metadata-${{ github.run_id }} | |
| path: pr-metadata/ | |
| retention-days: 1 | |
| # Record tests for each provider | |
| record-providers: | |
| needs: compute-pr-info | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Expandable provider matrix - add new providers here with their configuration | |
| provider: | |
| - setup: ollama | |
| suite: base | |
| - setup: ollama-vision | |
| suite: vision | |
| - setup: ollama-reasoning | |
| suite: ollama-reasoning | |
| - setup: ollama | |
| suite: messages | |
| - setup: gpt | |
| suite: responses | |
| - setup: gpt | |
| suite: messages-openai | |
| - setup: azure | |
| suite: responses | |
| - setup: bedrock | |
| suite: bedrock | |
| - setup: bedrock | |
| suite: bedrock-responses | |
| - setup: watsonx | |
| suite: responses | |
| - setup: vertexai | |
| suite: responses | |
| - setup: gemini | |
| suite: interactions | |
| steps: | |
| - name: Check if provider should run | |
| id: should_run | |
| env: | |
| PROVIDERS_TO_RUN: ${{ needs.compute-pr-info.outputs.providers_to_run }} | |
| CURRENT_PROVIDER: ${{ matrix.provider.setup }} | |
| run: | | |
| # Trim whitespace and normalize to comma-separated list | |
| PROVIDERS=$(echo "$PROVIDERS_TO_RUN" | tr -d ' ') | |
| CURRENT="$CURRENT_PROVIDER" | |
| if [[ ",$PROVIDERS," == *",$CURRENT,"* ]]; then | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| echo "Recording for provider: $CURRENT" | |
| else | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| echo "Skipping provider: $CURRENT" | |
| fi | |
| - name: Checkout PR code | |
| if: steps.should_run.outputs.run == 'true' | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: ${{ needs.compute-pr-info.outputs.pr_head_repo }} | |
| ref: ${{ needs.compute-pr-info.outputs.pr_head_sha }} | |
| fetch-depth: 0 | |
| - name: Authenticate to Google Cloud (Vertex AI) | |
| if: steps.should_run.outputs.run == 'true' && matrix.provider.setup == 'vertexai' && needs.compute-pr-info.outputs.is_fork_pr != 'true' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| project_id: ${{ secrets.VERTEX_AI_PROJECT }} | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Setup test environment | |
| if: steps.should_run.outputs.run == 'true' | |
| uses: ogx-ai/ogx/.github/actions/setup-test-environment@e1ba4f9f10fa45750f5b4f186a0c4ae59bc93e4d | |
| with: | |
| python-version: "3.12" | |
| client-version: "latest" | |
| setup: ${{ matrix.provider.setup }} | |
| suite: ${{ inputs.suite || matrix.provider.suite }} | |
| inference-mode: 'record-if-missing' | |
| - name: Prepare AWS web identity for Bedrock | |
| if: steps.should_run.outputs.run == 'true' && matrix.provider.setup == 'bedrock' | |
| env: | |
| AWS_BEDROCK_ROLE_ARN: ${{ secrets.AWS_BEDROCK_ROLE_ARN }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -z "${AWS_BEDROCK_ROLE_ARN}" ]]; then | |
| echo "AWS_BEDROCK_ROLE_ARN is not configured; using Bedrock bearer token or default AWS credential chain." | |
| exit 0 | |
| fi | |
| token_file="${RUNNER_TEMP}/bedrock-web-identity-token" | |
| curl -fsSL \ | |
| -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ | |
| "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" \ | |
| | jq -r '.value' > "${token_file}" | |
| { | |
| echo "AWS_ROLE_ARN=${AWS_BEDROCK_ROLE_ARN}" | |
| echo "AWS_WEB_IDENTITY_TOKEN_FILE=${token_file}" | |
| echo "AWS_DEFAULT_REGION=us-west-2" | |
| } >> "${GITHUB_ENV}" | |
| - name: Run and record tests | |
| if: steps.should_run.outputs.run == 'true' | |
| uses: ogx-ai/ogx/.github/actions/run-and-record-tests@e1ba4f9f10fa45750f5b4f186a0c4ae59bc93e4d | |
| env: | |
| OPENAI_API_KEY: ${{ matrix.provider.setup == 'gpt' && secrets.OPENAI_API_KEY || '' }} | |
| AZURE_API_KEY: ${{ matrix.provider.setup == 'azure' && secrets.AZURE_API_KEY || '' }} | |
| AZURE_API_BASE: ${{ matrix.provider.setup == 'azure' && secrets.AZURE_API_BASE || '' }} | |
| WATSONX_API_KEY: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_API_KEY || '' }} | |
| WATSONX_BASE_URL: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_BASE_URL || '' }} | |
| WATSONX_PROJECT_ID: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_PROJECT_ID || '' }} | |
| VERTEX_AI_PROJECT: ${{ matrix.provider.setup == 'vertexai' && secrets.VERTEX_AI_PROJECT || '' }} | |
| VERTEX_AI_LOCATION: ${{ matrix.provider.setup == 'vertexai' && 'global' || '' }} | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ matrix.provider.setup == 'vertexai' && secrets.GOOGLE_APPLICATION_CREDENTIALS || '' }} | |
| GEMINI_API_KEY: ${{ matrix.provider.setup == 'gemini' && secrets.GEMINI_API_KEY || '' }} | |
| TAVILY_SEARCH_API_KEY: ${{ contains(fromJSON('["gpt","azure","vertexai"]'), matrix.provider.setup) && secrets.TAVILY_SEARCH_API_KEY || '' }} | |
| AWS_BEDROCK_BEARER_TOKEN: ${{ matrix.provider.setup == 'bedrock' && secrets.AWS_BEARER_TOKEN_BEDROCK || '' }} | |
| AWS_BEARER_TOKEN_BEDROCK: ${{ matrix.provider.setup == 'bedrock' && secrets.AWS_BEARER_TOKEN_BEDROCK || '' }} | |
| AWS_DEFAULT_REGION: ${{ matrix.provider.setup == 'bedrock' && 'us-west-2' || '' }} | |
| with: | |
| stack-config: 'server:ci-tests' | |
| setup: ${{ matrix.provider.setup }} | |
| inference-mode: 'record-if-missing' | |
| suite: ${{ inputs.suite || matrix.provider.suite }} | |
| subdirs: ${{ inputs.subdirs || '' }} | |
| pattern: ${{ inputs.pattern || '' }} | |
| # Don't commit here - upload as artifacts instead | |
| skip-commit: 'true' | |
| # Upload recordings as artifacts for the companion workflow to commit | |
| - name: Upload recordings as artifacts | |
| if: steps.should_run.outputs.run == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: recordings-${{ matrix.provider.setup }}-${{ matrix.provider.suite }}-${{ github.run_id }}-${{ github.run_attempt || '1' }} | |
| path: | | |
| tests/integration/recordings/ | |
| tests/integration/*/recordings/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| # Summary comment on PR | |
| comment-summary: | |
| needs: [compute-pr-info, record-providers] | |
| if: always() && needs.compute-pr-info.outputs.pr_number != 'manual' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Update PR comment | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| PROVIDERS_TO_RUN: ${{ needs.compute-pr-info.outputs.providers_to_run }} | |
| RECORD_STATUS: ${{ needs.record-providers.result }} | |
| PR_NUM: ${{ needs.compute-pr-info.outputs.pr_number }} | |
| IS_FORK_PR: ${{ needs.compute-pr-info.outputs.is_fork_pr }} | |
| with: | |
| script: | | |
| const COMMENT_MARKER = '<!-- record-integration-tests-bot -->'; | |
| const providers = process.env.PROVIDERS_TO_RUN.split(','); | |
| const status = process.env.RECORD_STATUS; | |
| const prNumber = parseInt(process.env.PR_NUM, 10); | |
| const isForkPr = process.env.IS_FORK_PR === 'true'; | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| if (isNaN(prNumber)) { | |
| core.warning('PR number is not valid, skipping comment'); | |
| return; | |
| } | |
| let message = `${COMMENT_MARKER}\n**Recording workflow ${status === 'success' ? 'completed' : 'finished with status: ' + status}**\n\nProviders: ${providers.join(', ')}\n\n`; | |
| if (status === 'success') { | |
| message += 'Recordings have been generated and will be committed automatically by the companion workflow.\n\n'; | |
| } else { | |
| message += 'Recording attempt finished. Check the workflow run for details.\n\n'; | |
| } | |
| message += `[View workflow run](${runUrl})`; | |
| if (isForkPr) { | |
| message += '\n\n**Fork PR**: Recordings will be committed if you have "Allow edits from maintainers" enabled.'; | |
| } | |
| try { | |
| // Find existing bot comment, paginating to avoid duplicates on busy PRs | |
| let existing = null; | |
| for await (const response of github.paginate.iterator(github.rest.issues.listComments, { | |
| issue_number: prNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| per_page: 100, | |
| })) { | |
| existing = response.data.find(c => c.body.includes(COMMENT_MARKER)); | |
| if (existing) break; | |
| } | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| comment_id: existing.id, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: message, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| issue_number: prNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: message, | |
| }); | |
| } | |
| } catch (error) { | |
| core.warning(`Could not post PR comment: ${error.message}`); | |
| } |