fix: keep a reference to the throttled cleanup background task (avoid GC before completion) #3277

Workflow file for this run

.github/workflows/record-integration-tests.yml at b4c8b0f

	# Auto-records missing integration test recordings when PRs are opened or updated.
	# Records for multiple providers (ollama, gpt, azure, etc.) and uploads recordings as artifacts.
	# A companion workflow (commit-recordings.yml) commits the artifacts back to the PR.
	# Expandable: add new providers by adding entries to the providers matrix.
	#
	# SECURITY NOTE: This workflow uses pull_request (not pull_request_target) for security.
	# Security measures:
	# 1. Runs with read-only permissions (no write access to repo or secrets exposure)
	# 2. Only ollama variants run automatically on pull_request (no secrets needed)
	# 3. API key providers (gpt, azure, bedrock) only run via manual workflow_dispatch
	# 4. Fork PRs with API key providers are blocked (prevents secret theft)
	# 5. Recordings uploaded as artifacts; companion workflow handles commits
	name: Integration Tests (Record)

	run-name: Auto-record missing test recordings for PR #${{ github.event.pull_request.number \|\| inputs.pr_number \|\| github.run_number }}

	on:
	pull_request:
	types: [opened, synchronize, reopened]
	paths:
	- 'tests/integration/**'
	- 'src/ogx/**'
	- '!src/ogx_ui/**'
	# Exclude recordings to prevent a feedback loop with commit-recordings.yml:
	# that workflow pushes recordings back to the PR branch, which would
	# otherwise re-trigger this workflow via pull_request: synchronize.
	- '!tests/integration//recordings/'
	workflow_dispatch:
	inputs:
	pr_number:
	description: 'PR number to record for (optional, uses current branch if not provided)'
	type: number
	required: false
	providers:
	description: 'Comma-separated list of providers to record'
	type: string
	required: false
	default: 'gpt,azure,watsonx,vertexai'
	suite:
	description: 'Test suite override (default: per-provider from matrix)'
	type: string
	required: false
	default: ''
	subdirs:
	description: 'Comma-separated list of test subdirectories to run; overrides suite'
	type: string
	required: false
	default: ''
	pattern:
	description: 'Regex pattern to pass to pytest -k'
	type: string
	required: false
	default: ''
	commit_mode:
	description: 'How to land recordings: pr (commit to PR branch), branch (open a new PR against the dispatched branch), none (artifacts only). Leave blank to auto-select.'
	type: string
	required: false
	default: ''

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| inputs.pr_number \|\| github.run_number }}
	cancel-in-progress: true

	# Read-only permissions - no write access
	# id-token: write is required for GCP Workload Identity Federation (OIDC token exchange)
	permissions:
	contents: read
	pull-requests: read
	id-token: write

	jobs:
	# Compute PR information for both pull_request and workflow_dispatch
	compute-pr-info:
	runs-on: ubuntu-latest
	outputs:
	pr_number: ${{ steps.compute.outputs.pr_number }}
	pr_head_ref: ${{ steps.compute.outputs.pr_head_ref }}
	pr_head_sha: ${{ steps.compute.outputs.pr_head_sha }}
	pr_head_repo: ${{ steps.compute.outputs.pr_head_repo }}
	is_fork_pr: ${{ steps.compute.outputs.is_fork_pr }}
	providers_to_run: ${{ steps.compute.outputs.providers_to_run }}
	commit_mode: ${{ steps.compute.outputs.commit_mode }}
	steps:
	- name: Compute PR metadata
	id: compute
	env:
	GH_TOKEN: ${{ github.token }}
	# Pass attacker-controllable values via env vars to prevent code injection.
	# Never interpolate these directly in shell with ${{ }} syntax.
	EVENT_NAME: ${{ github.event_name }}
	INPUT_PR_NUMBER: ${{ inputs.pr_number }}
	INPUT_PROVIDERS: ${{ inputs.providers }}
	INPUT_COMMIT_MODE: ${{ inputs.commit_mode }}
	REPO: ${{ github.repository }}
	REF_NAME: ${{ github.ref_name }}
	SHA: ${{ github.sha }}
	PR_NUMBER_EVENT: ${{ github.event.pull_request.number }}
	PR_HEAD_REF_EVENT: ${{ github.event.pull_request.head.ref }}
	PR_HEAD_SHA_EVENT: ${{ github.event.pull_request.head.sha }}
	PR_HEAD_REPO_EVENT: ${{ github.event.pull_request.head.repo.full_name }}
	run: \|
	if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
	if [ -n "$INPUT_PR_NUMBER" ]; then
	# Fetch PR info via API
	PR_DATA=$(gh pr view "$INPUT_PR_NUMBER" --repo "$REPO" --json number,headRefName,headRefOid,headRepository,headRepositoryOwner)
	PR_NUMBER="$INPUT_PR_NUMBER"
	HEAD_REF=$(echo "$PR_DATA" \| jq -r '.headRefName')
	HEAD_SHA=$(echo "$PR_DATA" \| jq -r '.headRefOid')
	# headRepository.nameWithOwner can be empty for fork PRs, so construct it manually
	HEAD_REPO=$(echo "$PR_DATA" \| jq -r '"\(.headRepositoryOwner.login)/\(.headRepository.name)"')
	else
	# Use current branch
	PR_NUMBER="manual"
	HEAD_REF="$REF_NAME"
	HEAD_SHA="$SHA"
	HEAD_REPO="$REPO"
	fi
	else
	# pull_request event
	PR_NUMBER="$PR_NUMBER_EVENT"
	HEAD_REF="$PR_HEAD_REF_EVENT"
	HEAD_SHA="$PR_HEAD_SHA_EVENT"
	HEAD_REPO="$PR_HEAD_REPO_EVENT"
	fi

	BASE_REPO="$REPO"
	if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
	IS_FORK_PR="true"
	else
	IS_FORK_PR="false"
	fi

	# Determine how recordings should land.
	# - PR-targeted runs (pull_request event, or dispatch with a pr_number) commit to the PR branch.
	# - Dispatch on a branch with no PR (e.g. main) opens a new PR with the refreshed recordings.
	# An explicit commit_mode input overrides the auto-selection.
	if [ -n "$INPUT_COMMIT_MODE" ]; then
	COMMIT_MODE="$INPUT_COMMIT_MODE"
	elif [ "$PR_NUMBER" = "manual" ]; then
	COMMIT_MODE="branch"
	else
	COMMIT_MODE="pr"
	fi

	# Determine which providers to run
	# Security: For pull_request, only run ollama variants (no secrets needed)
	# Manual workflow_dispatch uses input providers (defaults to API providers)
	if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
	PROVIDERS="$INPUT_PROVIDERS"
	else
	# Auto-trigger on PR: only ollama (no secrets exposed to any PR)
	PROVIDERS="ollama,ollama-reasoning"
	fi

	{
	echo "pr_number=${PR_NUMBER}"
	echo "pr_head_ref=${HEAD_REF}"
	echo "pr_head_sha=${HEAD_SHA}"
	echo "pr_head_repo=${HEAD_REPO}"
	echo "is_fork_pr=${IS_FORK_PR}"
	echo "providers_to_run=${PROVIDERS}"
	echo "commit_mode=${COMMIT_MODE}"
	} >> "$GITHUB_OUTPUT"

	echo "Recording for PR #${PR_NUMBER}"
	echo " Branch: ${HEAD_REF}"
	echo " SHA: ${HEAD_SHA}"
	echo " Repo: ${HEAD_REPO}"
	echo " Fork PR: ${IS_FORK_PR}"
	echo " Providers: ${PROVIDERS}"
	echo " Commit mode: ${COMMIT_MODE}"

	# Upload PR metadata for companion workflow
	upload-pr-metadata:
	needs: compute-pr-info
	runs-on: ubuntu-latest
	steps:
	- name: Create PR metadata artifact
	env:
	# Pass via env vars to prevent code injection (same pattern as compute-pr-info step)
	PR_NUMBER: ${{ needs.compute-pr-info.outputs.pr_number }}
	PR_HEAD_REF: ${{ needs.compute-pr-info.outputs.pr_head_ref }}
	PR_HEAD_SHA: ${{ needs.compute-pr-info.outputs.pr_head_sha }}
	PR_HEAD_REPO: ${{ needs.compute-pr-info.outputs.pr_head_repo }}
	IS_FORK_PR: ${{ needs.compute-pr-info.outputs.is_fork_pr }}
	COMMIT_MODE: ${{ needs.compute-pr-info.outputs.commit_mode }}
	run: \|
	mkdir -p pr-metadata
	cat > pr-metadata/pr-info.json <<EOF
	{
	"pr_number": "${PR_NUMBER}",
	"pr_head_ref": "${PR_HEAD_REF}",
	"pr_head_sha": "${PR_HEAD_SHA}",
	"pr_head_repo": "${PR_HEAD_REPO}",
	"is_fork_pr": "${IS_FORK_PR}",
	"commit_mode": "${COMMIT_MODE}"
	}
	EOF
	cat pr-metadata/pr-info.json

	- name: Upload PR metadata
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: pr-metadata-${{ github.run_id }}
	path: pr-metadata/
	retention-days: 1

	# Record tests for each provider
	record-providers:
	needs: compute-pr-info
	runs-on: ubuntu-latest
	timeout-minutes: 30
	strategy:
	fail-fast: false
	matrix:
	# Expandable provider matrix - add new providers here with their configuration
	provider:
	- setup: ollama
	suite: base
	- setup: ollama-vision
	suite: vision
	- setup: ollama-reasoning
	suite: ollama-reasoning
	- setup: ollama
	suite: messages
	- setup: gpt
	suite: responses
	- setup: gpt
	suite: messages-openai
	- setup: azure
	suite: responses
	- setup: bedrock
	suite: bedrock
	- setup: bedrock
	suite: bedrock-responses
	- setup: watsonx
	suite: responses
	- setup: vertexai
	suite: responses
	- setup: gemini
	suite: interactions
	steps:
	- name: Check if provider should run
	id: should_run
	env:
	PROVIDERS_TO_RUN: ${{ needs.compute-pr-info.outputs.providers_to_run }}
	CURRENT_PROVIDER: ${{ matrix.provider.setup }}
	run: \|
	# Trim whitespace and normalize to comma-separated list
	PROVIDERS=$(echo "$PROVIDERS_TO_RUN" \| tr -d ' ')
	CURRENT="$CURRENT_PROVIDER"

	if [[ ",$PROVIDERS," == ",$CURRENT," ]]; then
	echo "run=true" >> "$GITHUB_OUTPUT"
	echo "Recording for provider: $CURRENT"
	else
	echo "run=false" >> "$GITHUB_OUTPUT"
	echo "Skipping provider: $CURRENT"
	fi

	- name: Checkout PR code
	if: steps.should_run.outputs.run == 'true'
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: ${{ needs.compute-pr-info.outputs.pr_head_repo }}
	ref: ${{ needs.compute-pr-info.outputs.pr_head_sha }}
	fetch-depth: 0

	- name: Authenticate to Google Cloud (Vertex AI)
	if: steps.should_run.outputs.run == 'true' && matrix.provider.setup == 'vertexai' && needs.compute-pr-info.outputs.is_fork_pr != 'true'
	uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
	with:
	project_id: ${{ secrets.VERTEX_AI_PROJECT }}
	workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}

	- name: Setup test environment
	if: steps.should_run.outputs.run == 'true'
	uses: ogx-ai/ogx/.github/actions/setup-test-environment@e1ba4f9f10fa45750f5b4f186a0c4ae59bc93e4d
	with:
	python-version: "3.12"
	client-version: "latest"
	setup: ${{ matrix.provider.setup }}
	suite: ${{ inputs.suite \|\| matrix.provider.suite }}
	inference-mode: 'record-if-missing'

	- name: Prepare AWS web identity for Bedrock
	if: steps.should_run.outputs.run == 'true' && matrix.provider.setup == 'bedrock'
	env:
	AWS_BEDROCK_ROLE_ARN: ${{ secrets.AWS_BEDROCK_ROLE_ARN }}
	run: \|
	set -euo pipefail

	if [[ -z "${AWS_BEDROCK_ROLE_ARN}" ]]; then
	echo "AWS_BEDROCK_ROLE_ARN is not configured; using Bedrock bearer token or default AWS credential chain."
	exit 0
	fi

	token_file="${RUNNER_TEMP}/bedrock-web-identity-token"
	curl -fsSL \
	-H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
	"${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" \
	\| jq -r '.value' > "${token_file}"

	{
	echo "AWS_ROLE_ARN=${AWS_BEDROCK_ROLE_ARN}"
	echo "AWS_WEB_IDENTITY_TOKEN_FILE=${token_file}"
	echo "AWS_DEFAULT_REGION=us-west-2"
	} >> "${GITHUB_ENV}"

	- name: Run and record tests
	if: steps.should_run.outputs.run == 'true'
	uses: ogx-ai/ogx/.github/actions/run-and-record-tests@e1ba4f9f10fa45750f5b4f186a0c4ae59bc93e4d
	env:
	OPENAI_API_KEY: ${{ matrix.provider.setup == 'gpt' && secrets.OPENAI_API_KEY \|\| '' }}
	AZURE_API_KEY: ${{ matrix.provider.setup == 'azure' && secrets.AZURE_API_KEY \|\| '' }}
	AZURE_API_BASE: ${{ matrix.provider.setup == 'azure' && secrets.AZURE_API_BASE \|\| '' }}
	WATSONX_API_KEY: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_API_KEY \|\| '' }}
	WATSONX_BASE_URL: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_BASE_URL \|\| '' }}
	WATSONX_PROJECT_ID: ${{ matrix.provider.setup == 'watsonx' && secrets.WATSONX_PROJECT_ID \|\| '' }}
	VERTEX_AI_PROJECT: ${{ matrix.provider.setup == 'vertexai' && secrets.VERTEX_AI_PROJECT \|\| '' }}
	VERTEX_AI_LOCATION: ${{ matrix.provider.setup == 'vertexai' && 'global' \|\| '' }}
	GOOGLE_APPLICATION_CREDENTIALS: ${{ matrix.provider.setup == 'vertexai' && secrets.GOOGLE_APPLICATION_CREDENTIALS \|\| '' }}
	GEMINI_API_KEY: ${{ matrix.provider.setup == 'gemini' && secrets.GEMINI_API_KEY \|\| '' }}
	TAVILY_SEARCH_API_KEY: ${{ contains(fromJSON('["gpt","azure","vertexai"]'), matrix.provider.setup) && secrets.TAVILY_SEARCH_API_KEY \|\| '' }}
	AWS_BEDROCK_BEARER_TOKEN: ${{ matrix.provider.setup == 'bedrock' && secrets.AWS_BEARER_TOKEN_BEDROCK \|\| '' }}
	AWS_BEARER_TOKEN_BEDROCK: ${{ matrix.provider.setup == 'bedrock' && secrets.AWS_BEARER_TOKEN_BEDROCK \|\| '' }}
	AWS_DEFAULT_REGION: ${{ matrix.provider.setup == 'bedrock' && 'us-west-2' \|\| '' }}
	with:
	stack-config: 'server:ci-tests'
	setup: ${{ matrix.provider.setup }}
	inference-mode: 'record-if-missing'
	suite: ${{ inputs.suite \|\| matrix.provider.suite }}
	subdirs: ${{ inputs.subdirs \|\| '' }}
	pattern: ${{ inputs.pattern \|\| '' }}
	# Don't commit here - upload as artifacts instead
	skip-commit: 'true'

	# Upload recordings as artifacts for the companion workflow to commit
	- name: Upload recordings as artifacts
	if: steps.should_run.outputs.run == 'true'
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: recordings-${{ matrix.provider.setup }}-${{ matrix.provider.suite }}-${{ github.run_id }}-${{ github.run_attempt \|\| '1' }}
	path: \|
	tests/integration/recordings/
	tests/integration/*/recordings/
	retention-days: 7
	if-no-files-found: ignore

	# Summary comment on PR
	comment-summary:
	needs: [compute-pr-info, record-providers]
	if: always() && needs.compute-pr-info.outputs.pr_number != 'manual'
	runs-on: ubuntu-latest
	permissions:
	pull-requests: write
	steps:
	- name: Update PR comment
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	env:
	PROVIDERS_TO_RUN: ${{ needs.compute-pr-info.outputs.providers_to_run }}
	RECORD_STATUS: ${{ needs.record-providers.result }}
	PR_NUM: ${{ needs.compute-pr-info.outputs.pr_number }}
	IS_FORK_PR: ${{ needs.compute-pr-info.outputs.is_fork_pr }}
	with:
	script: \|
	const COMMENT_MARKER = '<!-- record-integration-tests-bot -->';
	const providers = process.env.PROVIDERS_TO_RUN.split(',');
	const status = process.env.RECORD_STATUS;
	const prNumber = parseInt(process.env.PR_NUM, 10);
	const isForkPr = process.env.IS_FORK_PR === 'true';
	const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

	if (isNaN(prNumber)) {
	core.warning('PR number is not valid, skipping comment');
	return;
	}

	let message = `${COMMENT_MARKER}\nRecording workflow ${status === 'success' ? 'completed' : 'finished with status: ' + status}\n\nProviders: ${providers.join(', ')}\n\n`;

	if (status === 'success') {
	message += 'Recordings have been generated and will be committed automatically by the companion workflow.\n\n';
	} else {
	message += 'Recording attempt finished. Check the workflow run for details.\n\n';
	}

	message += `[View workflow run](${runUrl})`;

	if (isForkPr) {
	message += '\n\nFork PR: Recordings will be committed if you have "Allow edits from maintainers" enabled.';
	}

	try {
	// Find existing bot comment, paginating to avoid duplicates on busy PRs
	let existing = null;
	for await (const response of github.paginate.iterator(github.rest.issues.listComments, {
	issue_number: prNumber,
	owner: context.repo.owner,
	repo: context.repo.repo,
	per_page: 100,
	})) {
	existing = response.data.find(c => c.body.includes(COMMENT_MARKER));
	if (existing) break;
	}

	if (existing) {
	await github.rest.issues.updateComment({
	comment_id: existing.id,
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: message,
	});
	} else {
	await github.rest.issues.createComment({
	issue_number: prNumber,
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: message,
	});
	}
	} catch (error) {
	core.warning(`Could not post PR comment: ${error.message}`);
	}

Navigation Menu

Uh oh!

fix: keep a reference to the throttled cleanup background task (avoid GC before completion) #3277

Workflow file

fix: keep a reference to the throttled cleanup background task (avoid GC before completion) #3277

Uh oh!

Workflow file for this run