Skip to content

Commit b8c8c6f

Browse files
skamenan7skamenan7
committed
fix(ci): keep GPU recording workflow dispatchable
Signed-off-by: Sumanth Kamenani <skamenan@redhat.com>
1 parent b0e3247 commit b8c8c6f

2 files changed

Lines changed: 334 additions & 0 deletions

File tree

‎.github/workflows/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ OGX uses GitHub Actions for Continuous Integration (CI). Below is a table detail
2121
| Messages API - Claude Code Client Smoke Tests | [integration-tests-messages-clients.yml](integration-tests-messages-clients.yml) | Drive the Claude Code CLI and Agent SDK against /v1/messages (live, Ollama) |
2222
| Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
2323
| Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
24+
| Launch GPU EC2 Runner | [launch-gpu-ec2-runner.yml](launch-gpu-ec2-runner.yml) | GPU recording for gpt-oss:20b (${{ inputs.suite }} suite) |
2425
| OpenAPI Generator SDK Validation | [openapi-generator-validation.yml](openapi-generator-validation.yml) | Validate OpenAPI Generator SDK generation |
2526
| OpenResponses Conformance Tests | [openresponses-conformance.yml](openresponses-conformance.yml) | Run OpenResponses conformance tests against ogx Responses API |
2627
| Post-release automation | [post-release.yml](post-release.yml) | Post-release automation |

‎.github/workflows/launch-gpu-ec2-runner.yml‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
name: 'Launch GPU EC2 Runner'
2+
3+
run-name: GPU recording for gpt-oss:20b (${{ inputs.suite }} suite)
4+
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
suite:
9+
description: 'Test suite to run'
10+
required: false
11+
type: choice
12+
default: 'base'
13+
options:
14+
- base
15+
- responses
16+
- vllm-reasoning
17+
18+
concurrency:
19+
group: gpu-vllm-record-${{ github.run_id }}
20+
cancel-in-progress: false # Don't cancel - EC2 cleanup is critical
21+
22+
# OIDC authentication for AWS - no long-lived credentials!
23+
permissions:
24+
contents: read
25+
actions: read # Required for the hosted cleanup job to watch the GPU job state
26+
id-token: write # Required for OIDC authentication to AWS
27+
28+
jobs:
29+
# Job 1: Launch GPU EC2 instance with multi-AZ fallback
30+
start-gpu-runner:
31+
runs-on: ubuntu-latest
32+
outputs:
33+
label: ${{ steps.start-ec2-runner.outputs.label }}
34+
instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
35+
steps:
36+
- name: Checkout code
37+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
38+
with:
39+
persist-credentials: false
40+
41+
- name: Validate GPU runner configuration
42+
env:
43+
AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
44+
RELEASE_PAT: ${{ secrets.RELEASE_PAT }}
45+
AWS_EC2_AMI_US_EAST_2: ${{ vars.AWS_EC2_AMI_US_EAST_2 }}
46+
SUBNET_US_EAST_2A: ${{ vars.SUBNET_US_EAST_2A }}
47+
SUBNET_US_EAST_2B: ${{ vars.SUBNET_US_EAST_2B }}
48+
SUBNET_US_EAST_2C: ${{ vars.SUBNET_US_EAST_2C }}
49+
SECURITY_GROUP_ID_US_EAST_2: ${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}
50+
run: |
51+
missing=()
52+
for name in \
53+
AWS_ROLE_ARN \
54+
RELEASE_PAT \
55+
AWS_EC2_AMI_US_EAST_2 \
56+
SUBNET_US_EAST_2A \
57+
SUBNET_US_EAST_2B \
58+
SUBNET_US_EAST_2C \
59+
SECURITY_GROUP_ID_US_EAST_2; do
60+
if [ -z "${!name}" ]; then
61+
missing+=("$name")
62+
fi
63+
done
64+
65+
if [ "${#missing[@]}" -gt 0 ]; then
66+
printf 'Missing required GPU runner configuration: %s\n' "${missing[*]}"
67+
exit 1
68+
fi
69+
70+
- name: Configure AWS credentials via OIDC
71+
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
72+
with:
73+
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
74+
aws-region: us-east-2
75+
role-session-name: GitHubActions-vLLM-GPU-${{ github.run_id }}
76+
77+
- name: Start EC2 runner
78+
id: start-ec2-runner
79+
uses: ./.github/actions/launch-gpu-runner
80+
with:
81+
mode: start
82+
github-token: ${{ secrets.RELEASE_PAT }}
83+
instance-type: g6.2xlarge
84+
aws-region: us-east-2
85+
availability-zones-config: |
86+
[
87+
{"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2A }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"},
88+
{"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2B }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"},
89+
{"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2C }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"}
90+
]
91+
ec2-instance-tags: |
92+
[
93+
{"Key": "Name", "Value": "llamastack-vllm-gpu-runner"},
94+
{"Key": "Project", "Value": "llama-stack"},
95+
{"Key": "Purpose", "Value": "vllm-gpu-recording"},
96+
{"Key": "Model", "Value": "gpt-oss:20b"},
97+
{"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
98+
{"Key": "GitHubRef", "Value": "${{ github.ref }}"},
99+
{"Key": "GitHubRunId", "Value": "${{ github.run_id }}"},
100+
{"Key": "ManagedBy", "Value": "GitHub-Actions"}
101+
]
102+
103+
- name: Runner launch summary
104+
run: |
105+
if [ -z "${{ steps.start-ec2-runner.outputs.ec2-instance-id }}" ] || [ -z "${{ steps.start-ec2-runner.outputs.label }}" ]; then
106+
echo "Failed to launch GPU runner with a usable instance id and runner label"
107+
exit 1
108+
fi
109+
echo "GPU runner launched successfully"
110+
echo " Instance ID: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}"
111+
echo " Runner Label: ${{ steps.start-ec2-runner.outputs.label }}"
112+
echo " Model: gpt-oss:20b"
113+
echo " Instance Type: g6.2xlarge"
114+
115+
# Job 2: Run vLLM tests on GPU runner
116+
record-vllm-tests:
117+
needs: start-gpu-runner
118+
runs-on: ${{ needs.start-gpu-runner.outputs.label }}
119+
permissions: {} # CRITICAL: No permissions - prevents secret theft from untrusted code
120+
env:
121+
TMPDIR: /home/tmp
122+
steps:
123+
- name: Checkout code
124+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
125+
with:
126+
persist-credentials: false
127+
128+
- name: Setup environment
129+
run: |
130+
mkdir -p /home/tmp
131+
echo "=== System Information ==="
132+
cat /etc/os-release
133+
echo ""
134+
echo "=== Disk Space ==="
135+
df -h
136+
echo ""
137+
echo "=== Memory ==="
138+
free -h
139+
echo ""
140+
echo "=== GPU Information ==="
141+
nvidia-smi
142+
143+
- name: Setup vLLM GPU
144+
uses: ./.github/actions/setup-vllm-gpu
145+
with:
146+
model: 'gpt-oss:20b'
147+
port: '8000'
148+
gpu-memory-utilization: '0.85'
149+
max-model-len: '8192'
150+
quantization: 'none'
151+
vllm-version: '0.22.1'
152+
153+
- name: Setup test environment
154+
uses: ./.github/actions/setup-test-environment
155+
with:
156+
python-version: '3.12'
157+
client-version: 'latest'
158+
setup: 'vllm-gpu-gpt-oss'
159+
suite: ${{ inputs.suite }}
160+
inference-mode: 'record'
161+
enable-hf-cache: 'false'
162+
163+
- name: Select vLLM-compatible pytest pattern
164+
id: test-pattern
165+
shell: bash
166+
run: |
167+
case "${{ inputs.suite }}" in
168+
base)
169+
pattern="not (test_openai_completion_guided_choice or test_multiple_tools_with_different_schemas or test_mcp_invocation)"
170+
;;
171+
responses)
172+
pattern="not (file_search or mcp or tool or compact or conversation or test_include_logprobs_with_web_search or test_openai_response_with_small_max_output_tokens or test_openai_response_with_max_output_tokens or test_openai_response_streaming_invalid_base64_image_failure_code_is_spec_compliant or test_openai_response_with_parallel_tool_calls_enabled or test_openai_response_background_completes)"
173+
;;
174+
*)
175+
pattern=""
176+
;;
177+
esac
178+
echo "pattern=${pattern}" >> "$GITHUB_OUTPUT"
179+
180+
- name: Run integration tests (record mode)
181+
uses: ./.github/actions/run-and-record-tests
182+
with:
183+
stack-config: 'server:ci-tests'
184+
setup: 'vllm-gpu-gpt-oss'
185+
inference-mode: 'record'
186+
suite: ${{ inputs.suite }}
187+
pattern: ${{ steps.test-pattern.outputs.pattern }}
188+
skip-commit: 'true' # Don't commit here - upload as artifacts
189+
190+
- name: Upload recordings as artifacts
191+
if: always()
192+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
193+
with:
194+
name: vllm-gpu-recordings-${{ github.run_id }}
195+
path: |
196+
tests/integration/recordings/
197+
tests/integration/*/recordings/
198+
retention-days: 7
199+
if-no-files-found: error
200+
201+
- name: Upload vLLM logs
202+
if: always()
203+
run: |
204+
if [ -f /tmp/vllm-server.log ]; then
205+
cat /tmp/vllm-server.log
206+
fi
207+
208+
- name: Disk space after tests
209+
if: always()
210+
run: |
211+
echo "=== Disk Space After Tests ==="
212+
df -h
213+
214+
# Job 3: Stop GPU EC2 instance (ALWAYS runs for cleanup)
215+
stop-gpu-runner:
216+
needs: start-gpu-runner
217+
runs-on: ubuntu-latest
218+
if: ${{ always() }} # CRITICAL: Always cleanup, even on failure or cancellation
219+
steps:
220+
- name: Checkout code
221+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
222+
with:
223+
persist-credentials: false
224+
225+
- name: Configure AWS credentials via OIDC
226+
if: needs.start-gpu-runner.outputs.instance-id != ''
227+
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
228+
with:
229+
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
230+
aws-region: us-east-2
231+
role-session-name: GitHubActions-vLLM-GPU-Cleanup-${{ github.run_id }}
232+
233+
- name: Wait for GPU test job before cleanup
234+
if: needs.start-gpu-runner.outputs.instance-id != ''
235+
env:
236+
GH_TOKEN: ${{ github.token }}
237+
RECORD_JOB_NAME: record-vllm-tests
238+
RUN_ID: ${{ github.run_id }}
239+
REPOSITORY: ${{ github.repository }}
240+
run: |
241+
queued_deadline=$((SECONDS + 1200))
242+
absolute_deadline=$((SECONDS + 7200))
243+
244+
while true; do
245+
job_status="$(
246+
gh api "repos/${REPOSITORY}/actions/runs/${RUN_ID}/jobs" --paginate \
247+
--jq ".jobs[] | select(.name == \"${RECORD_JOB_NAME}\") | [.status, (.conclusion // \"\")] | @tsv" \
248+
| tail -n 1
249+
)"
250+
251+
status="$(printf '%s' "$job_status" | cut -f1)"
252+
conclusion="$(printf '%s' "$job_status" | cut -f2)"
253+
254+
if [ "$status" = "completed" ]; then
255+
echo "GPU test job completed with conclusion: ${conclusion:-unknown}"
256+
break
257+
fi
258+
259+
if [ -z "$status" ]; then
260+
echo "Waiting for GPU test job to be created..."
261+
else
262+
echo "GPU test job status: $status"
263+
fi
264+
265+
if [ "$status" != "in_progress" ] && [ "$SECONDS" -ge "$queued_deadline" ]; then
266+
echo "GPU test job did not start within 20 minutes; cleaning up the EC2 runner."
267+
break
268+
fi
269+
270+
if [ "$SECONDS" -ge "$absolute_deadline" ]; then
271+
echo "GPU test job exceeded the 2 hour cleanup deadline; cleaning up the EC2 runner."
272+
break
273+
fi
274+
275+
sleep 30
276+
done
277+
278+
- name: Stop EC2 runner
279+
if: needs.start-gpu-runner.outputs.instance-id != ''
280+
uses: ./.github/actions/launch-gpu-runner
281+
with:
282+
mode: stop
283+
github-token: ${{ secrets.RELEASE_PAT }}
284+
aws-region: us-east-2
285+
label: ${{ needs.start-gpu-runner.outputs.label }}
286+
ec2-instance-id: ${{ needs.start-gpu-runner.outputs.instance-id }}
287+
288+
- name: Cleanup summary
289+
if: needs.start-gpu-runner.outputs.instance-id != ''
290+
run: |
291+
echo "GPU runner terminated successfully"
292+
echo " Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}"
293+
294+
- name: Cleanup skipped
295+
if: needs.start-gpu-runner.outputs.instance-id == ''
296+
run: |
297+
echo "No EC2 instance id was produced by start-gpu-runner; nothing to terminate."
298+
299+
# Job 4: Summary and next steps
300+
summary:
301+
needs: [start-gpu-runner, record-vllm-tests, stop-gpu-runner]
302+
runs-on: ubuntu-latest
303+
if: always()
304+
steps:
305+
- name: Workflow summary
306+
run: |
307+
{
308+
echo "## vLLM GPU Recording Summary"
309+
echo ""
310+
echo "**Model**: gpt-oss:20b"
311+
echo "**Instance Type**: g6.2xlarge"
312+
echo "**Test Suite**: ${{ inputs.suite }}"
313+
echo ""
314+
315+
if [ "${{ needs.record-vllm-tests.result }}" == "success" ]; then
316+
echo "**Test Status**: Successful"
317+
echo ""
318+
echo "Recordings have been uploaded as artifacts. Download them from the workflow run and commit manually."
319+
else
320+
echo "**Test Status**: Failed"
321+
echo ""
322+
echo "Check the test logs for errors."
323+
fi
324+
325+
echo ""
326+
echo "**Cleanup Status**: ${{ needs.stop-gpu-runner.result == 'success' && 'Instance terminated' || 'Check manually' }}"
327+
} >> "$GITHUB_STEP_SUMMARY"
328+
329+
- name: Check for cleanup issues
330+
if: needs.stop-gpu-runner.result != 'success'
331+
run: |
332+
echo "::warning::EC2 instance cleanup may have failed! Check AWS console for orphaned instances."
333+
echo "Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}"

0 commit comments

Comments
 (0)