GPU recording for gpt-oss:20b (base suite) #7
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Launch GPU EC2 Runner' | |
| run-name: GPU recording for gpt-oss:20b (${{ inputs.suite }} suite) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| suite: | |
| description: 'Test suite to run' | |
| required: false | |
| type: choice | |
| default: 'base' | |
| options: | |
| - base | |
| - responses | |
| - vllm-reasoning | |
| concurrency: | |
| group: gpu-vllm-record-${{ github.run_id }} | |
| cancel-in-progress: false # Don't cancel - EC2 cleanup is critical | |
| # OIDC authentication for AWS - no long-lived credentials! | |
| permissions: | |
| contents: read | |
| id-token: write # Required for OIDC authentication to AWS | |
| jobs: | |
| # Job 1: Launch GPU EC2 instance with multi-AZ fallback | |
| start-gpu-runner: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Configure AWS credentials via OIDC | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-2 | |
| role-session-name: GitHubActions-vLLM-GPU-${{ github.run_id }} | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/launch-gpu-runner | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.RELEASE_PAT }} | |
| instance-type: g6.2xlarge | |
| aws-region: us-east-2 | |
| availability-zones-config: | | |
| [ | |
| {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2A }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"}, | |
| {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2B }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"}, | |
| {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2C }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"} | |
| ] | |
| ec2-instance-tags: | | |
| [ | |
| {"Key": "Name", "Value": "llamastack-vllm-gpu-runner"}, | |
| {"Key": "Project", "Value": "llama-stack"}, | |
| {"Key": "Purpose", "Value": "vllm-gpu-recording"}, | |
| {"Key": "Model", "Value": "gpt-oss:20b"}, | |
| {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, | |
| {"Key": "GitHubRef", "Value": "${{ github.ref }}"}, | |
| {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"}, | |
| {"Key": "ManagedBy", "Value": "GitHub-Actions"} | |
| ] | |
| - name: Runner launch summary | |
| run: | | |
| echo "GPU runner launched successfully" | |
| echo " Instance ID: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}" | |
| echo " Runner Label: ${{ steps.start-ec2-runner.outputs.label }}" | |
| echo " Model: gpt-oss:20b" | |
| echo " Instance Type: g6.2xlarge" | |
| # Job 2: Run vLLM tests on GPU runner | |
| record-vllm-tests: | |
| needs: start-gpu-runner | |
| runs-on: ${{ needs.start-gpu-runner.outputs.label }} | |
| permissions: {} # CRITICAL: No permissions - prevents secret theft from untrusted code | |
| env: | |
| TMPDIR: /home/tmp | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup environment | |
| run: | | |
| mkdir -p /home/tmp | |
| echo "=== System Information ===" | |
| cat /etc/os-release | |
| echo "" | |
| echo "=== Disk Space ===" | |
| df -h | |
| echo "" | |
| echo "=== Memory ===" | |
| free -h | |
| echo "" | |
| echo "=== GPU Information ===" | |
| nvidia-smi | |
| - name: Setup vLLM GPU | |
| uses: ./.github/actions/setup-vllm-gpu | |
| with: | |
| model: 'gpt-oss:20b' | |
| port: '8000' | |
| gpu-memory-utilization: '0.85' | |
| max-model-len: '8192' | |
| quantization: 'none' | |
| - name: Setup test environment | |
| uses: ./.github/actions/setup-test-environment | |
| with: | |
| python-version: '3.12' | |
| client-version: 'latest' | |
| setup: 'vllm-gpu-gpt-oss' | |
| suite: ${{ inputs.suite }} | |
| inference-mode: 'record' | |
| - name: Run integration tests (record mode) | |
| uses: ./.github/actions/run-and-record-tests | |
| with: | |
| stack-config: 'server:ci-tests' | |
| setup: 'vllm-gpu-gpt-oss' | |
| inference-mode: 'record' | |
| suite: ${{ inputs.suite }} | |
| skip-commit: 'true' # Don't commit here - upload as artifacts | |
| - name: Upload recordings as artifacts | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: vllm-gpu-recordings-${{ github.run_id }} | |
| path: | | |
| tests/integration/recordings/ | |
| tests/integration/*/recordings/ | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: Upload vLLM logs | |
| if: always() | |
| run: | | |
| if [ -f /tmp/vllm-server.log ]; then | |
| cat /tmp/vllm-server.log | |
| fi | |
| - name: Disk space after tests | |
| if: always() | |
| run: | | |
| echo "=== Disk Space After Tests ===" | |
| df -h | |
| # Job 3: Stop GPU EC2 instance (ALWAYS runs for cleanup) | |
| stop-gpu-runner: | |
| needs: [start-gpu-runner, record-vllm-tests] | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} # CRITICAL: Always cleanup, even on failure or cancellation | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Configure AWS credentials via OIDC | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: us-east-2 | |
| role-session-name: GitHubActions-vLLM-GPU-Cleanup-${{ github.run_id }} | |
| - name: Stop EC2 runner | |
| uses: ./.github/actions/launch-gpu-runner | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.RELEASE_PAT }} | |
| aws-region: us-east-2 | |
| label: ${{ needs.start-gpu-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-gpu-runner.outputs.instance-id }} | |
| - name: Cleanup summary | |
| run: | | |
| echo "GPU runner terminated successfully" | |
| echo " Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}" | |
| # Job 4: Summary and next steps | |
| summary: | |
| needs: [start-gpu-runner, record-vllm-tests, stop-gpu-runner] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Workflow summary | |
| run: | | |
| { | |
| echo "## vLLM GPU Recording Summary" | |
| echo "" | |
| echo "**Model**: gpt-oss:20b" | |
| echo "**Instance Type**: g6.2xlarge" | |
| echo "**Test Suite**: ${{ inputs.suite }}" | |
| echo "" | |
| if [ "${{ needs.record-vllm-tests.result }}" == "success" ]; then | |
| echo "**Test Status**: Successful" | |
| echo "" | |
| echo "Recordings have been uploaded as artifacts. Download them from the workflow run and commit manually." | |
| else | |
| echo "**Test Status**: Failed" | |
| echo "" | |
| echo "Check the test logs for errors." | |
| fi | |
| echo "" | |
| echo "**Cleanup Status**: ${{ needs.stop-gpu-runner.result == 'success' && 'Instance terminated' || 'Check manually' }}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Check for cleanup issues | |
| if: needs.stop-gpu-runner.result != 'success' | |
| run: | | |
| echo "::warning::EC2 instance cleanup may have failed! Check AWS console for orphaned instances." | |
| echo "Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}" |