launch-gpu-ec2-runner.yml

name: 'Launch GPU EC2 Runner'

run-name: GPU recording for gpt-oss:20b (${{ inputs.suite }} suite)

on:
  workflow_dispatch:
    inputs:
      suite:
        description: 'Test suite to run'
        required: false
        type: choice
        default: 'base'
        options:
          - base
          - responses
          - vllm-reasoning

concurrency:
  group: gpu-vllm-record-${{ github.run_id }}
  cancel-in-progress: false  # Don't cancel - EC2 cleanup is critical

# OIDC authentication for AWS - no long-lived credentials!
permissions:
  contents: read
  id-token: write  # Required for OIDC authentication to AWS

jobs:
  # Job 1: Launch GPU EC2 instance with multi-AZ fallback
  start-gpu-runner:
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Configure AWS credentials via OIDC
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: GitHubActions-vLLM-GPU-${{ github.run_id }}

      - name: Start EC2 runner
        id: start-ec2-runner
        uses: ./.github/actions/launch-gpu-runner
        with:
          mode: start
          github-token: ${{ secrets.RELEASE_PAT }}
          instance-type: g6.2xlarge
          aws-region: us-east-2
          availability-zones-config: |
            [
              {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2A }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"},
              {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2B }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"},
              {"imageId": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", "subnetId": "${{ vars.SUBNET_US_EAST_2C }}", "securityGroupId": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"}
            ]
          ec2-instance-tags: |
            [
              {"Key": "Name", "Value": "llamastack-vllm-gpu-runner"},
              {"Key": "Project", "Value": "llama-stack"},
              {"Key": "Purpose", "Value": "vllm-gpu-recording"},
              {"Key": "Model", "Value": "gpt-oss:20b"},
              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
              {"Key": "GitHubRef", "Value": "${{ github.ref }}"},
              {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"},
              {"Key": "ManagedBy", "Value": "GitHub-Actions"}
            ]

      - name: Runner launch summary
        run: |
          echo "GPU runner launched successfully"
          echo "  Instance ID: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}"
          echo "  Runner Label: ${{ steps.start-ec2-runner.outputs.label }}"
          echo "  Model: gpt-oss:20b"
          echo "  Instance Type: g6.2xlarge"

  # Job 2: Run vLLM tests on GPU runner
  record-vllm-tests:
    needs: start-gpu-runner
    runs-on: ${{ needs.start-gpu-runner.outputs.label }}
    permissions: {}  # CRITICAL: No permissions - prevents secret theft from untrusted code
    env:
      TMPDIR: /home/tmp
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup environment
        run: |
          mkdir -p /home/tmp
          echo "=== System Information ==="
          cat /etc/os-release
          echo ""
          echo "=== Disk Space ==="
          df -h
          echo ""
          echo "=== Memory ==="
          free -h
          echo ""
          echo "=== GPU Information ==="
          nvidia-smi

      - name: Setup vLLM GPU
        uses: ./.github/actions/setup-vllm-gpu
        with:
          model: 'gpt-oss:20b'
          port: '8000'
          gpu-memory-utilization: '0.85'
          max-model-len: '8192'
          quantization: 'none'

      - name: Setup test environment
        uses: ./.github/actions/setup-test-environment
        with:
          python-version: '3.12'
          client-version: 'latest'
          setup: 'vllm-gpu-gpt-oss'
          suite: ${{ inputs.suite }}
          inference-mode: 'record'

      - name: Run integration tests (record mode)
        uses: ./.github/actions/run-and-record-tests
        with:
          stack-config: 'server:ci-tests'
          setup: 'vllm-gpu-gpt-oss'
          inference-mode: 'record'
          suite: ${{ inputs.suite }}
          pattern: ${{ inputs.suite == 'base' && 'not (test_openai_completion_guided_choice or test_multiple_tools_with_different_schemas or test_mcp_invocation)' || '' }}
          skip-commit: 'true'  # Don't commit here - upload as artifacts

      - name: Upload recordings as artifacts
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: vllm-gpu-recordings-${{ github.run_id }}
          path: |
            tests/integration/recordings/
            tests/integration/*/recordings/
          retention-days: 7
          if-no-files-found: warn

      - name: Upload vLLM logs
        if: always()
        run: |
          if [ -f /tmp/vllm-server.log ]; then
            cat /tmp/vllm-server.log
          fi

      - name: Disk space after tests
        if: always()
        run: |
          echo "=== Disk Space After Tests ==="
          df -h

  # Job 3: Stop GPU EC2 instance (ALWAYS runs for cleanup)
  stop-gpu-runner:
    needs: [start-gpu-runner, record-vllm-tests]
    runs-on: ubuntu-latest
    if: ${{ always() }}  # CRITICAL: Always cleanup, even on failure or cancellation
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Configure AWS credentials via OIDC
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: GitHubActions-vLLM-GPU-Cleanup-${{ github.run_id }}

      - name: Stop EC2 runner
        uses: ./.github/actions/launch-gpu-runner
        with:
          mode: stop
          github-token: ${{ secrets.RELEASE_PAT }}
          aws-region: us-east-2
          label: ${{ needs.start-gpu-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-gpu-runner.outputs.instance-id }}

      - name: Cleanup summary
        run: |
          echo "GPU runner terminated successfully"
          echo "  Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}"

  # Job 4: Summary and next steps
  summary:
    needs: [start-gpu-runner, record-vllm-tests, stop-gpu-runner]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Workflow summary
        run: |
          {
            echo "## vLLM GPU Recording Summary"
            echo ""
            echo "**Model**: gpt-oss:20b"
            echo "**Instance Type**: g6.2xlarge"
            echo "**Test Suite**: ${{ inputs.suite }}"
            echo ""

            if [ "${{ needs.record-vllm-tests.result }}" == "success" ]; then
              echo "**Test Status**: Successful"
              echo ""
              echo "Recordings have been uploaded as artifacts. Download them from the workflow run and commit manually."
            else
              echo "**Test Status**: Failed"
              echo ""
              echo "Check the test logs for errors."
            fi

            echo ""
            echo "**Cleanup Status**: ${{ needs.stop-gpu-runner.result == 'success' && 'Instance terminated' || 'Check manually' }}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Check for cleanup issues
        if: needs.stop-gpu-runner.result != 'success'
        run: |
          echo "::warning::EC2 instance cleanup may have failed! Check AWS console for orphaned instances."
          echo "Instance ID: ${{ needs.start-gpu-runner.outputs.instance-id }}"