eval.yml

name: eval

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to evaluate'
        required: false
        type: string
      submission_name:
        description: 'Name for this submission (e.g. "my_submission")'
        required: true
        type: string
      submission_url:
        description: 'Link to submission zip'
        required: true
        type: string
      runner:
        description: 'Runner to use'
        required: false
        type: choice
        default: 'ubuntu-latest'
        options:
          - 'ubuntu-latest'
          - 'linux-nvidia-t4'

jobs:
  test:
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 30
    env:
      UV_GROUP: ${{ inputs.runner == 'linux-nvidia-t4' && 'cu128' || 'cpu' }}
      EVAL_DEVICE: ${{ inputs.runner == 'linux-nvidia-t4' && 'cuda' || 'cpu' }}

    steps:
      - id: checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.pr_number && format('refs/pull/{0}/merge', inputs.pr_number) || 'master' }}

      - id: check_name
        name: Check submission_name is unique (except for "baseline" for testing)
        run: |
          git fetch origin master --depth=1
          if [ "${{ inputs.submission_name }}" != "baseline" ] && git ls-tree --name-only origin/master -- "submissions/${{ inputs.submission_name }}/" | grep -q .; then
            echo "::error::submission_name '${{ inputs.submission_name }}' already exists in master"
            exit 1
          fi

      - id: check_gpu
        name: Check GPU
        if: ${{ inputs.runner == 'linux-nvidia-t4' }}
        run: |
          nvidia-smi

      - id: download
        name: Download submission
        run: |
          mkdir -p ./submissions/${{ inputs.submission_name }}
          curl -L -o ./submissions/${{ inputs.submission_name }}/archive.zip "${{ inputs.submission_url }}"

      - id: install_lfs
        name: Install git-lfs
        run: |
          sudo apt-get update
          sudo apt-get install -y git-lfs
          git lfs install

      - id: pull_lfs
        name: Pull LFS files
        run: git lfs pull

      - id: install_uv
        name: Install uv
        uses: astral-sh/setup-uv@v4
        with:
          enable-cache: true

      - id: install_deps
        name: Install dependencies
        run: uv sync --group "$UV_GROUP"

      - id: install_ffmpeg
        name: Install ffmpeg
        run: sudo apt-get update && sudo apt-get install -y ffmpeg

      - id: evaluate
        name: Evaluate
        run: uv run --group "$UV_GROUP" bash evaluate.sh --device "$EVAL_DEVICE" --submission-dir ./submissions/${{ inputs.submission_name }}

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-${{ inputs.submission_name }}
          path: |
            ./submissions/${{ inputs.submission_name }}/archive.zip
            ./submissions/${{ inputs.submission_name }}/report.txt

  comment:
    needs: test
    if: always() && inputs.pr_number
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: eval-${{ inputs.submission_name }}
          path: ./artifacts
        continue-on-error: true

      - name: Comment on PR
        uses: actions/github-script@v7
        env:
          TEST_RESULT: ${{ needs.test.result }}
        with:
          script: |
            const fs = require('fs');
            const testResult = process.env.TEST_RESULT;
            let body;
            if (testResult === 'success') {
              const reportPath = './artifacts/report.txt';
              let report = 'No report.txt found.';
              if (fs.existsSync(reportPath)) {
                report = fs.readFileSync(reportPath, 'utf8');
              }
              body = `## Eval Results: \`${{ inputs.submission_name }}\`\n\n\`\`\`\n${report}\n\`\`\``;
            } else {
              let reason;
              if (testResult === 'cancelled') {
                reason = 'Job timed out or was cancelled';
              } else {
                reason = `Job failed`;
              }
              body = `## Eval Failed: \`${{ inputs.submission_name }}\`\n\n${reason}\n\n[View logs](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`;
            }
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: parseInt('${{ inputs.pr_number }}'),
              body
            });