eval #138
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: eval | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number to evaluate' | |
| required: false | |
| type: string | |
| submission_name: | |
| description: 'Name for this submission (e.g. "my_submission")' | |
| required: true | |
| type: string | |
| submission_url: | |
| description: 'Link to submission zip' | |
| required: true | |
| type: string | |
| runner: | |
| description: 'Runner to use' | |
| required: false | |
| type: choice | |
| default: 'ubuntu-latest' | |
| options: | |
| - 'ubuntu-latest' | |
| - 'linux-nvidia-t4' | |
| jobs: | |
| test: | |
| runs-on: ${{ inputs.runner }} | |
| timeout-minutes: 30 | |
| env: | |
| UV_GROUP: ${{ inputs.runner == 'linux-nvidia-t4' && 'cu128' || 'cpu' }} | |
| EVAL_DEVICE: ${{ inputs.runner == 'linux-nvidia-t4' && 'cuda' || 'cpu' }} | |
| steps: | |
| - id: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.pr_number && format('refs/pull/{0}/merge', inputs.pr_number) || 'master' }} | |
| - id: check_name | |
| name: Check submission_name is unique (except for "baseline" for testing) | |
| run: | | |
| git fetch origin master --depth=1 | |
| if [ "${{ inputs.submission_name }}" != "baseline" ] && git ls-tree --name-only origin/master -- "submissions/${{ inputs.submission_name }}/" | grep -q .; then | |
| echo "::error::submission_name '${{ inputs.submission_name }}' already exists in master" | |
| exit 1 | |
| fi | |
| - id: check_gpu | |
| name: Check GPU | |
| if: ${{ inputs.runner == 'linux-nvidia-t4' }} | |
| run: | | |
| nvidia-smi | |
| - id: download | |
| name: Download submission | |
| run: | | |
| mkdir -p ./submissions/${{ inputs.submission_name }} | |
| curl -L -o ./submissions/${{ inputs.submission_name }}/archive.zip "${{ inputs.submission_url }}" | |
| - id: install_lfs | |
| name: Install git-lfs | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y git-lfs | |
| git lfs install | |
| - id: pull_lfs | |
| name: Pull LFS files | |
| run: git lfs pull | |
| - id: install_uv | |
| name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| enable-cache: true | |
| - id: install_deps | |
| name: Install dependencies | |
| run: uv sync --group "$UV_GROUP" | |
| - id: install_ffmpeg | |
| name: Install ffmpeg | |
| run: sudo apt-get update && sudo apt-get install -y ffmpeg | |
| - id: evaluate | |
| name: Evaluate | |
| run: uv run --group "$UV_GROUP" bash evaluate.sh --device "$EVAL_DEVICE" --submission-dir ./submissions/${{ inputs.submission_name }} | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-${{ inputs.submission_name }} | |
| path: | | |
| ./submissions/${{ inputs.submission_name }}/archive.zip | |
| ./submissions/${{ inputs.submission_name }}/report.txt | |
| comment: | |
| needs: test | |
| if: always() && inputs.pr_number | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: eval-${{ inputs.submission_name }} | |
| path: ./artifacts | |
| continue-on-error: true | |
| - name: Comment on PR | |
| uses: actions/github-script@v7 | |
| env: | |
| TEST_RESULT: ${{ needs.test.result }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const testResult = process.env.TEST_RESULT; | |
| let body; | |
| if (testResult === 'success') { | |
| const reportPath = './artifacts/report.txt'; | |
| let report = 'No report.txt found.'; | |
| if (fs.existsSync(reportPath)) { | |
| report = fs.readFileSync(reportPath, 'utf8'); | |
| } | |
| body = `## Eval Results: \`${{ inputs.submission_name }}\`\n\n\`\`\`\n${report}\n\`\`\``; | |
| } else { | |
| let reason; | |
| if (testResult === 'cancelled') { | |
| reason = 'Job timed out or was cancelled'; | |
| } else { | |
| reason = `Job failed`; | |
| } | |
| body = `## Eval Failed: \`${{ inputs.submission_name }}\`\n\n${reason}\n\n[View logs](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: parseInt('${{ inputs.pr_number }}'), | |
| body | |
| }); |