8 GPU Integration Test on H100 #105
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 8 GPU Integration Test on H100 | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths-ignore: | |
| - 'torchtitan/experiments/**' | |
| pull_request: | |
| types: [labeled, synchronize] | |
| branches: [ main ] | |
| paths-ignore: | |
| - 'torchtitan/experiments/**' | |
| schedule: | |
| # Runs every 6 hours | |
| - cron: '0 */6 * * *' | |
| concurrency: | |
| group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash -l -eo pipefail {0} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| # CUDA: run the suite on an H100 runner via linux_job_v3. | |
| set-matrix: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ciflow/h100.8') | |
| uses: ./.github/workflows/set-matrix.yaml | |
| with: | |
| runner-cuda: mt-l-bx86iamx-176-1800-h100-8 | |
| gpu-arch: cuda | |
| build-test: | |
| needs: set-matrix | |
| # Skip cleanly when the matrix is empty (e.g. CUDA on a label-triggered, | |
| # ROCm-only run); an empty matrix would otherwise error at evaluation. | |
| if: ${{ needs.set-matrix.outputs.matrix != '' && fromJSON(needs.set-matrix.outputs.matrix).include[0] != null }} | |
| uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.set-matrix.outputs.matrix) }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| gpu-arch-type: ${{ matrix.gpu-arch-type }} | |
| gpu-arch-version: ${{ matrix.gpu-arch-version }} | |
| # Private-ECR image, tagged with the .ci/docker hash from set-matrix.yaml. | |
| docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtitan/${{ matrix.docker-image }}:${{ needs.set-matrix.outputs.docker-hash }} | |
| repository: pytorch/torchtitan | |
| upload-artifact: outputs | |
| timeout: 45 | |
| script: | | |
| TORCH_VERSION="${{ matrix.torch-version }}" \ | |
| INDEX_URL="${{ matrix.index-url }}" \ | |
| GPU_ARCH_TYPE="${{ matrix.gpu-arch-type }}" \ | |
| bash .github/scripts/run_8xgpu_integration_tests.sh | |
| # ROCm: run the same suite on a ROCm runner via linux_job_v2. v2 is required | |
| # because linux_job_v3 does not authenticate ROCm runners to the private ECR | |
| # (the pull fails with "no basic auth credentials"); the bare image name lets | |
| # v2 resolve and authenticate against ECR. | |
| set-matrix-rocm: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ciflow/h100.8') | |
| uses: ./.github/workflows/set-matrix.yaml | |
| with: | |
| gpu-arch: rocm | |
| build-test-rocm: | |
| needs: set-matrix-rocm | |
| # Skip cleanly when the matrix is empty (e.g. ROCm on an ordinary | |
| # synchronize push); an empty matrix would otherwise error at evaluation. | |
| if: ${{ needs.set-matrix-rocm.outputs.matrix != '' && fromJSON(needs.set-matrix-rocm.outputs.matrix).include[0] != null }} | |
| uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.set-matrix-rocm.outputs.matrix) }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| gpu-arch-type: ${{ matrix.gpu-arch-type }} | |
| gpu-arch-version: ${{ matrix.gpu-arch-version }} | |
| docker-image: ${{ matrix.docker-image }} | |
| repository: pytorch/torchtitan | |
| upload-artifact: outputs | |
| timeout: 45 | |
| script: | | |
| TORCH_VERSION="${{ matrix.torch-version }}" \ | |
| INDEX_URL="${{ matrix.index-url }}" \ | |
| GPU_ARCH_TYPE="${{ matrix.gpu-arch-type }}" \ | |
| bash .github/scripts/run_8xgpu_integration_tests.sh |