torchtitan-neuron/.github/workflows/integration_test_8gpu_h100.yaml at 2f1ca1d6b18212427419e4dd0987f80a642fa2bf · aws-neuron/torchtitan-neuron · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
name: 8 GPU Integration Test on H100

on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'torchtitan/experiments/**'
  pull_request:
    types: [labeled, synchronize]
    branches: [ main ]
    paths-ignore:
      - 'torchtitan/experiments/**'
  schedule:
    # Runs every 6 hours
    - cron: '0 */6 * * *'

concurrency:
  group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash -l -eo pipefail {0}

permissions:
  id-token: write
  contents: read

jobs:
  # CUDA: run the suite on an H100 runner via linux_job_v3.
  set-matrix:
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ciflow/h100.8')
    uses: ./.github/workflows/set-matrix.yaml
    with:
      runner-cuda: mt-l-bx86iamx-176-1800-h100-8
      gpu-arch: cuda

  build-test:
    needs: set-matrix
    # Skip cleanly when the matrix is empty (e.g. CUDA on a label-triggered,
    # ROCm-only run); an empty matrix would otherwise error at evaluation.
    if: ${{ needs.set-matrix.outputs.matrix != '' && fromJSON(needs.set-matrix.outputs.matrix).include[0] != null }}
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.set-matrix.outputs.matrix) }}
    with:
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      # Private-ECR image, tagged with the .ci/docker hash from set-matrix.yaml.
      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtitan/${{ matrix.docker-image }}:${{ needs.set-matrix.outputs.docker-hash }}
      repository: pytorch/torchtitan
      upload-artifact: outputs
      timeout: 45
      script: |
        TORCH_VERSION="${{ matrix.torch-version }}" \
        INDEX_URL="${{ matrix.index-url }}" \
        GPU_ARCH_TYPE="${{ matrix.gpu-arch-type }}" \
        bash .github/scripts/run_8xgpu_integration_tests.sh

  # ROCm: run the same suite on a ROCm runner via linux_job_v2. v2 is required
  # because linux_job_v3 does not authenticate ROCm runners to the private ECR
  # (the pull fails with "no basic auth credentials"); the bare image name lets
  # v2 resolve and authenticate against ECR.
  set-matrix-rocm:
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ciflow/h100.8')
    uses: ./.github/workflows/set-matrix.yaml
    with:
      gpu-arch: rocm

  build-test-rocm:
    needs: set-matrix-rocm
    # Skip cleanly when the matrix is empty (e.g. ROCm on an ordinary
    # synchronize push); an empty matrix would otherwise error at evaluation.
    if: ${{ needs.set-matrix-rocm.outputs.matrix != '' && fromJSON(needs.set-matrix-rocm.outputs.matrix).include[0] != null }}
    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.set-matrix-rocm.outputs.matrix) }}
    with:
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      docker-image: ${{ matrix.docker-image }}
      repository: pytorch/torchtitan
      upload-artifact: outputs
      timeout: 45
      script: |
        TORCH_VERSION="${{ matrix.torch-version }}" \
        INDEX_URL="${{ matrix.index-url }}" \
        GPU_ARCH_TYPE="${{ matrix.gpu-arch-type }}" \
        bash .github/scripts/run_8xgpu_integration_tests.sh