[Fix] Zero-init Ascend chunk_scaled_dot_kkt output and add solve_tril… #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nvidia-h100-ci | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| checks: read | |
| on: | |
| pull_request: | |
| branches: [ '*' ] | |
| types: [opened, synchronize, reopened, closed] | |
| pull_request_review: | |
| types: [submitted] | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| # Detect PRs that only touch NPU-only code (triton-ascend backend files, | |
| # Ascend workflow definitions) so the H100 jobs below can skip them. | |
| # Shared files — including backend __init__.py files, which GPU imports | |
| # unconditionally — always get the full GPU pipeline. Push events and | |
| # titles containing '[nv]' force the full pipeline. | |
| changes: | |
| name: Detect NPU-only changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| npu_only: ${{ steps.detect.outputs.npu_only }} | |
| steps: | |
| - name: Detect NPU-only diff | |
| id: detect | |
| uses: actions/github-script@v9.0.0 | |
| with: | |
| script: | | |
| if (context.eventName !== 'pull_request' && context.eventName !== 'pull_request_review') { | |
| core.setOutput('npu_only', 'false'); | |
| return; | |
| } | |
| const pr = context.payload.pull_request; | |
| if (pr.title.includes('[nv]')) { | |
| core.info('title contains [nv]: forcing H100 CI'); | |
| core.setOutput('npu_only', 'false'); | |
| return; | |
| } | |
| const { owner, repo } = context.repo; | |
| const files = await github.paginate(github.rest.pulls.listFiles, { | |
| owner, repo, pull_number: pr.number, per_page: 100, | |
| }); | |
| // Safe set: triton-ascend backend files (except __init__.py, which | |
| // GPU imports unconditionally) and Ascend workflow definitions. | |
| const safePattern = /^fla\/(.+\/)?backends\/triton_ascend\/(?!.*__init__\.py$)|^\.github\/workflows\/ascend-/; | |
| const unsafe = files.filter(f => !safePattern.test(f.filename)).map(f => f.filename); | |
| if (unsafe.length > 0) { | |
| core.info('GPU-affecting files changed: ' + unsafe.join(', ')); | |
| } | |
| core.setOutput('npu_only', files.length > 0 && unsafe.length === 0 ? 'true' : 'false'); | |
| test-h100-pytorch-2-12: | |
| name: Test H100 (PyTorch 2.12) | |
| # Test on all main commits and PRs, but skip on closed PRs | |
| # to avoid running tests on merged PRs, and skip NPU-only PRs. | |
| needs: changes | |
| if: needs.changes.outputs.npu_only != 'true' && github.event_name != 'pull_request_review' && (github.event_name != 'pull_request' || github.event.action != 'closed') | |
| uses: ./.github/workflows/reusable-ci-tests.yml | |
| with: | |
| runner: 'nvidia-h100-1' | |
| gpu_type: 'nvidia' | |
| conda_env_name: 'pytorch_2_12' | |
| skip_gpu_check: true | |
| check_h100_pytorch_2_7: | |
| name: Check H100 PyTorch 2.7 Eligibility | |
| needs: changes | |
| if: > | |
| needs.changes.outputs.npu_only != 'true' && | |
| ((github.event_name == 'pull_request' && github.event.action != 'closed') || | |
| (github.event_name == 'pull_request_review' && github.event.review.state == 'approved' && github.event.pull_request.state == 'open')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| outputs: | |
| allowed: ${{ steps.check.outputs.allowed }} | |
| steps: | |
| - name: Check eligibility and wait for PyTorch 2.12 H100 jobs | |
| id: check | |
| uses: actions/github-script@v9.0.0 | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const ref = context.payload.pull_request.head.sha; | |
| const eventName = context.eventName; | |
| let username, association; | |
| if (eventName === 'pull_request') { | |
| const pr = context.payload.pull_request; | |
| username = pr.user.login; | |
| association = pr.author_association; | |
| } else if (eventName === 'pull_request_review') { | |
| username = context.payload.review.user.login; | |
| association = context.payload.review.author_association; | |
| } else { | |
| core.setOutput('allowed', 'false'); | |
| return; | |
| } | |
| const timeoutMs = 85 * 60 * 1000; | |
| const pollMs = 60 * 1000; | |
| const deadline = Date.now() + timeoutMs; | |
| const requiredChecks = [ | |
| 'Test H100 (PyTorch 2.12) / test-ops', | |
| 'Test H100 (PyTorch 2.12) / test-models', | |
| ]; | |
| const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); | |
| let permission = 'none'; | |
| let allowed = association === 'OWNER'; | |
| if (!allowed) { | |
| try { | |
| const response = await github.rest.repos.getCollaboratorPermissionLevel({ | |
| owner, | |
| repo, | |
| username, | |
| }); | |
| permission = response.data.permission; | |
| allowed = permission === 'admin'; | |
| } catch (error) { | |
| core.info(`Could not read ${username}'s repo permission: ${error.message}`); | |
| } | |
| } | |
| core.info(`${username}: association=${association}, permission=${permission}`); | |
| if (!allowed) { | |
| core.setOutput('allowed', 'false'); | |
| return; | |
| } | |
| while (true) { | |
| const checkRuns = await github.paginate(github.rest.checks.listForRef, { | |
| owner, | |
| repo, | |
| ref, | |
| per_page: 100, | |
| }); | |
| const latestByName = new Map(); | |
| for (const run of checkRuns) { | |
| if (run.app?.slug !== 'github-actions') { | |
| continue; | |
| } | |
| const prev = latestByName.get(run.name); | |
| const runTime = Date.parse(run.started_at || run.completed_at || run.created_at || 0); | |
| const prevTime = prev ? Date.parse(prev.started_at || prev.completed_at || prev.created_at || 0) : 0; | |
| if (!prev || runTime > prevTime) { | |
| latestByName.set(run.name, run); | |
| } | |
| } | |
| let allPassed = true; | |
| let shouldWait = false; | |
| for (const name of requiredChecks) { | |
| const run = latestByName.get(name); | |
| if (!run) { | |
| core.info(`${name}: missing`); | |
| allPassed = false; | |
| shouldWait = true; | |
| continue; | |
| } | |
| core.info(`${name}: status=${run.status}, conclusion=${run.conclusion}`); | |
| if (run.status !== 'completed') { | |
| allPassed = false; | |
| shouldWait = true; | |
| continue; | |
| } | |
| if (run.conclusion !== 'success') { | |
| core.setFailed(`Required check failed: ${name}`); | |
| return; | |
| } | |
| } | |
| if (allPassed) { | |
| core.setOutput('allowed', 'true'); | |
| return; | |
| } | |
| if (!shouldWait || Date.now() >= deadline) { | |
| core.setFailed('Timed out waiting for PyTorch 2.12 H100 checks to pass'); | |
| return; | |
| } | |
| await sleep(pollMs); | |
| } | |
| test-h100-pytorch-2-7-ops: | |
| name: Test H100 Ops (PyTorch 2.7) | |
| needs: check_h100_pytorch_2_7 | |
| if: needs.check_h100_pytorch_2_7.outputs.allowed == 'true' | |
| uses: ./.github/workflows/reusable-ci-tests.yml | |
| with: | |
| runner: 'nvidia-h100-1' | |
| gpu_type: 'nvidia' | |
| conda_env_name: 'pytorch_2_7' | |
| skip_gpu_check: true | |
| skip_models_tests: true | |
| benchmark-h100: | |
| name: Benchmark H100 (PyTorch 2.12) | |
| needs: test-h100-pytorch-2-12 | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/workflows/reusable-ci-benchmarks.yml | |
| with: | |
| runner: 'nvidia-h100-1' | |
| conda_env_name: 'pytorch_2_12' | |
| base_ref: origin/${{ github.event.pull_request.base.ref }} |