Nightly CI Pipeline #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: Nightly CI Pipeline | |
| on: | |
| schedule: | |
| - cron: '0 8 * * *' # Every day at 12:00 AM PST (08:00 UTC) | |
| workflow_dispatch: # Allow manual triggering for testing | |
| inputs: | |
| release: | |
| description: 'Stage NGC images, Artifactory wheels, and trigger the GitLab release pipeline. Schedule always sets release=true.' | |
| required: false | |
| type: boolean | |
| default: false | |
| run_tests: | |
| description: 'Run vllm/sglang/trtllm tests + dynamo-pipeline checks. Schedule always runs them.' | |
| required: false | |
| type: boolean | |
| default: true | |
| skip_gitlab_pipeline: | |
| description: 'Skip the GitLab release automation pipeline trigger. Emergency use only.' | |
| required: false | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| # Lets the build jobs' compliance-extract query the Actions API for the | |
| # previous successful cron-triggered nightly run (manual runs are excluded) | |
| # and download its baseline compliance artifact to diff the OSRB CSV against. | |
| # A reusable workflow's token can't exceed the caller's. | |
| actions: read | |
| env: | |
| BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }} | |
| jobs: | |
| # ============================================================================ | |
| # PRE-WARM K8S BUILDER | |
| # ============================================================================ | |
| create-fresh-builder: | |
| name: Create fresh K8s builder | |
| runs-on: prod-default-small-v2 | |
| permissions: | |
| contents: read | |
| outputs: | |
| builder_name: ${{ steps.export-builder-name.outputs.builder_name }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Export builder name | |
| id: export-builder-name | |
| run: | | |
| echo "builder_name=${{ env.BUILDER_NAME }}" >> $GITHUB_OUTPUT | |
| - name: Create and bootstrap fresh K8s builder | |
| uses: ./.github/actions/bootstrap-buildkit | |
| with: | |
| builder_name: ${{ steps.export-builder-name.outputs.builder_name }} | |
| buildkit_worker_addresses: '' | |
| suppress_fallback_warning: 'true' | |
| # ============================================================================ | |
| # RESOLVE SOURCE SHA | |
| # ============================================================================ | |
| # Single SHA every downstream job (shared-build-image, dynamo-pipeline, | |
| # shared-test, release.yml) builds, tags, and releases against. On schedule | |
| # and workflow_dispatch from main, this is just github.sha. | |
| resolve-source-sha: | |
| name: Resolve source SHA | |
| runs-on: prod-default-v2 | |
| permissions: | |
| contents: read | |
| outputs: | |
| source_sha: ${{ steps.resolve.outputs.source_sha }} | |
| steps: | |
| - id: resolve | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| SHA="${GITHUB_SHA}" | |
| echo "Using caller SHA: ${SHA}" | |
| echo "source_sha=${SHA}" >> "$GITHUB_OUTPUT" | |
| # ============================================================================ | |
| # COMPUTE NIGHTLY DEV VERSION | |
| # ============================================================================ | |
| # Emits a PEP 440 dev suffix (e.g. .dev20260423) forwarded to every | |
| # pipeline below. At the leaf, the suffix is stamped into pyproject / Cargo | |
| # versions on the runner before docker build, so wheels produced by the | |
| # wheel_builder stage carry the dev version. | |
| compute-dev-version: | |
| name: Compute dev version suffix | |
| runs-on: prod-default-v2 | |
| permissions: | |
| contents: read | |
| outputs: | |
| dev_suffix: ${{ steps.compute.outputs.dev_suffix }} | |
| steps: | |
| - id: compute | |
| shell: bash | |
| run: | | |
| DATE=$(date -u +%Y%m%d) | |
| echo "dev_suffix=.dev${DATE}" >> $GITHUB_OUTPUT | |
| # ============================================================================ | |
| # COMPUTE RELEASE MODE | |
| # ============================================================================ | |
| # schedule → always release. workflow_dispatch → honor the `release` input. | |
| # Output is consumed by the `release` job below to gate the workflow_call | |
| # into release.yml. | |
| compute-release-mode: | |
| name: Compute release mode | |
| runs-on: prod-default-v2 | |
| permissions: | |
| contents: read | |
| outputs: | |
| release: ${{ steps.compute.outputs.release }} | |
| run_tests: ${{ steps.compute.outputs.run_tests }} | |
| steps: | |
| - id: compute | |
| shell: bash | |
| env: | |
| DISPATCH_RELEASE: ${{ inputs.release }} | |
| DISPATCH_RUN_TESTS: ${{ inputs.run_tests }} | |
| run: | | |
| case "${GITHUB_EVENT_NAME}" in | |
| schedule) | |
| # cron must keep tests on so a failing nightly blocks the release. | |
| echo "release=true" >> "$GITHUB_OUTPUT" | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| ;; | |
| workflow_dispatch) | |
| if [ "${DISPATCH_RELEASE}" = "false" ]; then | |
| echo "release=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "release=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "${DISPATCH_RUN_TESTS}" = "false" ]; then | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| ;; | |
| *) | |
| echo "release=false" >> "$GITHUB_OUTPUT" | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| ;; | |
| esac | |
| # Seeds the nightly Slack thread; exposes ts so downstream jobs (and the | |
| # GitLab pipeline) can reply. continue-on-error so a Slack outage doesn't | |
| # block the nightly — downstream jobs skip on empty outputs. | |
| notify-slack-start: | |
| name: Notify Slack — nightly started | |
| runs-on: prod-default-v2 | |
| needs: [compute-release-mode, resolve-source-sha, compute-dev-version] | |
| if: ${{ needs.compute-release-mode.outputs.release == 'true' }} | |
| permissions: | |
| contents: read | |
| outputs: | |
| thread_ts: ${{ steps.post.outputs.thread_ts }} | |
| steps: | |
| - name: Post start message | |
| id: post | |
| continue-on-error: true | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_RELEASE_BOT_TOKEN }} | |
| SLACK_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }} | |
| SOURCE_SHA: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| DEV_SUFFIX: ${{ needs.compute-dev-version.outputs.dev_suffix }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| DATE_UTC=$(date -u +%Y-%m-%d) | |
| SHORT_SHA="${SOURCE_SHA:0:7}" | |
| PAYLOAD=$(jq -n \ | |
| --arg channel "${SLACK_CHANNEL_ID}" \ | |
| --arg date "${DATE_UTC}" \ | |
| --arg sha "${SHORT_SHA}" \ | |
| --arg run_url "${RUN_URL}" \ | |
| --arg dev "${DEV_SUFFIX}" \ | |
| '{ | |
| channel: $channel, | |
| text: ("Dynamo Nightly build started — " + $date), | |
| blocks: [ | |
| { type: "header", | |
| text: { type: "plain_text", text: (":crescent_moon: Dynamo Nightly build started — " + $date) } }, | |
| { type: "section", | |
| fields: [ | |
| { type: "mrkdwn", text: ("*Commit:*\n`" + $sha + "`") }, | |
| { type: "mrkdwn", text: ("*Dev version suffix:*\n`" + $dev + "`") } | |
| ] }, | |
| { type: "context", | |
| elements: [ { type: "mrkdwn", text: ("<" + $run_url + "|GitHub Actions run>") } ] } | |
| ] | |
| }') | |
| RESPONSE=$(curl -fsSL -X POST https://slack.com/api/chat.postMessage \ | |
| -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ | |
| -H "Content-type: application/json; charset=utf-8" \ | |
| --data "${PAYLOAD}") | |
| OK=$(echo "${RESPONSE}" | jq -r '.ok') | |
| if [ "${OK}" != "true" ]; then | |
| # RESPONSE may carry channel/ts/warning fields; surface only .error. | |
| ERR=$(echo "${RESPONSE}" | jq -r '.error // "unknown"') | |
| echo "::error::Slack chat.postMessage failed: ${ERR}" | |
| exit 1 | |
| fi | |
| TS=$(echo "${RESPONSE}" | jq -r '.ts') | |
| # ts is a runtime per-message identifier (not a secret) and is the only | |
| # value other jobs need; channel ID is read by every downstream Slack | |
| # step directly from secrets.SLACK_RELEASE_CHANNEL_ID so it stays | |
| # auto-masked. Do NOT ::add-mask:: the ts — masking it before writing | |
| # GITHUB_OUTPUT caused the value to arrive empty across the | |
| # workflow_call boundary, silently skipping downstream reply steps. | |
| echo "thread_ts=${TS}" >> "${GITHUB_OUTPUT}" | |
| echo "Posted nightly start message" | |
| # ============================================================================ | |
| # MANUAL RELEASE APPROVAL GATE | |
| # ============================================================================ | |
| # Manual workflow_dispatch that would publish (release=true) must be approved | |
| # by a reviewer of the `manual-release-approval` environment before the | |
| # `release` job runs release.yml. Scheduled runs skip this job; the `release` | |
| # job below accepts `skipped` as success. | |
| manual-release-approval: | |
| name: Manual release approval gate | |
| needs: [compute-release-mode] | |
| if: ${{ github.event_name == 'workflow_dispatch' && needs.compute-release-mode.outputs.release == 'true' }} | |
| runs-on: prod-default-small-v2 | |
| environment: manual-release-approval | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Record approval | |
| run: | | |
| echo "Manual nightly release approved by ${{ github.actor }}" | |
| echo "Run: ${{ github.run_id }} attempt ${{ github.run_attempt }}" | |
| # ============================================================================ | |
| # BUILD JOBS | |
| # ============================================================================ | |
| vllm-build: | |
| name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI | |
| needs: [create-fresh-builder, resolve-source-sha] | |
| uses: ./.github/workflows/shared-build-image.yml | |
| with: | |
| framework: vllm | |
| target: runtime | |
| cuda_version: '["13.0"]' | |
| platform: 'linux/amd64,linux/arm64' | |
| builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} | |
| inline_compliance: true | |
| build_timeout_minutes: 120 | |
| # Source archival temporarily disabled until the wheel-builder build | |
| # context includes deploy/inference-gateway/ext-proc (a cargo vendor | |
| # workspace member). | |
| archive_sources: false | |
| # OSRB CSV diff baseline: nightly context -> diff against the previous | |
| # successful scheduled nightly run's compliance artifacts; manual runs | |
| # are excluded (see resolve_diff_base.py). | |
| diff_event_context: nightly | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| sglang-build: | |
| name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI | |
| needs: [create-fresh-builder, resolve-source-sha] | |
| uses: ./.github/workflows/shared-build-image.yml | |
| with: | |
| framework: sglang | |
| target: runtime | |
| cuda_version: '["13.0"]' | |
| platform: 'linux/amd64,linux/arm64' | |
| builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} | |
| inline_compliance: true | |
| build_timeout_minutes: 120 | |
| archive_sources: false | |
| # OSRB CSV diff baseline: nightly context -> diff against the previous | |
| # successful scheduled nightly run's compliance artifacts; manual runs | |
| # are excluded (see resolve_diff_base.py). | |
| diff_event_context: nightly | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| trtllm-build: | |
| name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI | |
| needs: [create-fresh-builder, resolve-source-sha] | |
| uses: ./.github/workflows/shared-build-image.yml | |
| with: | |
| framework: trtllm | |
| target: runtime | |
| cuda_version: '["13.1"]' | |
| platform: 'linux/amd64,linux/arm64' | |
| builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} | |
| inline_compliance: true | |
| build_timeout_minutes: 120 | |
| archive_sources: false | |
| # OSRB CSV diff baseline: nightly context -> diff against the previous | |
| # successful scheduled nightly run's compliance artifacts; manual runs | |
| # are excluded (see resolve_diff_base.py). | |
| diff_event_context: nightly | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| # ============================================================================ | |
| # COMPLIANCE | |
| # ============================================================================ | |
| # Compliance extract + verify now runs as an inline step INSIDE each build job | |
| # (inline_compliance: true above), on the same warm builder — a cache hit + small | |
| # scratch export instead of a cold full-image rebuild on the fresh builder. The | |
| # build jobs currently set archive_sources: false (archival temporarily | |
| # disabled — see above), so /sources.zip is not emitted; /legal + /sboms still | |
| # upload at the default 30-day retention. A compliance failure fails the build. | |
| # ============================================================================ | |
| # TEST JOBS | |
| # ============================================================================ | |
| vllm-test: | |
| name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI | |
| needs: [vllm-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: vllm | |
| test_type: Test | |
| amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform | |
| target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.0"]' | |
| platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped | |
| enable_coverage: true | |
| run_cpu_only_tests: true | |
| cpu_only_test_markers: vllm and gpu_0 | |
| gpu_test_markers: vllm and gpu_1 | |
| gpu_test_timeout_minutes: 240 | |
| # Profiled tests run in the parallel stage; unprofiled fall through to sequential. | |
| # 24 GiB admits all currently profiled vLLM tests (max is ~20.4 GiB) on a 48 GiB GPU. | |
| run_gpu_parallel_tests: true | |
| gpu_parallel_max_vram_gib: '24' | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| vllm-multi-gpu-test: | |
| name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI | |
| needs: [vllm-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: vllm | |
| test_type: Multi-GPU Test | |
| amd_runner: prod-tester-amd-gpu-4-v2 | |
| target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.0"]' | |
| platform: '["amd64"]' # No ARM GPUs available | |
| enable_coverage: true | |
| run_sanity_check: false | |
| # Includes elastic_ep, locally validated DP=2->4->2 on H200 GPUs with | |
| # vLLM 0.24.0. That test remains pytest-skipped because this runner has | |
| # 24 GiB/GPU, while its unquantized EPLB model needs >=80 GiB/GPU. | |
| gpu_test_markers: vllm and (gpu_2 or gpu_4) | |
| gpu_test_timeout_minutes: 45 | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| sglang-test: | |
| name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI | |
| needs: [sglang-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: sglang | |
| test_type: Test | |
| amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform | |
| target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.0"]' | |
| platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped | |
| enable_coverage: true | |
| run_cpu_only_tests: true | |
| cpu_only_test_markers: sglang and gpu_0 | |
| # Exclude h100 tests: they need >24 GiB and run on the sglang-h100-test lane below. | |
| gpu_test_markers: sglang and gpu_1 and not h100 | |
| gpu_test_timeout_minutes: 240 | |
| # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential. | |
| # Current single-GPU runners are 24 GiB, so this cap admits the profiled SGLang pool | |
| # while yielding one auto slot today. Larger runners will get more slots from the same markers. | |
| run_gpu_parallel_tests: true | |
| gpu_parallel_max_vram_gib: '24' | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| sglang-multi-gpu-test: | |
| name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI | |
| needs: [sglang-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: sglang | |
| test_type: Multi-GPU Test | |
| amd_runner: prod-tester-amd-gpu-4-v2 | |
| target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.0"]' | |
| platform: '["amd64"]' # No ARM GPUs available | |
| enable_coverage: true | |
| run_sanity_check: false | |
| gpu_test_markers: sglang and (gpu_2 or gpu_4) | |
| gpu_test_timeout_minutes: 45 | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| sglang-h100-test: | |
| name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI | |
| needs: [sglang-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: sglang | |
| test_type: H100 Test | |
| # 80 GiB H100 (aws-dev-02): sglang tests that exceed the 24 GiB gpu_1 lane. | |
| amd_runner: aws-dev-02-tester-amd-gpu-v2 | |
| target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.0"]' | |
| platform: '["amd64"]' | |
| enable_coverage: true | |
| run_sanity_check: false | |
| gpu_test_markers: sglang and h100 and gpu_1 | |
| gpu_test_timeout_minutes: 60 | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| trtllm-test: | |
| name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI | |
| needs: [trtllm-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: trtllm | |
| test_type: Test | |
| amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform | |
| target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.1"]' | |
| platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped | |
| enable_coverage: true | |
| run_cpu_only_tests: true | |
| cpu_only_test_markers: trtllm and gpu_0 | |
| gpu_test_markers: trtllm and gpu_1 | |
| gpu_test_timeout_minutes: 240 | |
| # Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential. | |
| # Current single-GPU runners are 24 GiB, so this cap admits the profiled TRT-LLM pool | |
| # while yielding one auto slot today. Larger runners will get more slots from the same markers. | |
| run_gpu_parallel_tests: true | |
| gpu_parallel_max_vram_gib: '24' | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| trtllm-multi-gpu-test: | |
| name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI | |
| needs: [trtllm-build, resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/shared-test.yml | |
| with: | |
| dd_env: nightly | |
| dd_flaky_retry_enabled: 'false' | |
| test_suite_name: trtllm | |
| test_type: Multi-GPU Test | |
| amd_runner: prod-tester-amd-gpu-4-v2 | |
| target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }} | |
| cuda_version: '["13.1"]' | |
| platform: '["amd64"]' # No ARM GPUs available | |
| enable_coverage: true | |
| run_sanity_check: false | |
| gpu_test_markers: trtllm and (gpu_2 or gpu_4) | |
| gpu_test_timeout_minutes: 45 | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| # ============================================================================ | |
| # XPU TEST JOBS | |
| # ============================================================================ | |
| vllm-test-xpu: | |
| name: vllm-xpu | |
| needs: [resolve-source-sha, compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/xpu-ci.yaml | |
| with: | |
| framework: vllm | |
| pytest_markers: 'vllm' | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| # ============================================================================ | |
| # DYNAMO RUNTIME PIPELINE | |
| # ============================================================================ | |
| dynamo-pipeline: | |
| name: dynamo-runtime | |
| needs: [create-fresh-builder, compute-dev-version, resolve-source-sha] | |
| uses: ./.github/workflows/dynamo-pipeline.yml | |
| with: | |
| builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} | |
| fresh_builder: true | |
| no_cache: true | |
| build_timeout_minutes: 120 | |
| # TODO: widen beyond `pre_merge` — today it picks up tests | |
| # (e.g. fault_tolerance/deploy/*) that fail in this container-only | |
| # context. Matches the coverage of the old container-validation-dynamo | |
| # workflow. | |
| cpu_parallel_test_markers: 'pre_merge and parallel and not (vllm or sglang or trtllm) and (gpu_0)' | |
| cpu_sequential_test_markers: 'pre_merge and not parallel and not (vllm or sglang or trtllm) and (gpu_0)' | |
| gpu_test_markers: 'pre_merge and none and gpu_1' | |
| dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }} | |
| source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| image_tag_suffix: '-nightly' | |
| secrets: inherit | |
| # ============================================================================ | |
| # RUST COVERAGE | |
| # ============================================================================ | |
| rust-tests: | |
| name: rust-${{ matrix.dir == '.' && 'root' || matrix.dir }}-coverage | |
| needs: [compute-release-mode] | |
| if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} | |
| runs-on: | |
| group: Fastchecker | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dir: ['.', 'lib/bindings/python', 'lib/bindings/kvbm'] | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| lfs: true | |
| - name: Set up system dependencies | |
| run: | | |
| # Install protoc for Rust build dependencies (NOTE: much faster than apt install) | |
| PB_REL="https://github.com/protocolbuffers/protobuf/releases" | |
| PROTOC_VER="30.2" | |
| PROTOC_ZIP="protoc-${PROTOC_VER}-linux-x86_64.zip" | |
| PROTOC_SHA256="327e9397c6fb3ea2a542513a3221334c6f76f7aa524a7d2561142b67b312a01f" | |
| curl --retry 5 --retry-delay 2 -fsSLO "$PB_REL/download/v${PROTOC_VER}/${PROTOC_ZIP}" | |
| echo "${PROTOC_SHA256} ${PROTOC_ZIP}" | sha256sum -c - | |
| unzip "${PROTOC_ZIP}" -d $HOME/.local | |
| rm "${PROTOC_ZIP}" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| protoc --version | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Set up Rust Toolchain Components | |
| run: rustup component add llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| run: cargo-llvm-cov --version 2>/dev/null || cargo install cargo-llvm-cov --locked | |
| # Have an explicit step to build tests first to separate time spent on build vs execution. | |
| - name: Compile Tests | |
| working-directory: ${{ matrix.dir }} | |
| run: cargo test --locked --no-run | |
| - name: Run Unit Tests with Coverage | |
| working-directory: ${{ matrix.dir }} | |
| # NOTE: --all-targets doesn't run doc tests. | |
| # cargo llvm-cov is a drop-in for cargo test; --no-report defers output | |
| # so we can generate multiple formats without re-running the tests. | |
| run: | | |
| cargo llvm-cov --locked --all-targets --no-report | |
| cargo llvm-cov report --output-path coverage-rust.txt | |
| cargo llvm-cov report --lcov --output-path coverage-rust.lcov | |
| echo "Coverage summary:" | |
| grep "^TOTAL" coverage-rust.txt || tail -3 coverage-rust.txt | |
| SAFE_DIR=$(echo "${{ matrix.dir }}" | sed 's|^\.$|root|' | tr '/' '-') | |
| echo "RUST_COV_ARTIFACT_NAME=coverage-rust-${SAFE_DIR}-${{ github.run_id }}" >> $GITHUB_ENV | |
| - name: Upload Rust Coverage Data | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: always() | |
| with: | |
| name: ${{ env.RUST_COV_ARTIFACT_NAME }} | |
| path: | | |
| ${{ matrix.dir }}/coverage-rust.txt | |
| ${{ matrix.dir }}/coverage-rust.lcov | |
| retention-days: 7 | |
| # ============================================================================ | |
| # RELEASE (workflow_call into release.yml) | |
| # ============================================================================ | |
| # commit_sha is the SHA `resolve-source-sha` picked (github.sha). release.yml | |
| # requires post-merge CI to have already pushed images to ECR for that SHA; | |
| # on schedule this is true because post-merge runs on every merge to main | |
| # and nightly fires later. | |
| release: | |
| name: Release Nightly | |
| needs: | |
| - resolve-source-sha | |
| - compute-release-mode | |
| - notify-slack-start | |
| - manual-release-approval | |
| - vllm-build | |
| - sglang-build | |
| - trtllm-build | |
| - dynamo-pipeline | |
| # !cancelled() lets the job evaluate the explicit gates below rather than | |
| # implicitly requiring every need to succeed (notify-slack-start is | |
| # best-effort). All three runtime builds and dynamo-pipeline are gated | |
| # strictly: release.yml copies their -nightly ECR images to NGC and extracts | |
| # wheels from the dynamo-pipeline image, so a failed build must not publish a | |
| # partial release. manual-release-approval is `skipped` on cron, `success` on dispatch. | |
| if: ${{ !cancelled() | |
| && needs.compute-release-mode.outputs.release == 'true' | |
| && needs.vllm-build.result == 'success' | |
| && needs.sglang-build.result == 'success' | |
| && needs.trtllm-build.result == 'success' | |
| && needs.dynamo-pipeline.result == 'success' | |
| && (needs.manual-release-approval.result == 'success' | |
| || needs.manual-release-approval.result == 'skipped') }} | |
| uses: ./.github/workflows/release.yml | |
| with: | |
| commit_sha: ${{ needs.resolve-source-sha.outputs.source_sha }} | |
| nightly: true | |
| skip_gitlab_pipeline: ${{ inputs.skip_gitlab_pipeline || false }} | |
| slack_thread_ts: ${{ needs.notify-slack-start.outputs.thread_ts }} | |
| secrets: inherit | |
| # ============================================================================ | |
| # COVERAGE REPORT | |
| # ============================================================================ | |
| coverage-report: | |
| name: Generate Coverage Report | |
| runs-on: ubuntu-latest | |
| needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, rust-tests] | |
| if: always() | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install Coverage Tools | |
| run: | | |
| python -m pip install "coverage[toml]==7.13.1" | |
| python -m coverage --version | |
| echo "✅ Coverage tools installed" | |
| - name: Download All Python Coverage Artifacts | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| pattern: coverage-python-* | |
| path: coverage-artifacts/ | |
| merge-multiple: false | |
| - name: Download All Rust Coverage Artifacts | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| continue-on-error: true | |
| with: | |
| pattern: coverage-rust-* | |
| path: coverage-rust-artifacts/ | |
| merge-multiple: false | |
| - name: List Downloaded Artifacts | |
| run: | | |
| echo "📦 Downloaded coverage artifacts:" | |
| echo "==== Directory structure (including hidden files) ====" | |
| ls -Ra coverage-artifacts/ | |
| echo "" | |
| echo "==== Coverage files found ====" | |
| find coverage-artifacts/ -type f \( -name "*.xml" -o -name ".coverage*" \) | sort | |
| echo "" | |
| echo "==== Specifically looking for .coverage files ====" | |
| find coverage-artifacts/ -name ".coverage" -type f | |
| echo "" | |
| - name: Merge All Coverage Data | |
| run: | | |
| set -x | |
| echo "📊 Merging all test coverage..." | |
| # Expand $GITHUB_WORKSPACE in the config | |
| sed -i "s|\$GITHUB_WORKSPACE|${GITHUB_WORKSPACE}|g" .coveragerc | |
| echo "Updated .coveragerc with path remapping:" | |
| cat .coveragerc | |
| # Find all .coverage files and copy them with unique names | |
| mkdir -p coverage-combined | |
| find coverage-artifacts/ -name ".coverage*" 2>/dev/null | while read -r file; do | |
| unique_name=$(echo "$file" | tr '/' '_' | sed 's/coverage-artifacts_//') | |
| cp "$file" "coverage-combined/.coverage.${unique_name}" | |
| echo "Copied: $file -> coverage-combined/.coverage.${unique_name}" | |
| done | |
| # Check if we have any coverage files | |
| if ls coverage-combined/.coverage* 1> /dev/null 2>&1; then | |
| echo "✅ Found coverage files to merge" | |
| echo "Files to merge:" | |
| ls -lh coverage-combined/ | |
| # Combine all coverage data from the workspace root so that the | |
| # relative canonical path "components/src/dynamo" in .coveragerc | |
| # resolves to $GITHUB_WORKSPACE/components/src/dynamo (where the | |
| # source actually lives) rather than to the non-existent | |
| # coverage-combined/components/src/dynamo subdirectory. | |
| echo "Running coverage combine with path remapping..." | |
| set +e # Don't exit on error | |
| COVERAGE_RCFILE=.coveragerc coverage combine --keep coverage-combined/.coverage* 2>&1 | tee combine.log | |
| set -e | |
| if [ -f .coverage ]; then | |
| echo "✅ Combined .coverage file created" | |
| # Generate reports (continue even if some fail) | |
| echo "📊 Generating coverage reports..." | |
| set +e | |
| COVERAGE_RCFILE=.coveragerc coverage report --show-missing --data-file=.coverage 2>&1 | tee coverage-report.txt | |
| TOTAL_COVERAGE=$(awk '/^TOTAL/ {print $NF}' coverage-report.txt | tail -1) | |
| echo "TOTAL_COVERAGE=${TOTAL_COVERAGE:-0%}" >> $GITHUB_ENV | |
| COVERAGE_RCFILE=.coveragerc coverage html --data-file=.coverage -d coverage-html/ 2>&1 || echo "HTML generation failed" | |
| COVERAGE_RCFILE=.coveragerc coverage xml --data-file=.coverage -o coverage-merged.xml 2>&1 || echo "XML generation failed" | |
| set -e | |
| else | |
| echo "❌ Failed to create combined .coverage file" | |
| echo "No coverage data available" > coverage-report.txt | |
| echo "TOTAL_COVERAGE=0%" >> $GITHUB_ENV | |
| fi | |
| else | |
| echo "⚠️ No coverage data found" | |
| echo "No coverage data available" > coverage-report.txt | |
| echo "TOTAL_COVERAGE=0%" >> $GITHUB_ENV | |
| fi | |
| - name: Process Rust Coverage | |
| run: | | |
| RUST_TOTAL_COVERAGE="N/A" | |
| echo "No Rust coverage data available yet." > coverage-rust-report.txt | |
| if find coverage-rust-artifacts/ -name "coverage-rust.txt" -type f 2>/dev/null | grep -q .; then | |
| echo "📊 Processing Rust coverage reports..." | |
| # Concatenate full per-file coverage from every workspace dir | |
| > coverage-rust-report.txt | |
| find coverage-rust-artifacts/ -name "coverage-rust.txt" | sort | while read -r f; do | |
| DIR_NAME=$(basename $(dirname "$f")) | |
| echo "=== ${DIR_NAME} ===" >> coverage-rust-report.txt | |
| cat "$f" >> coverage-rust-report.txt | |
| echo "" >> coverage-rust-report.txt | |
| done | |
| # Aggregate line coverage across all crates by summing total/missed lines | |
| # from every TOTAL row. Columns on a `cargo llvm-cov report` TOTAL line: | |
| # $2 regions $3 missed_regions $4 region% | |
| # $5 funcs $6 missed_funcs $7 func% | |
| # $8 lines $9 missed_lines $10 line% | |
| RUST_TOTAL_COVERAGE=$(find coverage-rust-artifacts/ -name "coverage-rust.txt" -exec grep -h "^TOTAL" {} + \ | |
| | awk '{ total += $8; missed += $9 } END { if (total > 0) printf "%.2f%%", (total - missed) * 100 / total; else print "N/A" }') | |
| RUST_TOTAL_COVERAGE="${RUST_TOTAL_COVERAGE:-N/A}" | |
| echo "Rust line coverage (aggregated across $(find coverage-rust-artifacts/ -name "coverage-rust.txt" | wc -l | tr -d ' ') crates): ${RUST_TOTAL_COVERAGE}" | |
| else | |
| echo "ℹ️ No Rust coverage artifacts found" | |
| fi | |
| echo "RUST_TOTAL_COVERAGE=${RUST_TOTAL_COVERAGE}" >> $GITHUB_ENV | |
| - name: Create Coverage Summary | |
| run: | | |
| DATE=$(date +"%Y-%m-%d %H:%M:%S UTC") | |
| cat > coverage-summary.md << EOF | |
| # 📊 Test Coverage Report | |
| **Date:** ${DATE} | |
| **Run ID:** ${{ github.run_id }} | |
| **Workflow:** ${{ github.workflow }} | |
| | Language | Total Coverage | | |
| |----------|---------------| | |
| | Python | ${TOTAL_COVERAGE} | | |
| | Rust | ${RUST_TOTAL_COVERAGE} | | |
| --- | |
| ## Python Coverage Details | |
| \`\`\` | |
| $(cat coverage-report.txt 2>/dev/null || echo "No Python coverage data available") | |
| \`\`\` | |
| --- | |
| ## Rust Coverage Details | |
| \`\`\` | |
| $(cat coverage-rust-report.txt 2>/dev/null || echo "No Rust coverage data available") | |
| \`\`\` | |
| --- | |
| ## 📁 Artifacts | |
| - Full HTML Report: Download \`coverage-reports-${{ github.run_id }}\` artifact | |
| - Python Coverage XML: \`coverage-merged.xml\` | |
| - Rust Coverage LCov: \`coverage-rust.lcov\` (per workspace dir) | |
| EOF | |
| echo "📄 Coverage summary generated" | |
| cat coverage-summary.md | |
| - name: Post to Workflow Summary | |
| run: cat coverage-summary.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload Coverage Reports | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: always() | |
| with: | |
| name: coverage-reports-${{ github.run_id }} | |
| path: | | |
| coverage-html/ | |
| coverage-merged.xml | |
| coverage-report.txt | |
| coverage-rust-report.txt | |
| coverage-rust-artifacts/ | |
| coverage-summary.md | |
| .coverage | |
| retention-days: 30 | |
| # ============================================================================ | |
| # CLEANUP | |
| # ============================================================================ | |
| clean-k8s-builder: | |
| name: Clean K8s builder if exists | |
| runs-on: prod-default-small-v2 | |
| if: always() | |
| needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, create-fresh-builder] | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Register K8s builder context (skip bootstrap) | |
| uses: ./.github/actions/bootstrap-buildkit | |
| continue-on-error: true | |
| with: | |
| builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} | |
| buildkit_worker_addresses: '' | |
| skip_bootstrap: 'true' | |
| - name: Remove K8s builder | |
| shell: bash | |
| run: | | |
| docker buildx rm ${{ env.BUILDER_NAME }} || true | |
| ############################## SLACK NOTIFICATION ############################## | |
| notify-slack: | |
| if: always() | |
| needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, rust-tests] | |
| permissions: | |
| contents: read | |
| actions: read # grant the reusable notifier read access to list this run's jobs | |
| uses: ./.github/workflows/notify-slack.yml | |
| with: | |
| pipeline_name: Nightly | |
| runs_on: prod-default-v2 | |
| secrets: inherit |