cicd-main.yml

# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: CICD NeMo
on:
  schedule:
    - cron: 0 0 * * *
  push:
    branches:
      - main
      - "pull-request/[0-9]+"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read
  statuses: write

jobs:
  pre-flight:
    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@main

  cicd-wait-in-queue:
    needs: [pre-flight]
    runs-on: ubuntu-latest
    environment: test
    if: |
      needs.pre-flight.outputs.is_ci_workload == 'false'
      && needs.pre-flight.outputs.docs_only == 'false'
    steps:
      - name: Running CI tests
        run: |
          echo "Running CI tests"

  cicd-container-build:
    uses: ./.github/workflows/_build_container.yml
    needs: [pre-flight, cicd-wait-in-queue]
    if: |
      (
        success()
        || needs.pre-flight.outputs.is_ci_workload == 'true'
        || needs.pre-flight.outputs.force_run_all == 'true'
      )
      && needs.pre-flight.outputs.docs_only == 'false'
      && !cancelled()
    with:
      image-name: emerging_optimizers
      dockerfile: docker/Dockerfile.ci
      runner: self-hosted-nemo
    secrets:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

  cicd-unit-tests:
    strategy:
      fail-fast: false
      matrix:
        include:
          - script: L0_Tests_GPU
            runner: self-hosted-nemo
            timeout: 30
          - script: L0_Tests_CPU
            runner: linux-amd64-cpu16
            timeout: 30
            cpu-only: true
    needs: [pre-flight, cicd-container-build]
    runs-on: ${{ matrix.runner }}
    name: ${{ matrix.script }}
    environment: nemo-ci
    if: |
      (
        success()
        || needs.pre-flight.outputs.is_ci_workload == 'true'
        || needs.pre-flight.outputs.force_run_all == 'true'
      )
      && !cancelled()
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: main
        uses: ./.github/actions/test-template
        with:
          runner: ${{ runner.name }}
          script: ${{ matrix.script }}
          timeout: ${{ matrix.timeout || 10 }}
          is_unit_test: "true"
          image: emerging_optimizers
          cpu-only: ${{ matrix.cpu-only || false }}
          has-azure-credentials: "true"
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

  Nemo_CICD_Test:
    needs:
      - pre-flight
      - cicd-container-build
      - cicd-unit-tests
    if: |
      (
        needs.pre-flight.outputs.docs_only == 'true'
        || needs.pre-flight.outputs.is_deployment_workflow == 'true'
        || needs.pre-flight.outputs.is_ci_workload == 'true'
        || always()
      )
      && !cancelled()
    runs-on: ubuntu-latest
    permissions: write-all
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Get workflow result
        id: result
        env:
          GH_TOKEN: ${{ github.token }}
          RUN_ID: ${{ github.run_id }}
          DOCS_ONLY: ${{ needs.pre-flight.outputs.docs_only }}
          IS_DEPLOYMENT: ${{ needs.pre-flight.outputs.is_deployment_workflow }}
          IS_CI_WORKLOAD: ${{ needs.pre-flight.outputs.is_ci_workload }}
        run: |
          # Get workflow run details and check job conclusions
          NUM_FAILED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "failure") | .name] | length')
          NUM_CANCELLED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "cancelled") | .name] | length')

          if [[ ($NUM_FAILED -eq 0 && $NUM_CANCELLED -eq 0) || $DOCS_ONLY == 'true' || $IS_DEPLOYMENT == 'true' || $IS_CI_WORKLOAD == 'true' ]]; then
            RESULT="success"
          elif [[ $NUM_CANCELLED -gt 0 ]]; then
            RESULT="cancelled"
          else
            RESULT="failure"
          fi

          # Output the final status
          echo "code=$RESULT" | tee -a $GITHUB_OUTPUT

      - name: Checkout for GH CLI
        uses: actions/checkout@v4

      - name: Remove label if not cancelled
        if: |
          steps.result.outputs.code != 'cancelled'
          && github.event.label.name == 'Run CICD'
          && github.event.pull_request.head.repo.full_name == github.repository
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.number }}
        run: gh pr edit "$PR_NUMBER" --remove-label "Run CICD"

      - name: Pipeline successful, add PR comment
        if: |
          steps.result.outputs.code == 'success'
          && github.event_name == 'pull_request'
          && env.SLACK_WEBHOOK != ''
        uses: peter-evans/create-or-update-comment@v4
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          REPOSITORY: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
        with:
          issue-number: ${{ github.event.number }}
          body: |
            [🤖]: Hi @${{ github.event.pull_request.user.login }} 👋,

            We wanted to let you know that a [CICD pipeline](https://github.com/${{ env.REPOSITORY }}/actions/runs/${{ env.RUN_ID }}) for this PR just finished successfully.

            So it might be time to merge this PR or get some approvals.

            //cc @chtruong814 @ko3n1g @pablo-garay @thomasdhc

      - name: "Pipeline not successful and not cancelled: Send Slack alert & create step summary"
        if: |
          steps.result.outputs.code == 'failure'
          && github.event.label.name == 'Run CICD'
          && env.SLACK_WEBHOOK != ''
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPOSITORY: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          PR_NUMBER: ${{ github.event.number }}
          SERVER_URL: ${{ github.server_url }}
        run: |
          set -x
          pip install PyGithub
          export BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}

          python .github/scripts/notify.py

      - name: Exit
        if: ${{ always() }}
        env:
          RESULT: ${{ steps.result.outputs.code }}
        run: |
          if [ $RESULT == "success" ]; then
            exit 0
          else
            exit 1
          fi

  Coverage_Fake:
    runs-on: ubuntu-latest
    needs: [Nemo_CICD_Test, pre-flight]
    if: |
      (
        needs.pre-flight.outputs.docs_only == 'true'
        || needs.pre-flight.outputs.is_deployment_workflow == 'true'
      )
      && needs.pre-flight.outputs.is_ci_workload == 'false'
      && !cancelled()
    environment: nemo-ci
    steps:
      - name: Generate fake coverage report
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.PAT }}
          script: |
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: context.sha,
              state: 'success',
              description: 'No code changes - coverage check skipped',
              context: 'codecov/patch'
            });

  Coverage:
    runs-on: ubuntu-latest
    needs: [pre-flight, Nemo_CICD_Test]
    if: |
      (
        (needs.pre-flight.outputs.is_ci_workload == 'true' && !failure())
        || success()
      )
      && !cancelled()
    strategy:
      matrix:
        flag: [unit-test]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download coverage reports of current branch
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-${{ matrix.flag }}-*

      - name: Get total coverage of current branch
        shell: bash -x -e -u -o pipefail {0}
        if: always()
        run: |
          pip install coverage

          ls -al .
          ls -al coverage-*/
          coverage combine --keep $(ls coverage-*/.coverage)
          coverage report -i
          rm -rf coverage-*
          ls -al

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          verbose: true
          flags: ${{ matrix.flag }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: coverage-${{ matrix.flag }}-aggregated
          path: |
            .coverage
          include-hidden-files: true