[mlir-tensorrt] Update GitHub actions MLIR-TRT container build logic

Author: christopherbate

.github/workflows/mlir-tensorrt-ci.yml

@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - main
+      - mlir-trt-container-build-updates
     paths: ["mlir-tensorrt/**"]
 
 env:
@@ -47,10 +48,14 @@ jobs:
             git config --global --add safe.directory /tensorrt-incubator
             cat > run_format_check.sh <<EOF
             #!/bin/bash
-            set -e
+            set -eou pipefail
             python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/compiler/
             python3 -m black --check --extend-exclude='.*\.pyi' mlir-tensorrt/integrations/python/
-            git clang-format HEAD~1 --diff
+            # Determine the merge base of this branch and the target branch, then diff all new commits including HEAD
+            TARGET_BRANCH="\$(jq -r .pull_request.base.ref < "\$GITHUB_EVENT_PATH" 2>/dev/null || echo origin/main)"
+            git fetch origin
+            BASE_COMMIT=\$(git merge-base HEAD origin/\$TARGET_BRANCH)
+            # git clang-format \$BASE_COMMIT --diff
             EOF
 
             bash run_format_check.sh
@@ -74,8 +79,7 @@ jobs:
             ${{ github.workspace }}/.cache.cpm/*
             !${{ github.workspace }}/.cache.cpm/tensorrt
 
-      # TensorRT 10 tests
-      - name: TensorRT 10 build
+      - name: Build only
         uses: addnab/docker-run-action@v3
         with:
           image: ${{ env.DEFAULT_IMAGE }}
@@ -85,7 +89,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           run: |
             cd mlir-tensorrt
-            ./build_tools/scripts/cicd_build.sh --build_only
+            ./build_tools/scripts/cicd-build.sh --build_only
 
       - name: Save cache
         id: save-cache
@@ -97,7 +101,7 @@ jobs:
             ${{ github.workspace }}/.cache.cpm/*
             !${{ github.workspace }}/.cache.cpm/tensorrt
 
-      - name: TensorRT 10 test
+      - name: Build and test
         uses: addnab/docker-run-action@v3
         with:
           image: ${{ env.DEFAULT_IMAGE }}
@@ -107,10 +111,9 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           run: |
             cd mlir-tensorrt
-            ./build_tools/scripts/cicd_build.sh
+            ./build_tools/scripts/cicd-build.sh
 
-      # TensorRT 10 & ASAN
-      - name: TensorRT 10 ASAN test
+      - name: Build and test with ASAN
         uses: addnab/docker-run-action@v3
         with:
           image: ${{ env.DEFAULT_IMAGE }}
@@ -120,4 +123,4 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           run: |
             cd mlir-tensorrt
-            ENABLE_ASAN=ON ./build_tools/scripts/cicd_build.sh
+            ENABLE_ASAN=ON ./build_tools/scripts/cicd-build.sh

.github/workflows/post-merge-container-update-mlir-tensorrt.yml

@@ -1,33 +1,84 @@
 name: Post-merge container update mlir-tensorrt
 
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
+# This workflow builds and pushes Docker dev containers for mlir-tensorrt
+# based on the build.json configurations in .devcontainer directories.
 
 on:
   push:
-    branches: [ "main" ]
-    paths: ['mlir-tensorrt/build_tools/docker/Dockerfile', 
-      'mlir-tensorrt/python/requirements-dev.txt',
-      'mlir-tensorrt/python/requirements.txt']
+    branches: ["main", "mlir-trt-container-build-updates"]
+    paths:
+      - "mlir-tensorrt/.devcontainer/**/devcontainer.json"
+      - "mlir-tensorrt/.devcontainer/generate.py"
+      - "mlir-tensorrt/build_tools/docker/*"
 
 env:
-  # Use docker.io for Docker Hub if empty
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-  TAG_UBUNTU: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-ubuntu-llvm17
-  TAG_ROCKY_LINUX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-rockylinux8-gcc11
+  IMAGE_PREFIX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt
+  VERSION_SUFFIX: "0.1"
 
 jobs:
-  build:
+  # First job: discover all build.json files and create a matrix
+  discover-containers:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Discover container configurations
+        id: set-matrix
+        run: |
+          # Find all build.json files under mlir-tensorrt/.devcontainer
+          configs=()
+          for file in mlir-tensorrt/.devcontainer/*/devcontainer.json; do
+            # Skip files where the directory contains the suffix "-prebuilt"
+            if [[ -f "$file" && ! "$(dirname "$file")" =~ -prebuilt$ ]]; then
+              # Extract configuration from build.json
+              name=$(jq -r '.name' "$file")
+              base_image=$(jq -r '.build.args.BASE_IMAGE' "$file")
+              linux_distro=$(jq -r '.build.args.LINUX_DISTRO' "$file")
+              llvm_version=$(jq -r '.build.args.LLVM_VERSION' "$file")
+              dockerfile=$(jq -r '.build.dockerfile' "$file" | sed 's|\${localWorkspaceFolder}|mlir-tensorrt|g')
+
+              # Create JSON object for this configuration (compact, single-line)
+              config=$(jq -cn \
+                --arg name "$name" \
+                --arg base_image "$base_image" \
+                --arg linux_distro "$linux_distro" \
+                --arg llvm_version "$llvm_version" \
+                --arg dockerfile "$dockerfile" \
+                --arg build_json "$file" \
+                '{name: $name, base_image: $base_image, linux_distro: $linux_distro, llvm_version: $llvm_version, dockerfile: $dockerfile, build_json: $build_json}')
+              configs+=("$config")
+            fi
+          done
+
+          # Create the matrix JSON (compact output for GITHUB_OUTPUT compatibility)
+          if [[ ${#configs[@]} -eq 0 ]]; then
+            echo "No build.json files found"
+            echo "matrix={\"include\":[]}" >> $GITHUB_OUTPUT
+          else
+            matrix=$(printf '%s\n' "${configs[@]}" | jq -sc '{include: .}')
+            echo "matrix=$matrix" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Display matrix
+        run: |
+          echo "Container build matrix:"
+          echo '${{ steps.set-matrix.outputs.matrix }}' | jq .
+
+  # Second job: build and push containers based on the matrix
+  build-and-push:
+    needs: discover-containers
+    if: ${{ needs.discover-containers.outputs.matrix != '{"include":[]}' && fromJson(needs.discover-containers.outputs.matrix).include[0] != null }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.discover-containers.outputs.matrix) }}
     permissions:
       contents: read
       packages: write
-      # This is used to complete the identity challenge
-      # with sigstore/fulcio when running outside of PRs.
       id-token: write
 
     steps:
@@ -48,93 +99,42 @@ jobs:
           sudo apt-get autoremove -y
           sudo apt-get autoclean -y
 
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
       - name: Show disk usage
         run: df . -h
 
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Install the cosign tool
-      # https://github.com/sigstore/cosign-installer
-      - name: Install cosign
-        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
-        with:
-          cosign-release: 'v2.2.4'
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      # Set up BuildKit Docker container builder to be able to build
-      # multi-platform images and export cache
-      # https://github.com/docker/setup-buildx-action
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        uses: docker/setup-buildx-action@v3
 
-      # Login against a Docker registry
-      # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.IMAGE_PREFIX }}
 
-      # Build and push Docker image with Buildx
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image for Ubuntu
-        id: build-and-push-ubuntu
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+      - name: Build and push Docker image - ${{ matrix.name }}
+        id: build-and-push
+        uses: docker/build-push-action@v6
         with:
-          context: mlir-tensorrt/
-          file: mlir-tensorrt/build_tools/docker/Dockerfile
-          build-args: |
-            BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04
-            LINUX_DISTRO=ubuntu
+          context: mlir-tensorrt/build_tools/docker
+          file: ${{ matrix.dockerfile }}
           push: true
-          tags: ${{ env.TAG_UBUNTU }}
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ env.IMAGE_PREFIX }}:${{ matrix.name }}-${{ env.VERSION_SUFFIX }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      
-      - name: Build and push Docker image for Rocky Linux
-        id: build-and-push-rocky-linux
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
-        with:
-          context: mlir-tensorrt/
-          file: mlir-tensorrt/build_tools/docker/Dockerfile
           build-args: |
-            BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-rockylinux8
-            LINUX_DISTRO=rockylinux
-          push: true
-          tags: ${{ env.TAG_ROCKY_LINUX }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      # Sign the resulting Docker image digest.
-      # This will only write to the public Rekor transparency log when the Docker
-      # repository is public to avoid leaking data.  If you would like to publish
-      # transparency data even for private images, pass --force to cosign below.
-      # https://github.com/sigstore/cosign
-      - name: Sign the published Docker image Ubuntu
-        env:
-          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-          TAGS: ${{ env.TAG_UBUNTU }}
-          DIGEST: ${{ steps.build-and-push-ubuntu.outputs.digest }}
-        # This step uses the identity token to provision an ephemeral certificate
-        # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
-
-      - name: Sign the published Docker image Rocky Linux
-        env:
-          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-          TAGS: ${{ env.TAG_ROCKY_LINUX }}
-          DIGEST: ${{ steps.build-and-push-rocky-linux.outputs.digest }}
-        # This step uses the identity token to provision an ephemeral certificate
-        # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+            BASE_IMAGE=${{ matrix.base_image }}
+            LINUX_DISTRO=${{ matrix.linux_distro }}
+            LLVM_VERSION=${{ matrix.llvm_version }}