[mlir-tensorrt] Update GitHub actions MLIR-TRT container build logic

Author: christopherbate

.github/workflows/post-merge-container-update-mlir-tensorrt.yml

@@ -1,33 +1,85 @@
 name: Post-merge container update mlir-tensorrt
 
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
+# This workflow builds and pushes Docker dev containers for mlir-tensorrt
+# based on the build.json configurations in .devcontainer directories.
 
 on:
   push:
-    branches: [ "main" ]
-    paths: ['mlir-tensorrt/build_tools/docker/Dockerfile', 
-      'mlir-tensorrt/python/requirements-dev.txt',
-      'mlir-tensorrt/python/requirements.txt']
+    branches: ["main", "mlir-trt-container-build-updates"]
+    paths:
+      - "mlir-tensorrt/.devcontainer/**/build.json"
+      - "mlir-tensorrt/.devcontainer/**/devcontainer.json"
+      - "mlir-tensorrt/build_tools/docker/Dockerfile"
+      - "mlir-tensorrt/build_tools/scripts/*.sh"
 
 env:
-  # Use docker.io for Docker Hub if empty
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-  TAG_UBUNTU: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-ubuntu-llvm17
-  TAG_ROCKY_LINUX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-rockylinux8-gcc11
+  IMAGE_PREFIX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt
 
 jobs:
-  build:
+  # First job: discover all build.json files and create a matrix
+  discover-containers:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Discover container configurations
+        id: set-matrix
+        run: |
+          # Find all build.json files under mlir-tensorrt/.devcontainer
+          configs=()
+          for file in mlir-tensorrt/.devcontainer/**/build.json; do
+            if [[ -f "$file" ]]; then
+              # Extract configuration from build.json
+              name=$(jq -r '.name' "$file")
+              base_image=$(jq -r '.build.args.BASE_IMAGE' "$file")
+              username=$(jq -r '.build.args.USERNAME' "$file")
+              linux_distro=$(jq -r '.build.args.LINUX_DISTRO' "$file")
+              llvm_version=$(jq -r '.build.args.LLVM_VERSION' "$file")
+              dockerfile=$(jq -r '.build.dockerfile' "$file" | sed 's|\${localWorkspaceFolder}|mlir-tensorrt|g')
+
+              # Create JSON object for this configuration (compact, single-line)
+              config=$(jq -cn \
+                --arg name "$name" \
+                --arg base_image "$base_image" \
+                --arg username "$username" \
+                --arg linux_distro "$linux_distro" \
+                --arg llvm_version "$llvm_version" \
+                --arg dockerfile "$dockerfile" \
+                --arg build_json "$file" \
+                '{name: $name, base_image: $base_image, username: $username, linux_distro: $linux_distro, llvm_version: $llvm_version, dockerfile: $dockerfile, build_json: $build_json}')
+              configs+=("$config")
+            fi
+          done
+
+          # Create the matrix JSON (compact output for GITHUB_OUTPUT compatibility)
+          if [[ ${#configs[@]} -eq 0 ]]; then
+            echo "No build.json files found"
+            echo "matrix={\"include\":[]}" >> $GITHUB_OUTPUT
+          else
+            matrix=$(printf '%s\n' "${configs[@]}" | jq -sc '{include: .}')
+            echo "matrix=$matrix" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Display matrix
+        run: |
+          echo "Container build matrix:"
+          echo '${{ steps.set-matrix.outputs.matrix }}' | jq .
+
+  # Second job: build and push containers based on the matrix
+  build-and-push:
+    needs: discover-containers
+    if: ${{ needs.discover-containers.outputs.matrix != '{"include":[]}' && fromJson(needs.discover-containers.outputs.matrix).include[0] != null }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.discover-containers.outputs.matrix) }}
     permissions:
       contents: read
       packages: write
-      # This is used to complete the identity challenge
-      # with sigstore/fulcio when running outside of PRs.
       id-token: write
 
     steps:
@@ -48,93 +100,43 @@ jobs:
           sudo apt-get autoremove -y
           sudo apt-get autoclean -y
 
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
       - name: Show disk usage
         run: df . -h
 
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      # Install the cosign tool
-      # https://github.com/sigstore/cosign-installer
-      - name: Install cosign
-        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
-        with:
-          cosign-release: 'v2.2.4'
-
-      # Set up BuildKit Docker container builder to be able to build
-      # multi-platform images and export cache
-      # https://github.com/docker/setup-buildx-action
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        uses: docker/setup-buildx-action@v3
 
-      # Login against a Docker registry
-      # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.IMAGE_PREFIX }}
 
-      # Build and push Docker image with Buildx
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image for Ubuntu
-        id: build-and-push-ubuntu
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+      - name: Build and push Docker image - ${{ matrix.name }}
+        id: build-and-push
+        uses: docker/build-push-action@v6
         with:
           context: mlir-tensorrt/
-          file: mlir-tensorrt/build_tools/docker/Dockerfile
-          build-args: |
-            BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04
-            LINUX_DISTRO=ubuntu
+          file: ${{ matrix.dockerfile }}
           push: true
-          tags: ${{ env.TAG_UBUNTU }}
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ env.IMAGE_PREFIX }}:${{ matrix.name }}-latest
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      
-      - name: Build and push Docker image for Rocky Linux
-        id: build-and-push-rocky-linux
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
-        with:
-          context: mlir-tensorrt/
-          file: mlir-tensorrt/build_tools/docker/Dockerfile
           build-args: |
-            BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-rockylinux8
-            LINUX_DISTRO=rockylinux
-          push: true
-          tags: ${{ env.TAG_ROCKY_LINUX }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      # Sign the resulting Docker image digest.
-      # This will only write to the public Rekor transparency log when the Docker
-      # repository is public to avoid leaking data.  If you would like to publish
-      # transparency data even for private images, pass --force to cosign below.
-      # https://github.com/sigstore/cosign
-      - name: Sign the published Docker image Ubuntu
-        env:
-          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-          TAGS: ${{ env.TAG_UBUNTU }}
-          DIGEST: ${{ steps.build-and-push-ubuntu.outputs.digest }}
-        # This step uses the identity token to provision an ephemeral certificate
-        # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
-
-      - name: Sign the published Docker image Rocky Linux
-        env:
-          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-          TAGS: ${{ env.TAG_ROCKY_LINUX }}
-          DIGEST: ${{ steps.build-and-push-rocky-linux.outputs.digest }}
-        # This step uses the identity token to provision an ephemeral certificate
-        # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+            BASE_IMAGE=${{ matrix.base_image }}
+            USERNAME=${{ matrix.username }}
+            LINUX_DISTRO=${{ matrix.linux_distro }}
+            LLVM_VERSION=${{ matrix.llvm_version }}

mlir-tensorrt/.devcontainer/cuda12.5-rockylinux8-gcc11/devcontainer.json

@@ -0,0 +1,17 @@
+{
+  "name": "cuda12.9-rockylinux8",
+  "build": {
+    "context": "${localWorkspaceFolder}",
+    "dockerfile": "${localWorkspaceFolder}/build_tools/docker/Dockerfile",
+    "args": {
+      "BASE_IMAGE": "nvcr.io/nvidia/cuda:12.9.1-cudnn-devel-rockylinux8",
+      "USERNAME": "nvidia",
+      "LINUX_DISTRO": "rockylinux",
+      "LLVM_VERSION": 19
+    },
+    "options": [
+      "--network",
+      "host"
+    ]
+  }
+}