Skip to content

Commit bdea629

Browse files
christopherbatechristopherbate
committed
[mlir-tensorrt] Update GitHub actions MLIR-TRT container build logic
1 parent 8b76ed8 commit bdea629

29 files changed

Lines changed: 1663 additions & 433 deletions

File tree

‎.github/workflows/post-merge-container-update-mlir-tensorrt.yml‎

Lines changed: 85 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,140 +1,125 @@
11
name: Post-merge container update mlir-tensorrt
22

3-
# This workflow uses actions that are not certified by GitHub.
4-
# They are provided by a third-party and are governed by
5-
# separate terms of service, privacy policy, and support
6-
# documentation.
3+
# This workflow builds and pushes Docker dev containers for mlir-tensorrt
4+
# based on the build.json configurations in .devcontainer directories.
75

86
on:
97
push:
10-
branches: [ "main" ]
11-
paths: ['mlir-tensorrt/build_tools/docker/Dockerfile',
12-
'mlir-tensorrt/python/requirements-dev.txt',
13-
'mlir-tensorrt/python/requirements.txt']
8+
branches: ["main", "mlir-trt-container-build-updates"]
9+
paths:
10+
- "mlir-tensorrt/.devcontainer/**/build.json"
11+
- "mlir-tensorrt/.devcontainer/**/devcontainer.json"
12+
- "mlir-tensorrt/build_tools/docker/Dockerfile"
13+
- "mlir-tensorrt/build_tools/scripts/*.sh"
1414

1515
env:
16-
# Use docker.io for Docker Hub if empty
1716
REGISTRY: ghcr.io
18-
# github.repository as <account>/<repo>
19-
IMAGE_NAME: ${{ github.repository }}
20-
TAG_UBUNTU: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-ubuntu-llvm17
21-
TAG_ROCKY_LINUX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt:cuda12.5-rockylinux8-gcc11
17+
IMAGE_PREFIX: ghcr.io/nvidia/tensorrt-incubator/mlir-tensorrt
2218

2319
jobs:
24-
build:
20+
# First job: discover all build.json files and create a matrix
21+
discover-containers:
2522
runs-on: ubuntu-latest
23+
outputs:
24+
matrix: ${{ steps.set-matrix.outputs.matrix }}
25+
steps:
26+
- name: Checkout repository
27+
uses: actions/checkout@v4
28+
29+
- name: Discover container configurations
30+
id: set-matrix
31+
run: |
32+
# Find all build.json files under mlir-tensorrt/.devcontainer
33+
configs=()
34+
for file in mlir-tensorrt/.devcontainer/**/build.json; do
35+
if [[ -f "$file" ]]; then
36+
# Extract configuration from build.json
37+
name=$(jq -r '.name' "$file")
38+
base_image=$(jq -r '.build.args.BASE_IMAGE' "$file")
39+
username=$(jq -r '.build.args.USERNAME' "$file")
40+
linux_distro=$(jq -r '.build.args.LINUX_DISTRO' "$file")
41+
llvm_version=$(jq -r '.build.args.LLVM_VERSION' "$file")
42+
dockerfile=$(jq -r '.build.dockerfile' "$file" | sed 's|\${localWorkspaceFolder}|mlir-tensorrt|g')
43+
44+
# Create JSON object for this configuration (compact, single-line)
45+
config=$(jq -cn \
46+
--arg name "$name" \
47+
--arg base_image "$base_image" \
48+
--arg username "$username" \
49+
--arg linux_distro "$linux_distro" \
50+
--arg llvm_version "$llvm_version" \
51+
--arg dockerfile "$dockerfile" \
52+
--arg build_json "$file" \
53+
'{name: $name, base_image: $base_image, username: $username, linux_distro: $linux_distro, llvm_version: $llvm_version, dockerfile: $dockerfile, build_json: $build_json}')
54+
configs+=("$config")
55+
fi
56+
done
57+
58+
# Create the matrix JSON (compact output for GITHUB_OUTPUT compatibility)
59+
if [[ ${#configs[@]} -eq 0 ]]; then
60+
echo "No build.json files found"
61+
echo "matrix={\"include\":[]}" >> $GITHUB_OUTPUT
62+
else
63+
matrix=$(printf '%s\n' "${configs[@]}" | jq -sc '{include: .}')
64+
echo "matrix=$matrix" >> $GITHUB_OUTPUT
65+
fi
66+
67+
- name: Display matrix
68+
run: |
69+
echo "Container build matrix:"
70+
echo '${{ steps.set-matrix.outputs.matrix }}' | jq .
71+
72+
# Second job: build and push containers based on the matrix
73+
build-and-push:
74+
needs: discover-containers
75+
if: ${{ needs.discover-containers.outputs.matrix != '{"include":[]}' && fromJson(needs.discover-containers.outputs.matrix).include[0] != null }}
76+
runs-on: ubuntu-latest
77+
strategy:
78+
fail-fast: false
79+
matrix: ${{ fromJson(needs.discover-containers.outputs.matrix) }}
2680
permissions:
2781
contents: read
2882
packages: write
29-
# This is used to complete the identity challenge
30-
# with sigstore/fulcio when running outside of PRs.
3183
id-token: write
3284

3385
steps:
34-
# Based on https://stackoverflow.com/q/75536771
35-
- name: Free disk space
36-
run: |
37-
sudo docker system prune -a -f
38-
sudo rm -rf \
39-
/usr/share/dotnet "$AGENT_TOOLSDIRECTORY" /usr/local/lib/android /opt/ghc \
40-
/usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
41-
/usr/lib/jvm
42-
43-
sudo apt-get purge microsoft-edge-stable || true
44-
sudo apt-get purge google-cloud-cli || true
45-
sudo apt-get purge dotnet-sdk-* || true
46-
sudo apt-get purge google-chrome-stable || true
47-
48-
sudo apt-get autoremove -y
49-
sudo apt-get autoclean -y
50-
5186
- name: Show disk usage
5287
run: df . -h
5388

5489
- name: Checkout repository
55-
uses: actions/checkout@v4
90+
uses: actions/checkout@v5
5691

57-
# Install the cosign tool
58-
# https://github.com/sigstore/cosign-installer
59-
- name: Install cosign
60-
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
61-
with:
62-
cosign-release: 'v2.2.4'
92+
- name: Set up QEMU
93+
uses: docker/setup-qemu-action@v3
6394

64-
# Set up BuildKit Docker container builder to be able to build
65-
# multi-platform images and export cache
66-
# https://github.com/docker/setup-buildx-action
6795
- name: Set up Docker Buildx
68-
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
96+
uses: docker/setup-buildx-action@v3
6997

70-
# Login against a Docker registry
71-
# https://github.com/docker/login-action
7298
- name: Log into registry ${{ env.REGISTRY }}
73-
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
99+
uses: docker/login-action@v3
74100
with:
75101
registry: ${{ env.REGISTRY }}
76102
username: ${{ github.actor }}
77103
password: ${{ secrets.GITHUB_TOKEN }}
78104

79-
# Extract metadata (tags, labels) for Docker
80-
# https://github.com/docker/metadata-action
81105
- name: Extract Docker metadata
82106
id: meta
83-
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
107+
uses: docker/metadata-action@v5
84108
with:
85-
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
109+
images: ${{ env.IMAGE_PREFIX }}
86110

87-
# Build and push Docker image with Buildx
88-
# https://github.com/docker/build-push-action
89-
- name: Build and push Docker image for Ubuntu
90-
id: build-and-push-ubuntu
91-
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
111+
- name: Build and push Docker image - ${{ matrix.name }}
112+
id: build-and-push
113+
uses: docker/build-push-action@v6
92114
with:
93115
context: mlir-tensorrt/
94-
file: mlir-tensorrt/build_tools/docker/Dockerfile
95-
build-args: |
96-
BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04
97-
LINUX_DISTRO=ubuntu
116+
file: ${{ matrix.dockerfile }}
98117
push: true
99-
tags: ${{ env.TAG_UBUNTU }}
118+
platforms: linux/amd64,linux/arm64
119+
tags: ${{ env.IMAGE_PREFIX }}:${{ matrix.name }}-latest
100120
labels: ${{ steps.meta.outputs.labels }}
101-
cache-from: type=gha
102-
cache-to: type=gha,mode=max
103-
104-
- name: Build and push Docker image for Rocky Linux
105-
id: build-and-push-rocky-linux
106-
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
107-
with:
108-
context: mlir-tensorrt/
109-
file: mlir-tensorrt/build_tools/docker/Dockerfile
110121
build-args: |
111-
BASE_IMAGE=nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-rockylinux8
112-
LINUX_DISTRO=rockylinux
113-
push: true
114-
tags: ${{ env.TAG_ROCKY_LINUX }}
115-
labels: ${{ steps.meta.outputs.labels }}
116-
cache-from: type=gha
117-
cache-to: type=gha,mode=max
118-
119-
# Sign the resulting Docker image digest.
120-
# This will only write to the public Rekor transparency log when the Docker
121-
# repository is public to avoid leaking data. If you would like to publish
122-
# transparency data even for private images, pass --force to cosign below.
123-
# https://github.com/sigstore/cosign
124-
- name: Sign the published Docker image Ubuntu
125-
env:
126-
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
127-
TAGS: ${{ env.TAG_UBUNTU }}
128-
DIGEST: ${{ steps.build-and-push-ubuntu.outputs.digest }}
129-
# This step uses the identity token to provision an ephemeral certificate
130-
# against the sigstore community Fulcio instance.
131-
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
132-
133-
- name: Sign the published Docker image Rocky Linux
134-
env:
135-
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
136-
TAGS: ${{ env.TAG_ROCKY_LINUX }}
137-
DIGEST: ${{ steps.build-and-push-rocky-linux.outputs.digest }}
138-
# This step uses the identity token to provision an ephemeral certificate
139-
# against the sigstore community Fulcio instance.
140-
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
122+
BASE_IMAGE=${{ matrix.base_image }}
123+
USERNAME=${{ matrix.username }}
124+
LINUX_DISTRO=${{ matrix.linux_distro }}
125+
LLVM_VERSION=${{ matrix.llvm_version }}

‎mlir-tensorrt/.devcontainer/cuda12.5-rockylinux8-gcc11/devcontainer.json‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎mlir-tensorrt/.devcontainer/cuda12.5-rockylinux8/build.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"build": {
3+
"context": "${localWorkspaceFolder}",
4+
"dockerfile": "${localWorkspaceFolder}/build_tools/docker/Dockerfile",
5+
"args": {
6+
"BASE_IMAGE": "nvcr.io/nvidia/cuda:12.5.1-cudnn-devel-rockylinux8",
7+
"USERNAME": "nvidia",
8+
"LINUX_DISTRO": "rockylinux",
9+
"LLVM_VERSION": 19
10+
},
11+
"options": [
12+
"--network",
13+
"host"
14+
]
15+
}
16+
}

0 commit comments

Comments
 (0)