Skip to content

ci: regenerate distribution docs in regenerate-artifacts action (#569) #2555

ci: regenerate distribution docs in regenerate-artifacts action (#569)

ci: regenerate distribution docs in regenerate-artifacts action (#569) #2555

name: Build, test, and publish Red Hat Distribution Containers
on:
pull_request:
branches:
- main
- rhoai-v*
- release-*
- konflux-poc*
types:
- opened
- synchronize
paths:
- '.github/actions/**'
- '.github/workflows/redhat-distro-container.yml'
- 'build/**'
- 'Containerfile'
- 'Containerfile.in'
- 'distribution/**'
- 'tests/**'
push:
branches:
- main
- rhoai-v*
- release-*
# build a custom image from an arbitrary ogx commit
# NOTE: workflow_dispatch intentionally skips all tests (vllm setup, postgres setup, smoke tests, integration tests)
# This allows building images for specific SHAs even when CI is failing on other commits,
# useful for testing specific changes without being blocked by unrelated test failures.
workflow_dispatch:
inputs:
ogx_commit_sha:
description: 'OGX commit SHA to build from - accept long and short commit SHAs'
required: true
type: string
# do a nightly test of the `main` branch of ogx at 6AM UTC every morning
schedule:
- cron: '0 6 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
REGISTRY: quay.io
IMAGE_NAME: quay.io/opendatahub/odh-ogx-core # tags for the image will be added dynamically
jobs:
build-test:
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: ubuntu-24.04
platform: linux/amd64
- arch: arm64
runner: ubuntu-24.04-arm
platform: linux/arm64
runs-on: ${{ matrix.runner }}
outputs:
distribution-changed: ${{ steps.distribution-changed.outputs.changed }}
env:
VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
# Model names - set defaults, overridden by MaaS configuration step
VERTEX_AI_INFERENCE_MODEL: vertexai/publishers/google/models/gemini-2.5-flash
VLLM_INFERENCE_MODEL: vllm-inference/Qwen/Qwen3.5-0.8B
OPENAI_INFERENCE_MODEL: openai/gpt-5-nano
GEMINI_INFERENCE_MODEL: gemini/models/gemini-2.5-flash
ANTHROPIC_INFERENCE_MODEL: anthropic/claude-sonnet-4-6
EMBEDDING_MODEL: vllm-embedding/ibm-granite/granite-embedding-125m-english
VLLM_URL: http://localhost:8000/v1
VLLM_EMBEDDING_URL: http://localhost:8001/v1
OGX_COMMIT_SHA: ${{ github.event.inputs.ogx_commit_sha || 'main' }}
permissions:
id-token: write # for Google Cloud authentication
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2 # Need parent commit to detect changes
- name: Check if distribution directory changed
id: distribution-changed
if: github.event_name == 'push'
run: |
# Check if any file in the distribution directory was modified in this push
# Use git diff instead of parsing event payload (which is unreliable for merge commits)
if git diff --name-only HEAD^ HEAD | grep -qE '^(distribution/|Containerfile$)'; then
echo "changed=true" >> "$GITHUB_OUTPUT"
echo "distribution/ or Containerfile was modified in this push, will publish"
else
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "distribution/ and Containerfile were not modified in this push, skipping publish"
fi
- name: Free disk space
uses: ./.github/actions/free-disk-space
- name: Install uv
uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
with:
python-version: 3.12
enable-cache: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Regenerate distribution artifacts from an arbitrary ogx commit (workflow_dispatch/schedule)
if: contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
uses: ./.github/actions/regenerate-artifacts
with:
ogx-version: ${{ env.OGX_COMMIT_SHA }}
- name: Build image for testing (${{ matrix.arch }})
id: build
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: Containerfile
platforms: ${{ matrix.platform }}
push: false
tags: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
load: true
cache-from: type=gha,scope=build-${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=build-${{ matrix.arch }}
- name: Verify OCI config labels
shell: bash
env:
IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
run: |
./tests/verify-config-label.sh "$IMAGE_TAG"
- name: Validate provider credentials
env:
_MAAS_VLLM_URL: ${{ vars.LITEMAAS_URL }}
run: |
# Fork/Dependabot PRs: secrets are unavailable
if [ "${{ github.event_name }}" = "pull_request" ]; then
if [ "${{ github.event.pull_request.head.repo.fork }}" = "true" ] || [ "${{ github.secret_source }}" = "Dependabot" ]; then
echo "::warning::Fork/Dependabot PR — unsetting all provider credentials"
{ echo "VERTEX_AI_PROJECT="; echo "GCP_WORKLOAD_IDENTITY_PROVIDER="; echo "OPENAI_API_KEY="; echo "GEMINI_API_KEY="; echo "ENABLE_GEMINI="; echo "ANTHROPIC_API_KEY="; } >> "$GITHUB_ENV"
exit 0
fi
fi
# OpenAI: validate key with a lightweight completion call (catches over-budget / invalid)
if [ -n "$OPENAI_API_KEY" ]; then
status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
-H "Authorization: Bearer $OPENAI_API_KEY" \
-H "Content-Type: application/json" \
-d '{"model":"gpt-5-nano","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}' \
https://api.openai.com/v1/chat/completions)
if [ "$status" != "200" ]; then
echo "::warning::OpenAI API returned HTTP $status, skipping OpenAI tests"
echo "OPENAI_API_KEY=" >> "$GITHUB_ENV"
fi
fi
# Gemini: validate key with a lightweight call
if [ -n "$GEMINI_API_KEY" ]; then
status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
"https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY")
if [ "$status" != "200" ]; then
echo "::warning::Gemini API returned HTTP $status, skipping Gemini tests"
{ echo "GEMINI_API_KEY="; echo "ENABLE_GEMINI="; } >> "$GITHUB_ENV"
else
echo "ENABLE_GEMINI=1" >> "$GITHUB_ENV"
fi
fi
# Vertex AI: presence check (auth validated by google-github-actions/auth step)
if [ -z "$VERTEX_AI_PROJECT" ] || [ -z "$GCP_WORKLOAD_IDENTITY_PROVIDER" ]; then
echo "::warning::Vertex AI credentials not configured, skipping Vertex AI tests"
{ echo "VERTEX_AI_PROJECT="; echo "GCP_WORKLOAD_IDENTITY_PROVIDER="; } >> "$GITHUB_ENV"
fi
# Anthropic: validate key with a lightweight call
if [ -n "$ANTHROPIC_API_KEY" ]; then
status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
-H "x-api-key: $ANTHROPIC_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-H "Content-Type: application/json" \
-d '{"model":"claude-sonnet-4-6","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}' \
https://api.anthropic.com/v1/messages)
if [ "$status" != "200" ]; then
echo "::warning::Anthropic API returned HTTP $status, skipping Anthropic tests"
echo "ANTHROPIC_API_KEY=" >> "$GITHUB_ENV"
fi
fi
# MaaS vLLM: presence check
if [ -z "$_MAAS_VLLM_URL" ]; then
echo "::warning::MaaS vLLM URL not configured, will use local vLLM"
fi
- name: Configure MaaS vLLM endpoints
env:
_MAAS_VLLM_URL: ${{ vars.LITEMAAS_URL }}
_MAAS_VLLM_API_TOKEN: ${{ secrets.LITEMAAS_API_KEY }}
_MAAS_EMBEDDING_URL: ${{ vars.LITEMAAS_URL }}
_MAAS_EMBEDDING_API_TOKEN: ${{ secrets.LITEMAAS_API_KEY }}
_MAAS_INFERENCE_MODEL: ${{ vars.LITEMAAS_INFERENCE_MODEL }}
_MAAS_EMBEDDING_MODEL: ${{ vars.LITEMAAS_EMBEDDING_MODEL }}
run: |
if [ -z "$_MAAS_VLLM_URL" ]; then
echo "MaaS vLLM URL not configured, skipping"
exit 0
fi
echo "Configuring MaaS vLLM endpoints for testing"
{
echo "VLLM_URL=$_MAAS_VLLM_URL"
echo "VLLM_API_TOKEN=$_MAAS_VLLM_API_TOKEN"
echo "VLLM_INFERENCE_MODEL=vllm-inference/$_MAAS_INFERENCE_MODEL"
echo "VLLM_EMBEDDING_URL=$_MAAS_EMBEDDING_URL"
echo "VLLM_EMBEDDING_API_TOKEN=$_MAAS_EMBEDDING_API_TOKEN"
echo "EMBEDDING_MODEL=vllm-embedding/$_MAAS_EMBEDDING_MODEL"
echo "EMBEDDING_PROVIDER=vllm-embedding"
echo "EMBEDDING_PROVIDER_MODEL_ID=$_MAAS_EMBEDDING_MODEL"
echo "USING_MAAS=true"
} >> "$GITHUB_ENV"
- name: Authenticate to Google Cloud (Vertex)
if: matrix.arch == 'amd64' && env.GCP_WORKLOAD_IDENTITY_PROVIDER != '' && github.event_name != 'workflow_dispatch' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.fork == false && github.secret_source != 'Dependabot'))
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
with:
project_id: ${{ env.VERTEX_AI_PROJECT }}
workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}
- name: Setup vLLM inference container
if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
id: vllm-inference
uses: ./.github/actions/setup-vllm
env:
VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3.5-0.8B-granite-embedding-125m-english
VLLM_MODE: inference
- name: Setup vLLM embedding container
if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
id: vllm-embedding
uses: ./.github/actions/setup-vllm
env:
VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3.5-0.8B-granite-embedding-125m-english
VLLM_MODE: embedding
- name: Setup PostgreSQL container
if: github.event_name != 'workflow_dispatch'
id: postgres
uses: ./.github/actions/setup-postgres
- name: Start and smoke test LLS distro image
if: github.event_name != 'workflow_dispatch'
id: smoke-test
shell: bash
env:
IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
run: ./tests/smoke.sh
- name: Integration tests
if: github.event_name != 'workflow_dispatch' && (matrix.arch == 'amd64' || env.USING_MAAS == 'true')
id: integration-tests
shell: bash
run: ./tests/run_integration_tests.sh
- name: Gather logs and debugging information
if: always()
shell: bash
run: |
# Create logs directory
mkdir -p logs
docker logs ogx > logs/ogx.log 2>&1 || echo "Failed to get ogx.logs" > logs/ogx.log
docker logs vllm-inference > logs/vllm-inference.log 2>&1 || echo "Failed to get vllm-inference logs" > logs/vllm-inference.log
docker logs vllm-embedding > logs/vllm-embedding.log 2>&1 || echo "Failed to get vllm-embedding logs" > logs/vllm-embedding.log
docker logs postgres > logs/postgres.log 2>&1 || echo "Failed to get postgres logs" > logs/postgres.log
# Gather system information
echo "=== System information ==="
{
echo "Disk usage:"
df -h
echo "Memory usage:"
free -h
echo "Docker images:"
docker images
echo "Docker containers:"
docker ps -a
} > logs/system-info.log 2>&1
# Gather integration test logs if they exist
echo "=== Integration test artifacts ==="
if [ -d "/tmp/ogx-integration-tests" ]; then
find /tmp/ogx-integration-tests -name "*.log" -o -name "pytest.log" -o -name "*.out" 2>/dev/null | while read -r file; do
cp "$file" "logs/$(basename "$file")" || true
done
fi
# Scrub secret values from log files before upload (CWE-532)
./tests/scrub_secrets.sh 'logs/*.log' \
VLLM_API_TOKEN VLLM_EMBEDDING_API_TOKEN \
OPENAI_API_KEY GEMINI_API_KEY ANTHROPIC_API_KEY \
POSTGRES_PASSWORD PGVECTOR_PASSWORD \
GOOGLE_APPLICATION_CREDENTIALS
- name: Upload logs as artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ci-logs-${{ matrix.arch }}-${{ github.sha }}
path: logs/
retention-days: 7
- name: Cleanup
if: always()
shell: bash
run: |
for c in vllm-inference vllm-embedding ogx postgres; do
docker rm -f "$c" 2>/dev/null || true
done
# Set status so the single Notify step knows whether to send success or failure message.
- name: Set Slack notify status
if: failure()
run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"
- name: Notify Slack
if: always() && github.event_name != 'pull_request' && failure()
env:
SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
IMAGE_NAME: ${{ env.IMAGE_NAME }}
IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
COMMIT_SHA: ${{ github.sha }}
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
TEXT=$(printf '%s\n%s\n%s' \
":failed: *Build failed for OGX* - [${TIMESTAMP}]" \
"Commit: ${COMMIT_SHA_SHORT}" \
"<${WORKFLOW_URL}|View workflow run>")
bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
else
TEXT=$(printf '%s\n%s\n%s\n%s' \
":greenchecked: *New image is available for OGX* - [${TIMESTAMP}]" \
"Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
"Commit: ${COMMIT_SHA_SHORT}" \
"<${WORKFLOW_URL}|View workflow run>")
bash .github/actions/notify-slack/notify.sh "$TEXT"
fi
- name: Output custom build information
if: matrix.arch == 'amd64' && contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
env:
OGX_COMMIT: ${{ env.OGX_COMMIT_SHA }}
run: |
echo "Custom container image built successfully!"
echo "Image: $IMAGE_NAME:source-$OGX_COMMIT-$GITHUB_SHA"
echo "OGX commit: $OGX_COMMIT"
echo ""
echo "You can pull this image using:"
echo "docker pull $IMAGE_NAME:source-$OGX_COMMIT-$GITHUB_SHA"
publish:
needs: build-test
if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && needs.build-test.outputs.distribution-changed == 'true')
runs-on: ubuntu-latest
env:
OGX_COMMIT_SHA: ${{ github.event.inputs.ogx_commit_sha || 'main' }}
permissions:
contents: read
id-token: write # for cosign keyless signing
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install uv
if: github.event_name == 'workflow_dispatch'
uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
with:
python-version: 3.12
- name: Regenerate distribution artifacts from an arbitrary ogx commit (workflow_dispatch)
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/regenerate-artifacts
with:
ogx-version: ${{ env.OGX_COMMIT_SHA }}
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Log in to Quay.io
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.QUAY_USERNAME }}
password: ${{ secrets.QUAY_PASSWORD }}
- name: Install cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Compute branch-specific image tag
if: github.event_name == 'push'
run: |
if [[ "$GITHUB_REF" == refs/heads/rhoai-v* ]]; then
echo "BRANCH_TAG=${GITHUB_REF_NAME}-latest" >> "$GITHUB_ENV"
elif [[ "$GITHUB_REF" == refs/heads/release-* ]]; then
version="${GITHUB_REF_NAME#release-}"
echo "BRANCH_TAG=rhoai-v${version}-latest" >> "$GITHUB_ENV"
fi
- name: Publish multi-arch image to Quay.io
id: push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: Containerfile
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ github.event_name == 'workflow_dispatch' && format('{0}:source-{1}-{2}', env.IMAGE_NAME, env.OGX_COMMIT_SHA, github.sha) || format('{0}:{1}{2}', env.IMAGE_NAME, github.sha, github.ref == 'refs/heads/main' && format(',{0}:latest', env.IMAGE_NAME) || (env.BRANCH_TAG != '' && format(',{0}:{1}', env.IMAGE_NAME, env.BRANCH_TAG)) || '') }}
cache-from: |
type=gha,scope=build-amd64
type=gha,scope=build-arm64
- name: Sign container image with cosign
run: cosign sign --yes "${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }}"
- name: Set Slack notify status
if: failure()
run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"
- name: Notify Slack
if: always()
env:
SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
IMAGE_NAME: ${{ env.IMAGE_NAME }}
IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
COMMIT_SHA: ${{ github.sha }}
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
TEXT=$(printf '%s\n%s\n%s' \
":failed: *Build failed for OGX* - [${TIMESTAMP}]" \
"Commit: ${COMMIT_SHA_SHORT}" \
"<${WORKFLOW_URL}|View workflow run>")
bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
else
TEXT=$(printf '%s\n%s\n%s\n%s' \
":greenchecked: *New image is available for OGX* - [${TIMESTAMP}]" \
"Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
"Commit: ${COMMIT_SHA_SHORT}" \
"<${WORKFLOW_URL}|View workflow run>")
bash .github/actions/notify-slack/notify.sh "$TEXT"
fi