redhat-distro-container.yml

name: Build, test, and publish Red Hat Distribution Containers

on:
  pull_request:
    branches:
      - main
      - rhoai-v*
      - release-*
      - konflux-poc*
    types:
      - opened
      - synchronize
    paths:
      - '.github/actions/**'
      - '.github/workflows/redhat-distro-container.yml'
      - 'build/**'
      - 'Containerfile'
      - 'Containerfile.in'
      - 'distribution/**'
      - 'tests/**'
  push:
    branches:
      - main
      - rhoai-v*
      - release-*

  # build a custom image from an arbitrary ogx commit
  # NOTE: workflow_dispatch intentionally skips all tests (vllm setup, postgres setup, smoke tests, integration tests)
  # This allows building images for specific SHAs even when CI is failing on other commits,
  # useful for testing specific changes without being blocked by unrelated test failures.
  workflow_dispatch:
    inputs:
      ogx_commit_sha:
        description: 'OGX commit SHA to build from - accept long and short commit SHAs'
        required: true
        type: string

  # do a nightly test of the `main` branch of ogx at 6AM UTC every morning
  schedule:
    - cron: '0 6 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  REGISTRY: quay.io
  IMAGE_NAME: quay.io/opendatahub/odh-ogx-core # tags for the image will be added dynamically

jobs:
  build-test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: ubuntu-24.04
            platform: linux/amd64
          - arch: arm64
            runner: ubuntu-24.04-arm
            platform: linux/arm64
    runs-on: ${{ matrix.runner }}
    outputs:
      distribution-changed: ${{ steps.distribution-changed.outputs.changed }}
    env:
      VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
      GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      # Model names - set defaults, overridden by MaaS configuration step
      VERTEX_AI_INFERENCE_MODEL: vertexai/publishers/google/models/gemini-2.5-flash
      VLLM_INFERENCE_MODEL: vllm-inference/Qwen/Qwen3.5-0.8B
      OPENAI_INFERENCE_MODEL: openai/gpt-5-nano
      GEMINI_INFERENCE_MODEL: gemini/models/gemini-2.5-flash
      ANTHROPIC_INFERENCE_MODEL: anthropic/claude-sonnet-4-6
      EMBEDDING_MODEL: vllm-embedding/ibm-granite/granite-embedding-125m-english
      VLLM_URL: http://localhost:8000/v1
      VLLM_EMBEDDING_URL: http://localhost:8001/v1
      OGX_COMMIT_SHA: ${{ github.event.inputs.ogx_commit_sha || 'main' }}
    permissions:
      id-token: write # for Google Cloud authentication
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 2  # Need parent commit to detect changes

      - name: Check if distribution directory changed
        id: distribution-changed
        if: github.event_name == 'push'
        run: |
          # Check if any file in the distribution directory was modified in this push
          # Use git diff instead of parsing event payload (which is unreliable for merge commits)
          if git diff --name-only HEAD^ HEAD | grep -qE '^(distribution/|Containerfile$)'; then
            echo "changed=true" >> "$GITHUB_OUTPUT"
            echo "distribution/ or Containerfile was modified in this push, will publish"
          else
            echo "changed=false" >> "$GITHUB_OUTPUT"
            echo "distribution/ and Containerfile were not modified in this push, skipping publish"
          fi

      - name: Free disk space
        uses: ./.github/actions/free-disk-space

      - name: Install uv
        uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
        with:
          python-version: 3.12
          enable-cache: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: Regenerate distribution artifacts from an arbitrary ogx commit (workflow_dispatch/schedule)
        if: contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
        uses: ./.github/actions/regenerate-artifacts
        with:
          ogx-version: ${{ env.OGX_COMMIT_SHA }}

      - name: Build image for testing (${{ matrix.arch }})
        id: build
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
        with:
          context: .
          file: Containerfile
          platforms: ${{ matrix.platform }}
          push: false
          tags: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
          load: true
          cache-from: type=gha,scope=build-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=build-${{ matrix.arch }}

      - name: Verify OCI config labels
        shell: bash
        env:
          IMAGE_TAG: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
        run: |
          ./tests/verify-config-label.sh "$IMAGE_TAG"

      - name: Validate provider credentials
        env:
          _MAAS_VLLM_URL: ${{ vars.LITEMAAS_URL }}
        run: |
          # Fork/Dependabot PRs: secrets are unavailable
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            if [ "${{ github.event.pull_request.head.repo.fork }}" = "true" ] || [ "${{ github.secret_source }}" = "Dependabot" ]; then
              echo "::warning::Fork/Dependabot PR — unsetting all provider credentials"
              { echo "VERTEX_AI_PROJECT="; echo "GCP_WORKLOAD_IDENTITY_PROVIDER="; echo "OPENAI_API_KEY="; echo "GEMINI_API_KEY="; echo "ENABLE_GEMINI="; echo "ANTHROPIC_API_KEY="; } >> "$GITHUB_ENV"
              exit 0
            fi
          fi

          # OpenAI: validate key with a lightweight completion call (catches over-budget / invalid)
          if [ -n "$OPENAI_API_KEY" ]; then
            status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
              -H "Authorization: Bearer $OPENAI_API_KEY" \
              -H "Content-Type: application/json" \
              -d '{"model":"gpt-5-nano","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}' \
              https://api.openai.com/v1/chat/completions)
            if [ "$status" != "200" ]; then
              echo "::warning::OpenAI API returned HTTP $status, skipping OpenAI tests"
              echo "OPENAI_API_KEY=" >> "$GITHUB_ENV"
            fi
          fi

          # Gemini: validate key with a lightweight call
          if [ -n "$GEMINI_API_KEY" ]; then
            status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
              "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY")
            if [ "$status" != "200" ]; then
              echo "::warning::Gemini API returned HTTP $status, skipping Gemini tests"
              { echo "GEMINI_API_KEY="; echo "ENABLE_GEMINI="; } >> "$GITHUB_ENV"
            else
              echo "ENABLE_GEMINI=1" >> "$GITHUB_ENV"
            fi
          fi

          # Vertex AI: presence check (auth validated by google-github-actions/auth step)
          if [ -z "$VERTEX_AI_PROJECT" ] || [ -z "$GCP_WORKLOAD_IDENTITY_PROVIDER" ]; then
            echo "::warning::Vertex AI credentials not configured, skipping Vertex AI tests"
            { echo "VERTEX_AI_PROJECT="; echo "GCP_WORKLOAD_IDENTITY_PROVIDER="; } >> "$GITHUB_ENV"
          fi

          # Anthropic: validate key with a lightweight call
          if [ -n "$ANTHROPIC_API_KEY" ]; then
            status=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
              -H "x-api-key: $ANTHROPIC_API_KEY" \
              -H "anthropic-version: 2023-06-01" \
              -H "Content-Type: application/json" \
              -d '{"model":"claude-sonnet-4-6","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}' \
              https://api.anthropic.com/v1/messages)
            if [ "$status" != "200" ]; then
              echo "::warning::Anthropic API returned HTTP $status, skipping Anthropic tests"
              echo "ANTHROPIC_API_KEY=" >> "$GITHUB_ENV"
            fi
          fi

          # MaaS vLLM: presence check
          if [ -z "$_MAAS_VLLM_URL" ]; then
            echo "::warning::MaaS vLLM URL not configured, will use local vLLM"
          fi

      - name: Configure MaaS vLLM endpoints
        env:
          _MAAS_VLLM_URL: ${{ vars.LITEMAAS_URL }}
          _MAAS_VLLM_API_TOKEN: ${{ secrets.LITEMAAS_API_KEY }}
          _MAAS_EMBEDDING_URL: ${{ vars.LITEMAAS_URL }}
          _MAAS_EMBEDDING_API_TOKEN: ${{ secrets.LITEMAAS_API_KEY }}
          _MAAS_INFERENCE_MODEL: ${{ vars.LITEMAAS_INFERENCE_MODEL }}
          _MAAS_EMBEDDING_MODEL: ${{ vars.LITEMAAS_EMBEDDING_MODEL }}
        run: |
          if [ -z "$_MAAS_VLLM_URL" ]; then
            echo "MaaS vLLM URL not configured, skipping"
            exit 0
          fi
          echo "Configuring MaaS vLLM endpoints for testing"
          {
            echo "VLLM_URL=$_MAAS_VLLM_URL"
            echo "VLLM_API_TOKEN=$_MAAS_VLLM_API_TOKEN"
            echo "VLLM_INFERENCE_MODEL=vllm-inference/$_MAAS_INFERENCE_MODEL"
            echo "VLLM_EMBEDDING_URL=$_MAAS_EMBEDDING_URL"
            echo "VLLM_EMBEDDING_API_TOKEN=$_MAAS_EMBEDDING_API_TOKEN"
            echo "EMBEDDING_MODEL=vllm-embedding/$_MAAS_EMBEDDING_MODEL"
            echo "EMBEDDING_PROVIDER=vllm-embedding"
            echo "EMBEDDING_PROVIDER_MODEL_ID=$_MAAS_EMBEDDING_MODEL"
            echo "USING_MAAS=true"
          } >> "$GITHUB_ENV"

      - name: Authenticate to Google Cloud (Vertex)
        if: matrix.arch == 'amd64' && env.GCP_WORKLOAD_IDENTITY_PROVIDER != '' && github.event_name != 'workflow_dispatch' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.fork == false && github.secret_source != 'Dependabot'))
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          project_id: ${{ env.VERTEX_AI_PROJECT }}
          workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}

      - name: Setup vLLM inference container
        if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
        id: vllm-inference
        uses: ./.github/actions/setup-vllm
        env:
          VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3.5-0.8B-granite-embedding-125m-english
          VLLM_MODE: inference

      - name: Setup vLLM embedding container
        if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
        id: vllm-embedding
        uses: ./.github/actions/setup-vllm
        env:
          VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3.5-0.8B-granite-embedding-125m-english
          VLLM_MODE: embedding

      - name: Setup PostgreSQL container
        if: github.event_name != 'workflow_dispatch'
        id: postgres
        uses: ./.github/actions/setup-postgres

      - name: Start and smoke test LLS distro image
        if: github.event_name != 'workflow_dispatch'
        id: smoke-test
        shell: bash
        env:
          IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
        run: ./tests/smoke.sh

      - name: Integration tests
        if: github.event_name != 'workflow_dispatch' && (matrix.arch == 'amd64' || env.USING_MAAS == 'true')
        id: integration-tests
        shell: bash
        run: ./tests/run_integration_tests.sh

      - name: Gather logs and debugging information
        if: always()
        shell: bash
        run: |
          # Create logs directory
          mkdir -p logs

          docker logs ogx > logs/ogx.log 2>&1 || echo "Failed to get ogx.logs" > logs/ogx.log
          docker logs vllm-inference > logs/vllm-inference.log 2>&1 || echo "Failed to get vllm-inference logs" > logs/vllm-inference.log
          docker logs vllm-embedding > logs/vllm-embedding.log 2>&1 || echo "Failed to get vllm-embedding logs" > logs/vllm-embedding.log
          docker logs postgres > logs/postgres.log 2>&1 || echo "Failed to get postgres logs" > logs/postgres.log

          # Gather system information
          echo "=== System information ==="
          {
            echo "Disk usage:"
            df -h
            echo "Memory usage:"
            free -h
            echo "Docker images:"
            docker images
            echo "Docker containers:"
            docker ps -a
          } > logs/system-info.log 2>&1

          # Gather integration test logs if they exist
          echo "=== Integration test artifacts ==="
          if [ -d "/tmp/ogx-integration-tests" ]; then
            find /tmp/ogx-integration-tests -name "*.log" -o -name "pytest.log" -o -name "*.out" 2>/dev/null | while read -r file; do
              cp "$file" "logs/$(basename "$file")" || true
            done
          fi

          # Scrub secret values from log files before upload (CWE-532)
          ./tests/scrub_secrets.sh 'logs/*.log' \
            VLLM_API_TOKEN VLLM_EMBEDDING_API_TOKEN \
            OPENAI_API_KEY GEMINI_API_KEY ANTHROPIC_API_KEY \
            POSTGRES_PASSWORD PGVECTOR_PASSWORD \
            GOOGLE_APPLICATION_CREDENTIALS

      - name: Upload logs as artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ci-logs-${{ matrix.arch }}-${{ github.sha }}
          path: logs/
          retention-days: 7

      - name: Cleanup
        if: always()
        shell: bash
        run: |
          for c in vllm-inference vllm-embedding ogx postgres; do
            docker rm -f "$c" 2>/dev/null || true
          done

      # Set status so the single Notify step knows whether to send success or failure message.
      - name: Set Slack notify status
        if: failure()
        run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"

      - name: Notify Slack
        if: always() && github.event_name != 'pull_request' && failure()
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
          IMAGE_NAME: ${{ env.IMAGE_NAME }}
          IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
          COMMIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
          TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)

          if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
            TEXT=$(printf '%s\n%s\n%s' \
              ":failed: *Build failed for OGX* - [${TIMESTAMP}]" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
          else
            TEXT=$(printf '%s\n%s\n%s\n%s' \
              ":greenchecked: *New image is available for OGX* - [${TIMESTAMP}]" \
              "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT"
          fi

      - name: Output custom build information
        if: matrix.arch == 'amd64' && contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
        env:
          OGX_COMMIT: ${{ env.OGX_COMMIT_SHA }}
        run: |
          echo "Custom container image built successfully!"
          echo "Image: $IMAGE_NAME:source-$OGX_COMMIT-$GITHUB_SHA"
          echo "OGX commit: $OGX_COMMIT"
          echo ""
          echo "You can pull this image using:"
          echo "docker pull $IMAGE_NAME:source-$OGX_COMMIT-$GITHUB_SHA"

  publish:
    needs: build-test
    if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && needs.build-test.outputs.distribution-changed == 'true')
    runs-on: ubuntu-latest
    env:
      OGX_COMMIT_SHA: ${{ github.event.inputs.ogx_commit_sha || 'main' }}
    permissions:
      contents: read
      id-token: write # for cosign keyless signing
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        if: github.event_name == 'workflow_dispatch'
        uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
        with:
          python-version: 3.12

      - name: Regenerate distribution artifacts from an arbitrary ogx commit (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/regenerate-artifacts
        with:
          ogx-version: ${{ env.OGX_COMMIT_SHA }}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: Log in to Quay.io
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

      - name: Install cosign
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2

      - name: Compute branch-specific image tag
        if: github.event_name == 'push'
        run: |
          if [[ "$GITHUB_REF" == refs/heads/rhoai-v* ]]; then
            echo "BRANCH_TAG=${GITHUB_REF_NAME}-latest" >> "$GITHUB_ENV"
          elif [[ "$GITHUB_REF" == refs/heads/release-* ]]; then
            version="${GITHUB_REF_NAME#release-}"
            echo "BRANCH_TAG=rhoai-v${version}-latest" >> "$GITHUB_ENV"
          fi

      - name: Publish multi-arch image to Quay.io
        id: push
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
        with:
          context: .
          file: Containerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ github.event_name == 'workflow_dispatch' && format('{0}:source-{1}-{2}', env.IMAGE_NAME, env.OGX_COMMIT_SHA, github.sha) || format('{0}:{1}{2}', env.IMAGE_NAME, github.sha, github.ref == 'refs/heads/main' && format(',{0}:latest', env.IMAGE_NAME) || (env.BRANCH_TAG != '' && format(',{0}:{1}', env.IMAGE_NAME, env.BRANCH_TAG)) || '') }}
          cache-from: |
            type=gha,scope=build-amd64
            type=gha,scope=build-arm64

      - name: Sign container image with cosign
        run: cosign sign --yes "${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }}"

      - name: Set Slack notify status
        if: failure()
        run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"

      - name: Notify Slack
        if: always()
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
          IMAGE_NAME: ${{ env.IMAGE_NAME }}
          IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && format('source-{0}-{1}', env.OGX_COMMIT_SHA, github.sha) || github.sha }}
          COMMIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
          TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)

          if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
            TEXT=$(printf '%s\n%s\n%s' \
              ":failed: *Build failed for OGX* - [${TIMESTAMP}]" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
          else
            TEXT=$(printf '%s\n%s\n%s\n%s' \
              ":greenchecked: *New image is available for OGX* - [${TIMESTAMP}]" \
              "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT"
          fi