redhat-distro-container.yml

name: Build, test, and publish Red Hat Distribution Containers

on:
  pull_request:
    branches:
      - main
      - rhoai-v*
      - konflux-poc*
    types:
      - opened
      - synchronize
    paths:
      - '.github/actions/**'
      - '.github/workflows/redhat-distro-container.yml'
      - 'distribution/**'
      - 'tests/**'
  push:
    branches:
      - main
      - rhoai-v*

  # build a custom image from an arbitrary llama-stack commit
  # NOTE: workflow_dispatch intentionally skips all tests (vllm setup, postgres setup, smoke tests, integration tests)
  # This allows building images for specific SHAs even when CI is failing on other commits,
  # useful for testing specific changes without being blocked by unrelated test failures.
  workflow_dispatch:
    inputs:
      llama_stack_commit_sha:
        description: 'Llama Stack commit SHA to build from - accept long and short commit SHAs'
        required: true
        type: string

  # do a nightly test of the `main` branch of llama-stack at 6AM UTC every morning
  schedule:
    - cron: '0 6 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  REGISTRY: quay.io
  IMAGE_NAME: quay.io/opendatahub/llama-stack # tags for the image will be added dynamically

jobs:
  build-test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: ubuntu-24.04
            platform: linux/amd64
          - arch: arm64
            runner: ubuntu-24.04-arm
            platform: linux/arm64
    runs-on: ${{ matrix.runner }}
    outputs:
      distribution-changed: ${{ steps.distribution-changed.outputs.changed }}
    env:
      VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
      GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      # Model names - set defaults, overridden by MaaS configuration step
      VLLM_INFERENCE_MODEL: vllm-inference/Qwen/Qwen3-0.6B
      VERTEX_AI_INFERENCE_MODEL: vertexai/publishers/google/models/gemini-2.0-flash
      OPENAI_INFERENCE_MODEL: openai/gpt-4o-mini
      EMBEDDING_MODEL: vllm-embedding/ibm-granite/granite-embedding-125m-english
      VLLM_URL: http://localhost:8000/v1
      VLLM_EMBEDDING_URL: http://localhost:8001/v1
      LLAMA_STACK_COMMIT_SHA: ${{ github.event.inputs.llama_stack_commit_sha || 'main' }}
    permissions:
      id-token: write # for Google Cloud authentication
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 2  # Need parent commit to detect changes

      - name: Check if distribution directory changed
        id: distribution-changed
        if: github.event_name == 'push'
        run: |
          # Check if any file in the distribution directory was modified in this push
          # Use git diff instead of parsing event payload (which is unreliable for merge commits)
          if git diff --name-only HEAD^ HEAD | grep -q '^distribution/'; then
            echo "changed=true" >> "$GITHUB_OUTPUT"
            echo "distribution/ was modified in this push, will publish"
          else
            echo "changed=false" >> "$GITHUB_OUTPUT"
            echo "distribution/ was not modified in this push, skipping publish"
          fi

      - name: Install uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
        with:
          python-version: 3.12
          enable-cache: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Generate Containerfile to build an image from an arbitrary llama-stack commit (workflow_dispatch/schedule)
        if: contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
        env:
          LLAMA_STACK_VERSION: ${{ env.LLAMA_STACK_COMMIT_SHA }}
        run: |
          tmp_build_dir=$(mktemp -d)
          git clone --filter=blob:none --no-checkout https://github.com/opendatahub-io/llama-stack.git "$tmp_build_dir"
          cd "$tmp_build_dir"
          git checkout "$LLAMA_STACK_VERSION"
          python3 -m venv .venv
          source .venv/bin/activate
          pip install --no-cache -e .
          # now remove the install line from the Containerfile
          cd -
          python3 distribution/build.py
          sed -i '/^RUN pip install --no-cache llama-stack==/d' distribution/Containerfile

      - name: Build image for testing (${{ matrix.arch }})
        id: build
        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
        with:
          context: .
          file: distribution/Containerfile
          platforms: ${{ matrix.platform }}
          push: false
          tags: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }}
          load: true
          cache-from: type=gha,scope=build-${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=build-${{ matrix.arch }}

      - name: Unset cloud provider credentials for fork and Dependabot PRs (secrets not available)
        if: github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork == true || github.secret_source == 'Dependabot')
        run: |
          echo "Unsetting cloud provider credentials for fork/Dependabot PR (secrets not available)"
          {
            echo "VERTEX_AI_PROJECT="
            echo "GCP_WORKLOAD_IDENTITY_PROVIDER="
            echo "OPENAI_API_KEY="
          } >> "$GITHUB_ENV"

      - name: Configure MaaS vLLM endpoints
        env:
          _MAAS_VLLM_URL: ${{ secrets.MAAS_VLLM_URL }}
          _MAAS_VLLM_API_TOKEN: ${{ secrets.MAAS_VLLM_API_TOKEN }}
          _MAAS_EMBEDDING_URL: ${{ secrets.MAAS_EMBEDDING_URL }}
          _MAAS_EMBEDDING_API_TOKEN: ${{ secrets.MAAS_EMBEDDING_API_TOKEN }}
        run: |
          if [ -z "$_MAAS_VLLM_URL" ]; then
            echo "MaaS vLLM URL not configured, skipping"
            exit 0
          fi
          echo "Configuring MaaS vLLM endpoints for testing"
          {
            echo "VLLM_URL=$_MAAS_VLLM_URL"
            echo "VLLM_API_TOKEN=$_MAAS_VLLM_API_TOKEN"
            echo "VLLM_INFERENCE_MODEL=vllm-inference/llama-3-2-3b"
            echo "VLLM_EMBEDDING_URL=$_MAAS_EMBEDDING_URL"
            echo "VLLM_EMBEDDING_API_TOKEN=$_MAAS_EMBEDDING_API_TOKEN"
            echo "EMBEDDING_MODEL=vllm-embedding/nomic-embed-text-v1-5"
            echo "EMBEDDING_PROVIDER=vllm-embedding"
            echo "EMBEDDING_PROVIDER_MODEL_ID=nomic-embed-text-v1-5"
            echo "USING_MAAS=true"
          } >> "$GITHUB_ENV"

      - name: Authenticate to Google Cloud (Vertex)
        if: matrix.arch == 'amd64' && github.event_name != 'workflow_dispatch' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.fork == false && github.secret_source != 'Dependabot'))
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          project_id: ${{ env.VERTEX_AI_PROJECT }}
          workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}

      - name: Setup vLLM inference container
        if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
        id: vllm-inference
        uses: ./.github/actions/setup-vllm
        env:
          VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3-0.6B-granite-embedding-125m-english
          VLLM_MODE: inference

      - name: Setup vLLM embedding container
        if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true'
        id: vllm-embedding
        uses: ./.github/actions/setup-vllm
        env:
          VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3-0.6B-granite-embedding-125m-english
          VLLM_MODE: embedding

      - name: Setup PostgreSQL container
        if: github.event_name != 'workflow_dispatch'
        id: postgres
        uses: ./.github/actions/setup-postgres

      - name: Start and smoke test LLS distro image
        if: github.event_name != 'workflow_dispatch'
        id: smoke-test
        shell: bash
        run: ./tests/smoke.sh

      - name: Integration tests
        if: github.event_name != 'workflow_dispatch' && (matrix.arch == 'amd64' || env.USING_MAAS == 'true')
        id: integration-tests
        shell: bash
        run: ./tests/run_integration_tests.sh

      - name: Gather logs and debugging information
        if: always()
        shell: bash
        run: |
          # Create logs directory
          mkdir -p logs

          docker logs llama-stack > logs/llama-stack.log 2>&1 || echo "Failed to get llama-stack logs" > logs/llama-stack.log
          docker logs vllm-inference > logs/vllm-inference.log 2>&1 || echo "Failed to get vllm-inference logs" > logs/vllm-inference.log
          docker logs vllm-embedding > logs/vllm-embedding.log 2>&1 || echo "Failed to get vllm-embedding logs" > logs/vllm-embedding.log
          docker logs postgres > logs/postgres.log 2>&1 || echo "Failed to get postgres logs" > logs/postgres.log

          # Gather system information
          echo "=== System information ==="
          {
            echo "Disk usage:"
            df -h
            echo "Memory usage:"
            free -h
            echo "Docker images:"
            docker images
            echo "Docker containers:"
            docker ps -a
          } > logs/system-info.log 2>&1

          # Gather integration test logs if they exist
          echo "=== Integration test artifacts ==="
          if [ -d "/tmp/llama-stack-integration-tests" ]; then
            find /tmp/llama-stack-integration-tests -name "*.log" -o -name "pytest.log" -o -name "*.out" 2>/dev/null | while read -r file; do
              cp "$file" "logs/$(basename "$file")" || true
            done
          fi

      - name: Upload logs as artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ci-logs-${{ matrix.arch }}-${{ github.sha }}
          path: logs/
          retention-days: 7

      - name: Cleanup
        if: always()
        shell: bash
        run: |
          for c in vllm-inference vllm-embedding llama-stack postgres; do
            docker rm -f "$c" 2>/dev/null || true
          done

      # Set status so the single Notify step knows whether to send success or failure message.
      - name: Set Slack notify status
        if: failure()
        run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"

      - name: Notify Slack
        if: always() && github.event_name != 'pull_request' && failure()
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
          IMAGE_NAME: ${{ env.IMAGE_NAME }}
          IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }}
          COMMIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
          TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)

          if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
            TEXT=$(printf '%s\n%s\n%s' \
              ":failed: *Build failed for Llama Stack* - [${TIMESTAMP}]" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
          else
            TEXT=$(printf '%s\n%s\n%s\n%s' \
              ":greenchecked: *New image is available for Llama Stack* - [${TIMESTAMP}]" \
              "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT"
          fi

      - name: Output custom build information
        if: matrix.arch == 'amd64' && contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name)
        env:
          LLS_COMMIT: ${{ env.LLAMA_STACK_COMMIT_SHA }}
        run: |
          echo "Custom container image built successfully!"
          echo "Image: $IMAGE_NAME:source-$LLS_COMMIT-$GITHUB_SHA"
          echo "Llama Stack commit: $LLS_COMMIT"
          echo ""
          echo "You can pull this image using:"
          echo "docker pull $IMAGE_NAME:source-$LLS_COMMIT-$GITHUB_SHA"

  publish:
    needs: build-test
    if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && needs.build-test.outputs.distribution-changed == 'true')
    runs-on: ubuntu-latest
    env:
      LLAMA_STACK_COMMIT_SHA: ${{ github.event.inputs.llama_stack_commit_sha || 'main' }}
    permissions:
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install uv
        if: github.event_name == 'workflow_dispatch'
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
        with:
          python-version: 3.12

      - name: Generate Containerfile to build an image from an arbitrary llama-stack commit (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        env:
          LLAMA_STACK_VERSION: ${{ env.LLAMA_STACK_COMMIT_SHA }}
        run: |
          tmp_build_dir=$(mktemp -d)
          git clone --filter=blob:none --no-checkout https://github.com/opendatahub-io/llama-stack.git "$tmp_build_dir"
          cd "$tmp_build_dir"
          git checkout "$LLAMA_STACK_VERSION"
          python3 -m venv .venv
          source .venv/bin/activate
          pip install --no-cache -e .
          # now remove the install line from the Containerfile
          cd -
          python3 distribution/build.py
          sed -i '/^RUN pip install --no-cache llama-stack==/d' distribution/Containerfile

      - name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Log in to Quay.io
        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

      - name: Publish multi-arch image to Quay.io
        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
        with:
          context: .
          file: distribution/Containerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ github.event_name == 'workflow_dispatch' && format('{0}:source-{1}-{2}', env.IMAGE_NAME, env.LLAMA_STACK_COMMIT_SHA, github.sha) || format('{0}:{1}{2}', env.IMAGE_NAME, github.sha, github.ref == 'refs/heads/main' && format(',{0}:latest', env.IMAGE_NAME) || (startsWith(github.ref, 'refs/heads/rhoai-v') && format(',{0}:{1}-latest', env.IMAGE_NAME, github.ref_name)) || '') }}
          cache-from: |
            type=gha,scope=build-amd64
            type=gha,scope=build-arm64

      - name: Set Slack notify status
        if: failure()
        run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV"

      - name: Notify Slack
        if: always()
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }}
          IMAGE_NAME: ${{ env.IMAGE_NAME }}
          IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }}
          COMMIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}"
          TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)

          if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then
            TEXT=$(printf '%s\n%s\n%s' \
              ":failed: *Build failed for Llama Stack* - [${TIMESTAMP}]" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000"
          else
            TEXT=$(printf '%s\n%s\n%s\n%s' \
              ":greenchecked: *New image is available for Llama Stack* - [${TIMESTAMP}]" \
              "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \
              "Commit: ${COMMIT_SHA_SHORT}" \
              "<${WORKFLOW_URL}|View workflow run>")
            bash .github/actions/notify-slack/notify.sh "$TEXT"
          fi