Publish Artifacts #210
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Artifacts | |
| # Workflow control - runs nightly or on manual trigger | |
| on: | |
| schedule: | |
| # Run nightly at 6.30 PM UTC or 12 AM IST | |
| - cron: '30 18 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| JOBS_TO_RUN: | |
| description: 'Jobs to run (manual trigger only)' | |
| required: true | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - wheel-only | |
| - containers-only | |
| - helm-chart-only | |
| CONTAINER_TAG: | |
| description: 'Custom tag for containers (optional)' | |
| required: false | |
| default: '' | |
| ARTIFACTORY_VERSION: | |
| description: 'Artifactory version (optional, defaults to auto-generated from get_version.sh)' | |
| required: false | |
| default: '' | |
| HELM_CHART_VERSION: | |
| description: 'Helm chart version for NGC (optional, defaults to auto-generated from get_version.sh)' | |
| required: false | |
| default: '' | |
| # Container-level selection (applies when JOBS_TO_RUN is 'all' or 'containers-only') | |
| PUBLISH_RAG_SERVER: | |
| description: 'Publish rag-server container' | |
| required: false | |
| default: true | |
| type: boolean | |
| PUBLISH_INGESTOR_SERVER: | |
| description: 'Publish ingestor-server container' | |
| required: false | |
| default: true | |
| type: boolean | |
| PUBLISH_RAG_FRONTEND: | |
| description: 'Publish rag-frontend container' | |
| required: false | |
| default: true | |
| type: boolean | |
| env: | |
| RELEASE_TYPE: dev | |
| jobs: | |
| # ============================================================================ | |
| # BRANCH GUARD — scheduled runs are limited to the develop branch. | |
| # workflow_dispatch is allowed from any branch. | |
| # ============================================================================ | |
| check-branch: | |
| name: Check branch for scheduled run | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' | |
| steps: | |
| - run: echo "Branch check passed — ref ${{ github.ref }}" | |
| # ============================================================================ | |
| # PUBLISH WHEEL | |
| # ============================================================================ | |
| publish-wheel: | |
| name: Build and Publish Python Wheel | |
| runs-on: ubuntu-latest | |
| needs: check-branch | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'wheel-only' | |
| container: | |
| image: python:3.10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set artifactory version | |
| run: | | |
| if [ -n "${{ github.event.inputs.ARTIFACTORY_VERSION }}" ]; then | |
| echo "Using custom Artifactory version: ${{ github.event.inputs.ARTIFACTORY_VERSION }}" | |
| echo "ARTIFACTORY_VERSION=${{ github.event.inputs.ARTIFACTORY_VERSION }}" >> $GITHUB_ENV | |
| else | |
| echo "Using version from get_version.sh..." | |
| chmod +x ./ci/get_version.sh | |
| DEFAULT_VERSION=$(./ci/get_version.sh) | |
| echo "Generated default version: $DEFAULT_VERSION" | |
| echo "ARTIFACTORY_VERSION=$DEFAULT_VERSION" >> $GITHUB_ENV | |
| fi | |
| - name: Install dependencies | |
| run: | | |
| pip install uv==0.8.12 | |
| apt-get update && apt-get install -y wget unzip | |
| - name: Build wheel with version | |
| run: | | |
| echo "Building wheel with version: $ARTIFACTORY_VERSION" | |
| sed -i "s#^version = \".*\"#version = \"$ARTIFACTORY_VERSION\"#" pyproject.toml | |
| uv build | |
| ls -la dist/ | |
| - name: Upload wheel artifact | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: wheel-${{ env.ARTIFACTORY_VERSION }} | |
| path: dist/*.whl | |
| retention-days: 30 | |
| - name: Install NGC CLI | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| echo "Installing NGC CLI..." | |
| wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip | |
| unzip -o ngccli_linux.zip | |
| chmod u+x ngc-cli/ngc | |
| # Add NGC CLI to PATH for subsequent steps | |
| echo "$(pwd)/ngc-cli" >> $GITHUB_PATH | |
| echo "NGC CLI installed successfully" | |
| - name: Publish wheel to NGC | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| # Find the wheel file | |
| WHEEL_FILE=$(ls dist/*.whl | tail -n 1) | |
| echo "Found wheel file: $WHEEL_FILE" | |
| # Extract just the filename for display | |
| WHEEL_FILENAME=$(basename "$WHEEL_FILE") | |
| echo "Wheel filename: $WHEEL_FILENAME" | |
| # Remove existing version to overwrite (ignore error if version does not exist) | |
| ngc registry resource remove-version "nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" --org nvstaging -y 2>/dev/null || true | |
| # Publish to NGC | |
| echo "Publishing wheel to NGC: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" | |
| ngc registry resource upload-version \ | |
| "nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" \ | |
| --source "$WHEEL_FILE" \ | |
| --org nvstaging | |
| echo "Wheel published to NGC successfully" | |
| echo "NGC Resource: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" | |
| # ============================================================================ | |
| # PUBLISH RAG SERVER CONTAINER | |
| # ============================================================================ | |
| publish-rag-server: | |
| name: Build and Publish RAG Server Container | |
| runs-on: ubuntu-latest | |
| needs: check-branch | |
| if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_RAG_SERVER != 'false') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push RAG server container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building rag-server container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-server | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing rag-server to NGC Container Registry..." | |
| docker tag nvcr.io/nvidia/blueprint/rag-server:$TAG nvcr.io/nvstaging/blueprint/rag-server:$TAG | |
| docker push nvcr.io/nvstaging/blueprint/rag-server:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/rag-server:$TAG nvcr.io/nvstaging/blueprint/rag-server:latest | |
| docker push nvcr.io/nvstaging/blueprint/rag-server:latest | |
| echo "RAG server container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up rag-server Docker images..." | |
| docker images | grep "rag-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-server images to delete" | |
| docker system prune -f || true | |
| # ============================================================================ | |
| # PUBLISH INGESTOR SERVER CONTAINER | |
| # ============================================================================ | |
| publish-ingestor-server: | |
| name: Build and Publish Ingestor Server Container | |
| runs-on: ubuntu-latest | |
| needs: check-branch | |
| if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_INGESTOR_SERVER != 'false') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push ingestor server container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building ingestor-server container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-ingestor-server.yaml build ingestor-server | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing ingestor-server to NGC Container Registry..." | |
| docker tag nvcr.io/nvidia/blueprint/ingestor-server:$TAG nvcr.io/nvstaging/blueprint/ingestor-server:$TAG | |
| docker push nvcr.io/nvstaging/blueprint/ingestor-server:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/ingestor-server:$TAG nvcr.io/nvstaging/blueprint/ingestor-server:latest | |
| docker push nvcr.io/nvstaging/blueprint/ingestor-server:latest | |
| echo "Ingestor server container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up ingestor-server Docker images..." | |
| docker images | grep "ingestor-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No ingestor-server images to delete" | |
| docker system prune -f || true | |
| # ============================================================================ | |
| # PUBLISH RAG FRONTEND CONTAINER | |
| # ============================================================================ | |
| publish-rag-frontend: | |
| name: Build and Publish RAG Frontend Container | |
| runs-on: ubuntu-latest | |
| needs: check-branch | |
| if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_RAG_FRONTEND != 'false') | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Determine TAG | |
| id: tag | |
| run: | | |
| if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then | |
| echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" | |
| TAG="${{ github.event.inputs.CONTAINER_TAG }}" | |
| else | |
| echo "Using auto-generated version as TAG" | |
| VERSION=$(./ci/get_version.sh) | |
| TAG=$VERSION | |
| fi | |
| echo "TAG=$TAG" >> $GITHUB_ENV | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Final TAG value: $TAG" | |
| - name: Login to NGC Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| - name: Build and push RAG frontend container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "Building rag-frontend container with tag ${TAG}..." | |
| export TAG=${TAG} | |
| export DOWNLOAD_LEGAL_COMPLIANCE=true | |
| docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-frontend | |
| # Tag and push to NGC Container Registry | |
| echo "Pushing rag-frontend to NGC Container Registry..." | |
| docker tag nvcr.io/nvidia/blueprint/rag-frontend:$TAG nvcr.io/nvstaging/blueprint/rag-frontend:$TAG | |
| docker push nvcr.io/nvstaging/blueprint/rag-frontend:$TAG | |
| docker tag nvcr.io/nvstaging/blueprint/rag-frontend:$TAG nvcr.io/nvstaging/blueprint/rag-frontend:latest | |
| docker push nvcr.io/nvstaging/blueprint/rag-frontend:latest | |
| echo "RAG frontend container publishing completed successfully" | |
| - name: Cleanup Docker images | |
| if: always() | |
| run: | | |
| echo "Cleaning up rag-frontend Docker images..." | |
| docker images | grep "rag-frontend" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-frontend images to delete" | |
| docker system prune -f || true | |
| # ============================================================================ | |
| # PUBLISH HELM CHART TO NGC | |
| # ============================================================================ | |
| publish-helm-chart: | |
| name: Build and Publish Helm Chart to NGC | |
| runs-on: ubuntu-latest | |
| needs: check-branch | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'helm-chart-only' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Install Helm | |
| uses: azure/setup-helm@v5 | |
| with: | |
| version: 'v3.17.0' | |
| - name: Install NGC CLI | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| echo "Installing NGC CLI..." | |
| wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip | |
| unzip -o ngccli_linux.zip | |
| chmod u+x ngc-cli/ngc | |
| echo "$(pwd)/ngc-cli" >> $GITHUB_PATH | |
| echo "NGC CLI installed successfully" | |
| - name: Determine Helm chart version | |
| id: helm_version | |
| run: | | |
| if [ -n "${{ github.event.inputs.HELM_CHART_VERSION }}" ]; then | |
| echo "Using custom Helm chart version: ${{ github.event.inputs.HELM_CHART_VERSION }}" | |
| VERSION="${{ github.event.inputs.HELM_CHART_VERSION }}" | |
| else | |
| echo "Using auto-generated version from get_version.sh" | |
| chmod +x ./ci/get_version.sh | |
| VERSION=$(./ci/get_version.sh) | |
| echo "Generated version: $VERSION" | |
| fi | |
| # Helm requires strict semver. Convert Python-style version to semver: | |
| # 2026.04.14.dev0 -> 2026.4.14-dev.0 | |
| # 2026.04.14.post1 -> 2026.4.14-post.1 | |
| # 2026.04.14 -> 2026.4.14 | |
| SEMVER=$(echo "$VERSION" \ | |
| | sed -E 's/^([0-9]+)\.0*([0-9]+)\.0*([0-9]+)\.dev([0-9]+)$/\1.\2.\3-dev.\4/' \ | |
| | sed -E 's/^([0-9]+)\.0*([0-9]+)\.0*([0-9]+)\.post([0-9]+)$/\1.\2.\3-post.\4/' \ | |
| | sed -E 's/^([0-9]+)\.0*([0-9]+)\.0*([0-9]+)$/\1.\2.\3/') | |
| echo "Semver-normalised version: $SEMVER" | |
| VERSION="$SEMVER" | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| echo "HELM_CHART_VERSION=$VERSION" >> $GITHUB_ENV | |
| - name: Add Helm repositories | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| cd deploy/helm | |
| helm repo add nvidia-nim https://helm.ngc.nvidia.com/nim/nvidia/ --username='$oauthtoken' --password="$NGC_API_KEY" | |
| helm repo add nim https://helm.ngc.nvidia.com/nim/ --username='$oauthtoken' --password="$NGC_API_KEY" | |
| helm repo add nemo-microservices https://helm.ngc.nvidia.com/nvidia/nemo-microservices --username='$oauthtoken' --password="$NGC_API_KEY" | |
| helm repo add baidu-nim https://helm.ngc.nvidia.com/nim/baidu --username='$oauthtoken' --password="$NGC_API_KEY" | |
| helm repo add bitnami https://charts.bitnami.com/bitnami | |
| helm repo add elastic https://helm.elastic.co | |
| helm repo add otel https://open-telemetry.github.io/opentelemetry-helm-charts | |
| helm repo add zipkin https://zipkin.io/zipkin-helm | |
| helm repo add prometheus https://prometheus-community.github.io/helm-charts | |
| helm repo update | |
| - name: Package Helm chart | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| cd deploy/helm | |
| helm dependency update nvidia-blueprint-rag | |
| helm package nvidia-blueprint-rag/ --version "${{ env.HELM_CHART_VERSION }}" | |
| CHART_TGZ=$(ls nvidia-blueprint-rag-*.tgz) | |
| echo "Created: $CHART_TGZ" | |
| - name: Push Helm chart to NGC | |
| env: | |
| NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} | |
| run: | | |
| cd deploy/helm | |
| CHART_TGZ="nvidia-blueprint-rag-${{ env.HELM_CHART_VERSION }}.tgz" | |
| TARGET="nvstaging/blueprint/nvidia-blueprint-rag:${{ env.HELM_CHART_VERSION }}" | |
| # Remove existing version to overwrite (ignore error if version does not exist) | |
| ngc registry chart remove "$TARGET" --org nvstaging -y 2>/dev/null || true | |
| ngc registry chart push "$TARGET" --source "$CHART_TGZ" --org nvstaging | |
| echo "Helm chart published to NGC: $TARGET" |