Skip to content
CI Pipeline

ci(skills-eval): wire schedule to MANUAL_FULL_SWEEP + sync dispatch p… #2892

Sign in to view logs

ci(skills-eval): wire schedule to MANUAL_FULL_SWEEP + sync dispatch p…

ci(skills-eval): wire schedule to MANUAL_FULL_SWEEP + sync dispatch p… #2892

Workflow file for this run

.github/workflows/ci-pipeline.yml at 34cf9c8
name: CI Pipeline
# Workflow control - mirroring GitLab workflow rules
on:
# Run on PRs (including forks) - safe jobs without secrets run for all, secret jobs only for same-repo PRs
pull_request:
# Run on push only for default branch (e.g. after merge); avoids duplicate runs when pushing to a PR branch
push:
branches:
- develop
- main
- 'release-**'
workflow_dispatch:
schedule:
# Run nightly at 7.30 PM UTC or 1 AM IST
- cron: '30 19 * * *'
env:
# Common environment variables
MILVUS_VERSION: v2.6.5
NV_INGEST_MAX_UTIL: 8
APP_VECTORSTORE_ENABLEGPUSEARCH: False
APP_VECTORSTORE_ENABLEGPUINDEX: False
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
# ============================================================================
# TEST STAGE JOBS
# ============================================================================
helm-blueprint-compliance:
name: Helm Blueprint Compliance
runs-on: ubuntu-latest
# Only run if push to develop OR PR from same repo (not fork) - needs secrets
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.pull_request.head.repo.full_name == github.repository
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup Helm
uses: azure/setup-helm@v5
with:
version: 'latest'
- name: Add Helm repositories
env:
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
helm repo add nvidia-nim https://helm.ngc.nvidia.com/nim/nvidia/ --username='$oauthtoken' --password=$NGC_API_KEY
helm repo add nim https://helm.ngc.nvidia.com/nim/ --username='$oauthtoken' --password=$NGC_API_KEY
helm repo add nemo-microservices https://helm.ngc.nvidia.com/nvidia/nemo-microservices --username='$oauthtoken' --password=$NGC_API_KEY
helm repo add baidu-nim https://helm.ngc.nvidia.com/nim/baidu --username='$oauthtoken' --password=$NGC_API_KEY
helm repo add nvstaging-nim https://helm.ngc.nvidia.com/nvstaging/blueprint --username='$oauthtoken' --password=$CI_NVSTAGING_BLUEPRINT_KEY
helm repo update
- name: Run Helm Blueprint Compliance
run: |
# Add your helm blueprint compliance checks here
echo "Running Helm Blueprint Compliance checks..."
# The actual compliance command would depend on the blueprint compliance tool
lint:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v5
- uses: actions/setup-python@v3
- uses: pre-commit/action@v3.0.1
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
container:
image: python:3.12-slim
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install system dependencies
run: |
apt-get update && apt-get install -y gcc curl && rm -rf /var/lib/apt/lists/*
- name: Install package with dependencies
run: |
# Install uv (Python package and environment manager)
curl -LsSf https://astral.sh/uv/install.sh | sh
export PATH="$HOME/.local/bin:$PATH"
# Create a fresh virtual environment using uv
rm -rf venv-unit || echo "No existing venv-unit to clean up"
uv venv venv-unit
. venv-unit/bin/activate
uv pip install -e .[all]
uv pip install --no-cache -r tests/unit/requirements-test.txt
# rag-perf is a separate package under scripts/rag-perf/. Install its
# runtime deps into the same venv so tests/unit/test_rag_perf/ can
# import rag_perf.config (which pulls ruamel.yaml, click, pydantic).
uv pip install -e ./scripts/rag-perf
- name: Run unit tests with coverage
run: |
. venv-unit/bin/activate
python -m pytest -v -s --cov=src --cov-report=term-missing tests/unit --ignore=tests/unit/test_ingestor_server/test_nemo_retriever --ignore=tests/unit/test_utils/test_vdb/test_lancedb_vdb.py
frontend-unit-tests:
name: Frontend Unit Tests
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Sanitize branch name
id: sanitize
run: |
SANITIZED_REF="${GITHUB_REF_NAME//\//-}"
echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10
- name: Cache frontend dependencies
uses: actions/cache@v4
with:
path: |
frontend/node_modules
frontend/.pnpm-store
key: frontend-deps-${{ github.ref_name }}
restore-keys: |
frontend-deps-
- name: Configure pnpm and install dependencies
working-directory: frontend
run: |
pnpm config set store-dir .pnpm-store
pnpm install --frozen-lockfile || pnpm install
- name: Run frontend unit tests with coverage
working-directory: frontend
run: |
pnpm test:coverage
- name: Upload coverage artifacts
uses: actions/upload-artifact@v5
if: always()
with:
name: frontend-coverage-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }}
path: frontend/coverage/
retention-days: 7
check-markdown-links:
name: Check Markdown Links
runs-on: ubuntu-latest
container:
image: python:3.12-slim
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install required packages
run: |
pip install --no-cache-dir requests
- name: Run markdown link checker
run: |
python ci/check_markdown_links.py --root . --no-external
# ============================================================================
# INTEGRATION TESTS STAGE
# ============================================================================
integration-tests:
name: Integration Tests
runs-on: arc-runners-org-nvidia-ai-bp-2-gpu
# Only run if push to develop OR PR from same repo (not fork) - needs secrets
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.pull_request.head.repo.full_name == github.repository
# Multimodal query sequence uses NVIDIA API catalog cloud endpoints; works with Elasticsearch VDB.
env:
ENABLE_MULTIMODAL_QUERY_CI: "true"
# NeMo Retriever Library (NRL) + LanceDB integration sequences (set to "true" to run in CI).
ENABLE_NRL_INTEGRATION_TESTS: "false"
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install NGC CLI
env:
NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Installing NGC CLI..."
wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip
unzip -o ngccli_linux.zip
chmod u+x ngc-cli/ngc
# Add NGC CLI to PATH for subsequent steps
echo "$(pwd)/ngc-cli" >> $GITHUB_PATH
echo "NGC CLI installed successfully"
- name: Download test data files
env:
NGC_API_KEY: ${{ secrets.CI_NV_RAG_BLUEPRINT_KEY }}
run: |
echo "Downloading test data files to tests/data..."
# Create tests/data directory if it doesn't exist
mkdir -p tests/data
# Download integration test dataset from NGC
echo "Downloading integration_test_dataset:2.4.0 from NGC..."
ngc registry resource download-version "0648981100760671/integration_test_dataset:2.4.0" --dest ./tests/data --org 0648981100760671
# Move files from subdirectory to tests/data root
echo "Moving files to tests/data root directory..."
if [ -d "tests/data/integration_test_dataset_v2.4.0" ]; then
mv tests/data/integration_test_dataset_v2.4.0/* tests/data/
rmdir tests/data/integration_test_dataset_v2.4.0
echo "Files moved successfully"
else
echo "Warning: integration_test_dataset_v2.4.0 directory not found"
fi
# Verify downloads
echo "Files in tests/data:"
ls -lh tests/data/
echo "Test data download completed"
- name: Docker info
run: docker info
- name: Clean up existing containers
run: |
echo "Cleaning up existing containers..."
docker ps -a
docker stop $(docker ps -aq) || true
docker rm $(docker ps -aq) || true
- name: Load common environment variables
run: |
echo "Loading common environment variables..."
export TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^a-zA-Z0-9]/-/g')-${GITHUB_SHA::7}
export NGC_API_KEY=${{ secrets.NGC_API_KEY }}
echo "TAG=$TAG" >> $GITHUB_ENV
echo "NGC_API_KEY=$NGC_API_KEY" >> $GITHUB_ENV
# Persistent data lives in the `rag-vol-*` named Docker volumes
# (auto-created by compose). Ensure fresh volumes per CI run.
docker volume ls -q --filter "name=^rag-vol-" | xargs -r docker volume rm -f || true
# Load nvdev.env and export all variables to GITHUB_ENV
if [ -f ./deploy/compose/nvdev.env ]; then
set -a
source ./deploy/compose/nvdev.env
set +a
# Export all variables from nvdev.env to GITHUB_ENV
grep -E '^export ' ./deploy/compose/nvdev.env | sed 's/export //' | while IFS='=' read -r key value; do
# Evaluate the value to expand any variable references
eval "resolved_value=\"$value\""
echo "$key=$resolved_value" >> $GITHUB_ENV
done
fi
# Default vector DB in CI: Elasticsearch (matches docker-compose defaults)
echo "APP_VECTORSTORE_URL=http://elasticsearch:9200" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=elasticsearch" >> $GITHUB_ENV
echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV
echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV
- name: Docker login
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "$CI_NVSTAGING_BLUEPRINT_KEY" | docker login nvcr.io -u '$oauthtoken' --password-stdin
- name: Start services
run: |
echo "Starting vector database services (Elasticsearch is default VDB)..."
# Clear volumes before bringing up Elasticsearch so no stale index/data from prior runs
docker compose -f tests/integration/vectordb.yaml --profile milvus down -v || true
docker compose -f tests/integration/vectordb.yaml down -v || true
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch up -d
echo "Waiting for Elasticsearch..."
ES_UP=0
for i in $(seq 1 60); do
if curl -sf "http://localhost:9200/_cluster/health" >/dev/null; then
echo "Elasticsearch is up (attempt $i)"
ES_UP=1
break
fi
echo "Waiting for Elasticsearch... ($i/60)"
sleep 5
done
if [ "$ES_UP" -ne 1 ]; then
echo "::error::Elasticsearch did not become reachable within the timeout"
docker ps -a
docker logs --tail 200 elasticsearch 2>&1 || true
exit 1
fi
curl -sS "http://localhost:9200/_cluster/health?pretty" || true
echo "Starting RAG server..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
echo "Starting ingestor server..."
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
echo "Waiting for services to be ready..."
sleep 60
echo "Checking service status..."
docker ps
- name: Print logs for running containers
run: |
echo "=== LOGS FOR RUNNING CONTAINERS ==="
docker logs --tail 50 elasticsearch || echo "No logs for elasticsearch"
docker logs --tail 50 seaweedfs || echo "No logs for seaweedfs"
docker logs rag-server || echo "No logs for rag-server"
docker logs ingestor-server || echo "No logs for ingestor-server"
echo "Deploy stage completed successfully"
# ========================================================================
# BASIC TESTS
# ========================================================================
- name: Setup Python environment for tests
run: |
# Ensure required Python tooling is installed before use
if command -v apt-get >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y python3 python3-venv python3-pip fuse lsof
fi
python3 --version || echo "Python3 not found"
pip3 --version || echo "pip3 not found"
# Install uv (Python package and environment manager)
curl -LsSf https://astral.sh/uv/install.sh | sh
export PATH="$HOME/.local/bin:$PATH"
# Create a fresh virtual environment using uv
rm -rf venv || echo "No existing venv to clean up"
uv venv venv
source venv/bin/activate
uv pip install -e .[all]
uv pip install -r tests/integration/requirements.txt
- name: Run basic integration tests
id: basic-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running basic integration tests..."
python -m tests.integration.main --timeout 1200
echo "Basic integration tests completed"
- name: Collect logs after basic tests
if: always()
run: |
echo "Collecting container logs..."
mkdir -p logs/basic-tests
docker logs rag-server > logs/basic-tests/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/basic-tests/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/basic-tests/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/basic-tests/ 2>/dev/null || true
# ========================================================================
# MILVUS VECTOR DATABASE TESTS (dedicated Milvus sequence)
# All other integration suites use Elasticsearch (default VDB).
# ========================================================================
- name: Configure environment for Milvus sequence tests
run: |
echo "APP_VECTORSTORE_URL=http://milvus:19530" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=milvus" >> $GITHUB_ENV
echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV
echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV
- name: Restart vector database for Milvus
run: |
echo "Switching vector database from Elasticsearch to Milvus..."
docker compose -f tests/integration/vectordb.yaml down -v || true
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch down -v || true
docker compose -f tests/integration/vectordb.yaml --profile milvus up -d
echo "Waiting for Milvus to be ready..."
sleep 60
docker ps
docker logs --tail 80 milvus-standalone || true
- name: Restart services for Milvus sequence tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Relaunching RAG and ingestor with Milvus..."
export APP_VECTORSTORE_URL=http://milvus:19530
export APP_VECTORSTORE_NAME=milvus
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run Milvus sequence integration tests
id: milvus-sequence-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running Milvus sequence integration tests (sequence milvus)..."
python -m tests.integration.main --sequence milvus
echo "Milvus sequence integration tests completed"
- name: Collect logs after Milvus sequence tests
if: always()
run: |
mkdir -p logs/milvus-sequence
docker logs milvus-standalone > logs/milvus-sequence/milvus.log 2>&1 || true
docker logs milvus-etcd > logs/milvus-sequence/etcd.log 2>&1 || true
docker logs seaweedfs > logs/milvus-sequence/seaweedfs.log 2>&1 || true
docker logs rag-server > logs/milvus-sequence/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/milvus-sequence/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/milvus-sequence/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/milvus-sequence/ 2>/dev/null || true
- name: Restore Elasticsearch stack after Milvus sequence tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restoring Elasticsearch for remaining integration tests..."
docker compose -f tests/integration/vectordb.yaml down -v || true
docker compose -f tests/integration/vectordb.yaml --profile milvus down -v || true
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch down -v || true
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch up -d
echo "Waiting for Elasticsearch to become reachable..."
ES_UP=0
for i in $(seq 1 60); do
if curl -sf "http://localhost:9200/_cluster/health" >/dev/null; then
echo "Elasticsearch is up (attempt $i)"
ES_UP=1
break
fi
echo "Waiting for Elasticsearch... ($i/60)"
if [ "$((i % 5))" -eq 0 ] || [ "$i" -eq 1 ]; then
echo "=== Elasticsearch debug snapshot (attempt $i) ==="
docker ps -a --filter name=elasticsearch --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || true
docker inspect elasticsearch --format 'Status={{.State.Status}} ExitCode={{.State.ExitCode}} OOMKilled={{.State.OOMKilled}} Error={{.State.Error}}' 2>/dev/null || echo "inspect: container not found"
echo "--- docker logs elasticsearch (last 100 lines) ---"
docker logs --tail 100 elasticsearch 2>&1 || echo "Could not read elasticsearch logs"
echo "--- /usr/share/elasticsearch/logs/docker-cluster.log (last 80 lines) ---"
docker exec elasticsearch tail -n 80 /usr/share/elasticsearch/logs/docker-cluster.log 2>&1 || echo "(no docker-cluster.log yet or exec failed)"
echo "=== end snapshot ==="
fi
sleep 5
done
if [ "$ES_UP" -ne 1 ]; then
echo "::error::Elasticsearch did not become reachable within the timeout"
docker ps -a
echo "--- docker logs elasticsearch (last 250 lines) ---"
docker logs --tail 250 elasticsearch 2>&1 || true
echo "--- /usr/share/elasticsearch/logs/docker-cluster.log (last 200 lines) ---"
docker exec elasticsearch tail -n 200 /usr/share/elasticsearch/logs/docker-cluster.log 2>&1 || true
exit 1
fi
curl -sS "http://localhost:9200/_cluster/health?pretty" || true
docker ps
echo "APP_VECTORSTORE_URL=http://elasticsearch:9200" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=elasticsearch" >> $GITHUB_ENV
echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV
echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV
export APP_VECTORSTORE_URL=http://elasticsearch:9200
export APP_VECTORSTORE_NAME=elasticsearch
echo "Relaunching RAG and ingestor with Elasticsearch..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
# ========================================================================
# QUERY REWRITER TESTS
# ========================================================================
- name: Configure environment for query rewriter tests
run: |
echo "ENABLE_QUERYREWRITER=True" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=5" >> $GITHUB_ENV
- name: Restart services for query rewriter tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with query rewriter configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run query rewriter integration tests
id: query-rewriter-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running query rewriter integration tests..."
python -m tests.integration.main --sequence query_rewriter
echo "Query rewriter integration tests completed"
- name: Collect logs after query rewriter tests
if: always()
run: |
mkdir -p logs/query-rewriter
docker logs rag-server > logs/query-rewriter/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/query-rewriter/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/query-rewriter/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/query-rewriter/ 2>/dev/null || true
# ========================================================================
# REFLECTION TESTS
# ========================================================================
- name: Configure environment for reflection tests
run: |
echo "ENABLE_REFLECTION=True" >> $GITHUB_ENV
echo "RESPONSE_GROUNDEDNESS_THRESHOLD=3" >> $GITHUB_ENV
# Unset query rewriter settings
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for reflection tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with reflection configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run reflection integration tests
id: reflection-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running reflection integration tests..."
python -m tests.integration.main --sequence reflection
echo "Reflection integration tests completed"
- name: Collect logs after reflection tests
if: always()
run: |
mkdir -p logs/reflection
docker logs rag-server > logs/reflection/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/reflection/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/reflection/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/reflection/ 2>/dev/null || true
# ========================================================================
# NEMO GUARDRAILS TESTS
# ========================================================================
- name: Configure environment for guardrails tests
run: |
echo "ENABLE_GUARDRAILS=True" >> $GITHUB_ENV
# Unset reflection and query rewriter settings
echo "ENABLE_REFLECTION=False" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for guardrails tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with guardrails configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Start NemoGuardrails microservice
run: |
echo "Starting NemoGuardrails microservice..."
docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml up -d --no-deps nemo-guardrails-microservice
sleep 30
- name: Run NemoGuardrails integration tests
id: guardrails-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running NemoGuardrails integration tests..."
python -m tests.integration.main --sequence nemo_guardrails
echo "NemoGuardrails integration tests completed"
- name: Collect logs after guardrails tests
if: always()
run: |
mkdir -p logs/nemo-guardrails
docker logs rag-server > logs/nemo-guardrails/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/nemo-guardrails/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/nemo-guardrails/nvingest.log 2>&1 || true
docker logs nemo-guardrails-microservice > logs/nemo-guardrails/nemo-guardrails.log 2>&1 || true
cp tests/integration/integration_test.log logs/nemo-guardrails/ 2>/dev/null || true
- name: Stop NemoGuardrails microservice
if: always()
run: |
docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down nemo-guardrails-microservice || true
# ========================================================================
# IMAGE CAPTIONING TESTS
# ========================================================================
- name: Configure environment for image captioning tests
run: |
echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV
echo "ENABLE_GUARDRAILS=False" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for image captioning tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with image captioning configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run image captioning integration tests
id: image-captioning-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running image captioning integration tests..."
python -m tests.integration.main --sequence image_captioning
echo "Image captioning integration tests completed"
- name: Collect logs after image captioning tests
if: always()
run: |
mkdir -p logs/image-captioning
docker logs rag-server > logs/image-captioning/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/image-captioning/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/image-captioning/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/image-captioning/ 2>/dev/null || true
# ========================================================================
# VLM GENERATION TESTS
# ========================================================================
- name: Configure environment for VLM generation tests
run: |
echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV
echo "APP_NVINGEST_EXTRACTIMAGES=False" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for VLM generation tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with VLM generation configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run VLM generation integration tests
id: vlm-generation-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running VLM generation integration tests..."
python -m tests.integration.main --sequence vlm_generation
echo "VLM generation integration tests completed"
- name: Collect logs after VLM generation tests
if: always()
run: |
mkdir -p logs/vlm-generation
docker logs rag-server > logs/vlm-generation/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/vlm-generation/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/vlm-generation/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/vlm-generation/ 2>/dev/null || true
# ========================================================================
# MULTIMODAL QUERY TESTS
# ========================================================================
- name: Prepare multimodal test data
if: env.ENABLE_MULTIMODAL_QUERY_CI == 'true'
run: |
mkdir -p data/multimodal/query
[ -f tests/data/product_catalog.pdf ] && cp tests/data/product_catalog.pdf data/multimodal/ || true
[ -f tests/data/Creme_clutch_purse1-small.jpg ] && cp tests/data/Creme_clutch_purse1-small.jpg data/multimodal/query/ || true
[ -f tests/data/query/Creme_clutch_purse1-small.jpg ] && cp tests/data/query/Creme_clutch_purse1-small.jpg data/multimodal/query/ || true
- name: Configure environment for multimodal query tests
if: env.ENABLE_MULTIMODAL_QUERY_CI == 'true'
run: |
# VLM embedding (required for multimodal queries)
echo "APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-vl-1b-v2" >> $GITHUB_ENV
echo "APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV
# VLM model for generation
echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV
echo "APP_VLM_MODELNAME=nvidia/nemotron-nano-12b-v2-vl" >> $GITHUB_ENV
echo "APP_VLM_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV
# Disable reranker (not supported with multimodal)
echo "ENABLE_RERANKER=False" >> $GITHUB_ENV
echo "APP_RANKING_SERVERURL=" >> $GITHUB_ENV
# Image extraction for ingestion
echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV
echo "APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY=" >> $GITHUB_ENV
echo "APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=image" >> $GITHUB_ENV
# Reset other flags
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for multimodal query tests
if: env.ENABLE_MULTIMODAL_QUERY_CI == 'true'
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with multimodal query configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run multimodal query integration tests
id: multimodal-query-tests
if: env.ENABLE_MULTIMODAL_QUERY_CI == 'true'
continue-on-error: true
run: |
source venv/bin/activate
echo "Running multimodal query integration tests..."
python -m tests.integration.main --sequence multimodal_query
echo "Multimodal query integration tests completed"
- name: Collect logs after multimodal query tests
if: always() && env.ENABLE_MULTIMODAL_QUERY_CI == 'true'
run: |
mkdir -p logs/multimodal-query
docker logs rag-server > logs/multimodal-query/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/multimodal-query/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/multimodal-query/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/multimodal-query/ 2>/dev/null || true
# ========================================================================
# CUSTOM PROMPT TESTS
# ========================================================================
- name: Configure environment for custom prompt tests
run: |
echo "PROMPT_CONFIG_FILE=$(pwd)/tests/data/test_prompt.yaml" >> $GITHUB_ENV
echo "ENABLE_VLM_INFERENCE=False" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for custom prompt tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with custom prompt configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run custom prompt integration tests
id: custom-prompt-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running custom prompt integration tests..."
python -m tests.integration.main --sequence custom_prompt
echo "Custom prompt integration tests completed"
- name: Collect logs after custom prompt tests
if: always()
run: |
mkdir -p logs/custom-prompt
docker logs rag-server > logs/custom-prompt/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/custom-prompt/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/custom-prompt/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/custom-prompt/ 2>/dev/null || true
# ========================================================================
# LIBRARY USAGE TESTS
# ========================================================================
- name: Stop rag-server and ingestor-server for library tests
run: |
echo "Stopping rag-server and ingestor-server containers (library mode doesn't need them)..."
echo "Keeping nv-ingest-ms-runtime, Elasticsearch, Redis, object store running for library mode..."
docker stop rag-server || true
docker stop ingestor-server || true
echo "Services stopped. Library tests will use the nvidia_rag library directly."
echo "Remaining services (needed for library mode):"
docker ps
- name: Configure environment for library usage tests
run: |
echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Run library usage integration tests
id: library-usage-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running library usage integration tests..."
python -m tests.integration.main --sequence library_usage
echo "Library usage integration tests completed"
- name: Collect logs after library usage tests
if: always()
run: |
mkdir -p logs/library-usage
docker logs compose-nv-ingest-ms-runtime-1 > logs/library-usage/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/library-usage/ 2>/dev/null || true
# ========================================================================
# LIBRARY SUMMARIZATION TESTS
# ========================================================================
- name: Run library summarization integration tests
id: library-summarization-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running library summarization integration tests..."
python -m tests.integration.main --sequence library_summarization
echo "Library summarization integration tests completed"
- name: Collect logs after library summarization tests
if: always()
run: |
mkdir -p logs/library-summarization
docker logs compose-nv-ingest-ms-runtime-1 > logs/library-summarization/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/library-summarization/ 2>/dev/null || true
# ========================================================================
# OBSERVABILITY TESTS
# ========================================================================
- name: Configure environment for observability tests
run: |
echo "APP_TRACING_ENABLED=True" >> $GITHUB_ENV
echo "OPENTELEMETRY_CONFIG_FILE=$(pwd)/deploy/config/otel-collector-config.yaml" >> $GITHUB_ENV
echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for observability tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restarting services with observability configuration (Elasticsearch default VDB)..."
echo "(rag-server and ingestor-server were stopped for library tests, now restarting)"
docker compose -f tests/integration/vectordb.yaml --profile milvus down -v || true
docker compose -f tests/integration/vectordb.yaml down -v || true
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch down -v || true
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f tests/integration/vectordb.yaml --profile elasticsearch up -d || true
echo "APP_VECTORSTORE_URL=http://elasticsearch:9200" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=elasticsearch" >> $GITHUB_ENV
export APP_VECTORSTORE_URL=http://elasticsearch:9200
export APP_VECTORSTORE_NAME=elasticsearch
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Start observability services
run: |
echo "Starting observability services..."
docker compose -f deploy/compose/observability.yaml up -d otel-collector prometheus
sleep 30
- name: Run observability integration tests
id: observability-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running observability integration tests..."
python -m tests.integration.main --sequence observability
echo "Observability integration tests completed"
- name: Collect logs after observability tests
if: always()
run: |
mkdir -p logs/observability
echo "=== Container Status (docker ps -a) ===" | tee logs/observability/container-status.log
docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | tee -a logs/observability/container-status.log
echo "" | tee -a logs/observability/container-status.log
echo "=== Elasticsearch Container Stats ===" | tee -a logs/observability/container-status.log
docker stats --no-stream elasticsearch 2>&1 | tee -a logs/observability/container-status.log || echo "Elasticsearch container not running" | tee -a logs/observability/container-status.log
docker logs rag-server > logs/observability/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/observability/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/observability/nvingest.log 2>&1 || true
docker logs elasticsearch > logs/observability/elasticsearch.log 2>&1 || true
docker logs otel-collector > logs/observability/otel-collector.log 2>&1 || true
docker logs zipkin > logs/observability/zipkin.log 2>&1 || true
docker logs prometheus > logs/observability/prometheus.log 2>&1 || true
docker logs grafana-service > logs/observability/grafana.log 2>&1 || true
cp tests/integration/integration_test.log logs/observability/ 2>/dev/null || true
- name: Stop observability services
if: always()
run: |
docker compose -f deploy/compose/observability.yaml down || true
# ========================================================================
# MILVUS VDB AUTH VIA REST TESTS
# ========================================================================
- name: Configure Milvus VDB auth
run: |
echo "Configuring Milvus authentication and override compose..."
# Set auth credentials for the servers (running in Docker)
echo "APP_VECTORSTORE_USERNAME=root" >> $GITHUB_ENV
echo "APP_VECTORSTORE_PASSWORD=Milvus" >> $GITHUB_ENV
# Set auth token for the integration tests (running on host)
echo "VDB_AUTH_TOKEN=root:Milvus" >> $GITHUB_ENV
echo "MILVUS_ROOT_TOKEN=root:Milvus" >> $GITHUB_ENV
# Stop and clean up existing Milvus containers and volumes
echo "Stopping and cleaning up existing Milvus containers..."
docker compose -f tests/integration/vectordb.yaml down -v || true
# Wipe the rag-vol-* volumes to ensure a clean start under the new auth config.
echo "Removing rag-vol-* volumes to clear Milvus/etcd state..."
docker volume ls -q --filter "name=^rag-vol-" | xargs -r docker volume rm -f || true
# Ensure milvus.yaml exists in tests/integration by copying from running container
# Since we stopped the container, we need to start a temporary one to extract config
mkdir -p tests/integration
if [ ! -f tests/integration/milvus.yaml ]; then
echo "Creating temporary Milvus container to extract config..."
docker run --rm -d --name milvus-temp milvusdb/milvus:${MILVUS_VERSION:-v2.6.5-gpu} sleep 60
docker cp milvus-temp:/milvus/configs/milvus.yaml tests/integration/milvus.yaml
docker stop milvus-temp || true
fi
# Update authentication settings in milvus.yaml
sed -i 's/authorizationEnabled:.*/authorizationEnabled: true/' tests/integration/milvus.yaml
sed -i 's/defaultRootPassword:.*/defaultRootPassword: Milvus/' tests/integration/milvus.yaml
echo "MILVUS_CONFIG_FILE=$(pwd)/tests/integration/milvus.yaml" >> $GITHUB_ENV
# Verify the changes
echo "Verifying milvus.yaml authentication settings:"
grep -A2 "security:" tests/integration/milvus.yaml | head -5
# Wipe rag-vol-* volumes so Milvus starts with a fresh data dir under the auth config.
# (Replaces the previous trick of commenting out the data-volume bind-mount.)
docker volume ls -q --filter "name=^rag-vol-" | xargs -r docker volume rm -f || true
# Enable the milvus.yaml config file mount in vectordb.yaml
sed -i 's|# - \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|- ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|' tests/integration/vectordb.yaml
- name: Restart vector database with auth
run: |
echo "Starting vector database with authentication enabled..."
docker compose -f tests/integration/vectordb.yaml --profile milvus up -d
echo "Waiting for Milvus services to be ready..."
sleep 60
docker ps
echo "Checking Milvus logs..."
docker logs --tail 100 milvus-standalone || true
docker logs --tail 50 milvus-etcd || true
docker logs --tail 50 seaweedfs || true
- name: Restart services for Milvus VDB auth tests
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
# rag-server and ingestor-server read APP_VECTORSTORE_* at container start (see deploy/compose/docker-compose-*-server.yaml).
# that must use Milvus on the shared nvidia-rag network — same pattern as "Configure environment for Milvus sequence tests".
export APP_VECTORSTORE_URL=http://milvus:19530
export APP_VECTORSTORE_NAME=milvus
echo "APP_VECTORSTORE_URL=http://milvus:19530" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=milvus" >> $GITHUB_ENV
echo "Restarting rag/ingestor services to pick up Milvus auth configuration..."
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run Milvus VDB Auth tests
id: milvus-vdb-auth-tests
continue-on-error: true
env:
APP_VECTORSTORE_URL: http://localhost:19530
run: |
source venv/bin/activate
echo "Running Milvus VDB Auth tests..."
echo "APP_VECTORSTORE_URL set to: $APP_VECTORSTORE_URL"
python -m tests.integration.main --sequence milvus_vdb_auth_through_rest_api
echo "Milvus VDB Auth tests completed"
- name: Collect logs after Milvus VDB auth tests
if: always()
run: |
mkdir -p logs/milvus-vdb-auth
docker logs rag-server > logs/milvus-vdb-auth/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/milvus-vdb-auth/ingestor-server.log 2>&1 || true
docker logs milvus-standalone > logs/milvus-vdb-auth/milvus.log 2>&1 || true
docker logs milvus-etcd > logs/milvus-vdb-auth/etcd.log 2>&1 || true
docker logs seaweedfs > logs/milvus-vdb-auth/seaweedfs.log 2>&1 || true
cp tests/integration/integration_test.log logs/milvus-vdb-auth/ 2>/dev/null || true
- name: Revert Milvus VDB auth configurations
if: always()
run: |
echo "Reverting Milvus VDB auth configurations..."
# Stop Milvus with auth enabled
docker compose -f tests/integration/vectordb.yaml down -v || true
# Clean up auth-specific data to ensure fresh start
docker volume ls -q --filter "name=^rag-vol-" | xargs -r docker volume rm -f || true
# Remove the auth config file
rm -f tests/integration/milvus.yaml || true
# Revert vectordb.yaml to original state (data-volume mount is unchanged now;
# only the milvus.yaml config-file mount needs to be re-commented).
sed -i 's|- \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|# - ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|' tests/integration/vectordb.yaml
# Restart default vector DB stack (Elasticsearch + object store) for end of job
echo "Restarting vector database stack after Milvus auth tests..."
docker compose -f tests/integration/vectordb.yaml up -d
sleep 30
# Match default VDB in job env (same as post observability / post milvus-sequence restore)
echo "APP_VECTORSTORE_URL=http://elasticsearch:9200" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=elasticsearch" >> $GITHUB_ENV
# Unset auth environment variables
echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV
echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV
echo "VDB_AUTH_TOKEN=" >> $GITHUB_ENV
echo "MILVUS_ROOT_TOKEN=" >> $GITHUB_ENV
# ========================================================================
# NEMO RETRIEVER LIBRARY (NRL) + LANCEDB TESTS
# ========================================================================
# NeMo-Retriever Library runs in-process; LanceDB URI is the mounted path inside containers.
# Gated by ENABLE_NRL_INTEGRATION_TESTS (job env above).
- name: Prepare LanceDB volume directory
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
run: |
mkdir -p volumes/lancedb
# Clear prior DB only (APP_VECTORSTORE_URL=/volumes/lancedb/lancedb); keep .gitkeep
rm -rf volumes/lancedb/lancedb 2>/dev/null || true
ls -la volumes/lancedb
- name: Configure environment for NeMo Retriever Library (NRL) sequence tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
run: |
echo "APP_VECTORSTORE_URL=/volumes/lancedb/lancedb" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=lancedb" >> $GITHUB_ENV
echo "INGESTOR_BACKEND=nrl" >> $GITHUB_ENV
# Reset ingestor + shared compose env after prior suites (vs docker-compose-ingestor-server.yaml + nvdev.env).
echo "APP_NVINGEST_EXTRACTIMAGES=False" >> $GITHUB_ENV
echo "APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY=" >> $GITHUB_ENV
echo "APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=" >> $GITHUB_ENV
echo "APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2" >> $GITHUB_ENV
echo "APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV
echo "APP_TRACING_ENABLED=False" >> $GITHUB_ENV
# RAG compose (same job env for docker compose): restore nvdev / defaults after multimodal
echo "ENABLE_VLM_INFERENCE=False" >> $GITHUB_ENV
echo "ENABLE_RERANKER=True" >> $GITHUB_ENV
echo "APP_RANKING_SERVERURL=" >> $GITHUB_ENV
echo "MILVUS_CONFIG_FILE=" >> $GITHUB_ENV
- name: Restart services for NRL + LanceDB sequence tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Relaunching RAG and ingestor with LanceDB + NRL..."
export APP_VECTORSTORE_URL=/volumes/lancedb/lancedb
export APP_VECTORSTORE_NAME=lancedb
export INGESTOR_BACKEND=nrl
export APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2
export APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1
export APP_NVINGEST_EXTRACTIMAGES=False
export APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=
export APP_TRACING_ENABLED=False
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run NeMo Retriever Library integration tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
id: nemo-retriever-library-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running NeMo Retriever Library integration tests (sequence nemo_retriever_library)..."
python -m tests.integration.main --sequence nemo_retriever_library
echo "NeMo Retriever Library integration tests completed"
- name: Collect logs after NeMo Retriever Library tests
if: ${{ always() && env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
run: |
mkdir -p logs/nemo-retriever-library
docker logs rag-server > logs/nemo-retriever-library/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/nemo-retriever-library/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/nemo-retriever-library/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/nemo-retriever-library/ 2>/dev/null || true
# ------------------------------------------------------------------------
# NRL + LanceDB: VLM generation (same API tests as vlm_generation)
# ------------------------------------------------------------------------
- name: Configure environment for NeMo Retriever Library VLM generation tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
run: |
echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV
echo "APP_NVINGEST_EXTRACTIMAGES=False" >> $GITHUB_ENV
echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV
- name: Restart services for NeMo Retriever Library VLM generation tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Relaunching RAG and ingestor with LanceDB + NRL + VLM generation..."
export APP_VECTORSTORE_URL=/volumes/lancedb/lancedb
export APP_VECTORSTORE_NAME=lancedb
export INGESTOR_BACKEND=nrl
export APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2
export APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1
export APP_NVINGEST_EXTRACTIMAGES=False
export APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=
export APP_TRACING_ENABLED=False
export ENABLE_VLM_INFERENCE=True
export ENABLE_RERANKER=True
export APP_RANKING_SERVERURL=
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
- name: Run NeMo Retriever Library VLM generation integration tests
if: ${{ env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
id: nemo-retriever-library-vlm-generation-tests
continue-on-error: true
run: |
source venv/bin/activate
echo "Running NeMo Retriever Library VLM generation integration tests (sequence nemo_retriever_library_vlm_generation)..."
python -m tests.integration.main --sequence nemo_retriever_library_vlm_generation
echo "NeMo Retriever Library VLM generation integration tests completed"
- name: Collect logs after NeMo Retriever Library VLM generation tests
if: ${{ always() && env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
run: |
mkdir -p logs/nemo-retriever-library-vlm-generation
docker logs rag-server > logs/nemo-retriever-library-vlm-generation/rag-server.log 2>&1 || true
docker logs ingestor-server > logs/nemo-retriever-library-vlm-generation/ingestor-server.log 2>&1 || true
docker logs compose-nv-ingest-ms-runtime-1 > logs/nemo-retriever-library-vlm-generation/nvingest.log 2>&1 || true
cp tests/integration/integration_test.log logs/nemo-retriever-library-vlm-generation/ 2>/dev/null || true
- name: Restore default vector store and ingestor backend after NRL tests
if: ${{ always() && env.ENABLE_NRL_INTEGRATION_TESTS == 'true' }}
env:
CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
run: |
echo "Restoring Elasticsearch + nv_ingest for workflow cleanup consistency..."
echo "APP_VECTORSTORE_URL=http://elasticsearch:9200" >> $GITHUB_ENV
echo "APP_VECTORSTORE_NAME=elasticsearch" >> $GITHUB_ENV
echo "INGESTOR_BACKEND=nv_ingest" >> $GITHUB_ENV
export APP_VECTORSTORE_URL=http://elasticsearch:9200
export APP_VECTORSTORE_NAME=elasticsearch
export INGESTOR_BACKEND=nv_ingest
docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true
sleep 5
docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
sleep 30
docker ps
# ========================================================================
# FAIL JOB IF ANY INTEGRATION TEST FAILED
# ========================================================================
# All test steps use continue-on-error so every suite runs; this step
# marks the job as failed if any of them failed.
- name: Fail job if any integration test failed
if: always() && (steps.basic-tests.outcome == 'failure' || steps.milvus-sequence-tests.outcome == 'failure' || steps.query-rewriter-tests.outcome == 'failure' || steps.reflection-tests.outcome == 'failure' || steps.guardrails-tests.outcome == 'failure' || steps.image-captioning-tests.outcome == 'failure' || steps.vlm-generation-tests.outcome == 'failure' || steps.multimodal-query-tests.outcome == 'failure' || steps.custom-prompt-tests.outcome == 'failure' || steps.library-usage-tests.outcome == 'failure' || steps.library-summarization-tests.outcome == 'failure' || steps.observability-tests.outcome == 'failure' || steps.milvus-vdb-auth-tests.outcome == 'failure' || steps.nemo-retriever-library-tests.outcome == 'failure' || steps.nemo-retriever-library-vlm-generation-tests.outcome == 'failure')
run: |
echo "=== Failed integration test suites ==="
[ "${{ steps.basic-tests.outcome }}" = "failure" ] && echo " - basic-tests"
[ "${{ steps.milvus-sequence-tests.outcome }}" = "failure" ] && echo " - milvus-sequence-tests"
[ "${{ steps.query-rewriter-tests.outcome }}" = "failure" ] && echo " - query-rewriter-tests"
[ "${{ steps.reflection-tests.outcome }}" = "failure" ] && echo " - reflection-tests"
[ "${{ steps.guardrails-tests.outcome }}" = "failure" ] && echo " - guardrails-tests"
[ "${{ steps.image-captioning-tests.outcome }}" = "failure" ] && echo " - image-captioning-tests"
[ "${{ steps.vlm-generation-tests.outcome }}" = "failure" ] && echo " - vlm-generation-tests"
[ "${{ steps.multimodal-query-tests.outcome }}" = "failure" ] && echo " - multimodal-query-tests"
[ "${{ steps.custom-prompt-tests.outcome }}" = "failure" ] && echo " - custom-prompt-tests"
[ "${{ steps.library-usage-tests.outcome }}" = "failure" ] && echo " - library-usage-tests"
[ "${{ steps.library-summarization-tests.outcome }}" = "failure" ] && echo " - library-summarization-tests"
[ "${{ steps.observability-tests.outcome }}" = "failure" ] && echo " - observability-tests"
[ "${{ steps.milvus-vdb-auth-tests.outcome }}" = "failure" ] && echo " - milvus-vdb-auth-tests"
[ "${{ steps.nemo-retriever-library-tests.outcome }}" = "failure" ] && echo " - nemo-retriever-library-tests"
[ "${{ steps.nemo-retriever-library-vlm-generation-tests.outcome }}" = "failure" ] && echo " - nemo-retriever-library-vlm-generation-tests"
echo "One or more integration test suites failed. Failing job."
exit 1
# ========================================================================
# UPLOAD ALL LOGS
# ========================================================================
- name: Sanitize branch name for artifacts
if: always()
id: sanitize
run: |
SANITIZED_REF="${GITHUB_REF_NAME//\//-}"
echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT
- name: Upload all integration test logs
uses: actions/upload-artifact@v5
if: always()
with:
name: integration-tests-logs-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }}
path: logs/
retention-days: 7
# ========================================================================
# CLEANUP
# ========================================================================
- name: Cleanup virtual environment
if: always()
run: |
echo "Cleaning up virtual environment..."
rm -rf venv
echo "Virtual environment cleanup completed"
- name: Cleanup Docker containers and volumes
if: always()
run: |
echo "Cleaning up integration test environment..."
# Bring down only the specific compose stacks used in this workflow,
# and clean up their containers, networks, and locally built images.
docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down -v --remove-orphans --rmi local || true
docker compose -f deploy/compose/docker-compose-rag-server.yaml down -v --remove-orphans --rmi local || true
docker compose -f tests/integration/vectordb.yaml down -v --remove-orphans --rmi local || true
echo "Cleanup completed for integration test Docker resources"