docker-compose-rag-server.yaml

services:

  # Main orchestrator server which stiches together all calls to different services to fulfill the user request
  rag-server:
    container_name: rag-server
    image: nvcr.io/nvidia/blueprint/rag-server:${TAG:-2.5.0}
    build:
      # Set context to repo's root directory
      context: ../../
      dockerfile: src/nvidia_rag/rag_server/Dockerfile
      args:
        DOWNLOAD_LEGAL_COMPLIANCE: ${DOWNLOAD_LEGAL_COMPLIANCE:-false}
    # start the server on port 8081 with 8 workers for improved latency on concurrent requests.
    command: --port 8081 --host 0.0.0.0 --workers 8
    volumes:
      # Mount the prompt.yaml file to the container, path should be absolute
      - ${PROMPT_CONFIG_FILE}:${PROMPT_CONFIG_FILE}
    # Common customizations to the pipeline can be controlled using env variables
    environment:
      # Path to example directory relative to root
      EXAMPLE_PATH: './nvidia_rag/rag_server'

      # Absolute path to custom prompt.yaml file
      PROMPT_CONFIG_FILE: ${PROMPT_CONFIG_FILE:-/prompt.yaml}

      ##===MINIO specific configurations which is used to store the multimodal base64 content===
      MINIO_ENDPOINT: "minio:9010"
      MINIO_ACCESSKEY: "minioadmin"
      MINIO_SECRETKEY: "minioadmin"

      ##===Vector DB specific configurations===
      # URL on which vectorstore is hosted
      # For custom operators, point to your service (e.g., http://your-custom-vdb:1234)
      APP_VECTORSTORE_URL: ${APP_VECTORSTORE_URL:-http://milvus:19530}
      # Type of vectordb used to store embedding. Supported built-ins: "milvus", "elasticsearch".
      # You can also provide your custom value (e.g., "your_custom_vdb") when you register it in `_get_vdb_op`.
      APP_VECTORSTORE_NAME: ${APP_VECTORSTORE_NAME:-"milvus"}
      # Type of index to be used for vectorstore
      APP_VECTORSTORE_INDEXTYPE: ${APP_VECTORSTORE_INDEXTYPE:-"GPU_CAGRA"}
      
      # Type of vectordb search to be used
      APP_VECTORSTORE_SEARCHTYPE: ${APP_VECTORSTORE_SEARCHTYPE:-"dense"} # Can be dense or hybrid
      # Type of ranker to use for vector store in case of Hybrid Search
      APP_VECTORSTORE_RANKER_TYPE: ${APP_VECTORSTORE_RANKER_TYPE:-"rrf"} # Can be "rrf" or "weighted"
      # Weight for dense vector search in case of "weighted" Hybrid Search
      APP_VECTORSTORE_DENSE_WEIGHT: ${APP_VECTORSTORE_DENSE_WEIGHT:-0.5}
      # Weight for sparse vector search in case of "weighted" Hybrid Search
      APP_VECTORSTORE_SPARSE_WEIGHT: ${APP_VECTORSTORE_SPARSE_WEIGHT:-0.5}
      
      # Boolean to control GPU search for milvus vectorstore specific to rag-server
      APP_VECTORSTORE_ENABLEGPUSEARCH: ${APP_VECTORSTORE_ENABLEGPUSEARCH:-True}
      # ef: Parameter controlling query time/accuracy trade-off. Higher ef leads to more accurate but slower search.
      APP_VECTORSTORE_EF: ${APP_VECTORSTORE_EF:-100} # Must be greater or equal to VECTOR_DB_TOPK
      # Username for vector store 
      APP_VECTORSTORE_USERNAME: ${APP_VECTORSTORE_USERNAME:-""}
      # Password for vector store
      APP_VECTORSTORE_PASSWORD: ${APP_VECTORSTORE_PASSWORD:-""}
      # Elasticsearch API key auth (optional). Prefer these over username/password when set.
      # Provide either base64 APP_VECTORSTORE_APIKEY or split ID/SECRET.
      APP_VECTORSTORE_APIKEY_ID: ${APP_VECTORSTORE_APIKEY_ID:-""}
      APP_VECTORSTORE_APIKEY_SECRET: ${APP_VECTORSTORE_APIKEY_SECRET:-""}
      APP_VECTORSTORE_APIKEY: ${APP_VECTORSTORE_APIKEY:-""}
      # vectorstore collection name to store embeddings
      COLLECTION_NAME: ${COLLECTION_NAME:-multimodal_data}
      APP_RETRIEVER_SCORETHRESHOLD: 0.25
      # Top K from vector DB, which goes as input to reranker model if enabled, else goes to LLM prompt
      VECTOR_DB_TOPK: ${VECTOR_DB_TOPK:-100}

      ##===LLM Model specific configurations===
      APP_LLM_MODELNAME: ${APP_LLM_MODELNAME:-"nvidia/llama-3.3-nemotron-super-49b-v1.5"}
      # url on which llm model is hosted. If "", Nvidia hosted API is used
      APP_LLM_SERVERURL: ${APP_LLM_SERVERURL-"nim-llm:8000"}
      # LLM model parameters
      LLM_MAX_TOKENS: ${LLM_MAX_TOKENS:-32768}
      LLM_TEMPERATURE: ${LLM_TEMPERATURE:-0}
      LLM_TOP_P: ${LLM_TOP_P:-1.0}

      # Reasoning configuration (supported by Nemotron 3 and other reasoning models)
      LLM_ENABLE_THINKING: ${LLM_ENABLE_THINKING:-false}
      LLM_REASONING_BUDGET: ${LLM_REASONING_BUDGET:-0}
      LLM_LOW_EFFORT: ${LLM_LOW_EFFORT:-false}

      ##===Query Rewriter Model specific configurations===
      APP_QUERYREWRITER_MODELNAME: ${APP_QUERYREWRITER_MODELNAME:-"nvidia/llama-3.3-nemotron-super-49b-v1.5"}
      # url on which query rewriter model is hosted. If "", Nvidia hosted API is used
      APP_QUERYREWRITER_SERVERURL: ${APP_QUERYREWRITER_SERVERURL-"nim-llm:8000"}

      ##===Filter Expression Generator Model specific configurations===
      APP_FILTEREXPRESSIONGENERATOR_MODELNAME: ${APP_FILTEREXPRESSIONGENERATOR_MODELNAME:-"nvidia/llama-3.3-nemotron-super-49b-v1.5"}
      # url on which filter expression generator model is hosted. If "", Nvidia hosted API is used
      APP_FILTEREXPRESSIONGENERATOR_SERVERURL: ${APP_FILTEREXPRESSIONGENERATOR_SERVERURL-"nim-llm:8000"}
      # enable filter expression generator for natural language to filter expression conversion
      ENABLE_FILTER_GENERATOR: ${ENABLE_FILTER_GENERATOR:-False}

      ##===Embedding Model specific configurations===
      # url on which embedding model is hosted. If "", Nvidia hosted API is used
      APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-embedding-ms:8000/v1"}
      APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-1b-v2}
      APP_EMBEDDINGS_DIMENSIONS: ${APP_EMBEDDINGS_DIMENSIONS:-2048}
      # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1)
      # APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-vlm-embedding-ms:8000/v1"}
      # APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-vl-1b-v2}

      ##===Reranking Model specific configurations===
      # url on which ranking model is hosted. If "", Nvidia hosted API is used
      APP_RANKING_SERVERURL: ${APP_RANKING_SERVERURL-"nemotron-ranking-ms:8000"}
      APP_RANKING_MODELNAME: ${APP_RANKING_MODELNAME:-"nvidia/llama-nemotron-rerank-1b-v2"}
      ENABLE_RERANKER: ${ENABLE_RERANKER:-True}
      # Default score threshold for filtering documents by reranker relevance (0.0 to 1.0)
      RERANKER_SCORE_THRESHOLD: ${RERANKER_SCORE_THRESHOLD:-${RERANKER_CONFIDENCE_THRESHOLD:-0.0}}

      ##===VLM Model specific configurations===
      ENABLE_VLM_INFERENCE: ${ENABLE_VLM_INFERENCE:-false}
      # When true, allow falling back to LLM flow if no images are present in query/messages/context
      # When false, always route generation through VLM when ENABLE_VLM_INFERENCE is true
      VLM_TO_LLM_FALLBACK: ${VLM_TO_LLM_FALLBACK:-true}
      # Max images sent to VLM per request (query + context)
      APP_VLM_MAX_TOTAL_IMAGES: ${APP_VLM_MAX_TOTAL_IMAGES:-5}
      # VLM generation parameters
      APP_VLM_MAX_TOKENS: ${APP_VLM_MAX_TOKENS:-8192}
      APP_VLM_TEMPERATURE: ${APP_VLM_TEMPERATURE:-0.1}
      APP_VLM_TOP_P: ${APP_VLM_TOP_P:-1.0}
      # VLM server URL
      APP_VLM_SERVERURL: ${APP_VLM_SERVERURL-"http://vlm-ms:8000/v1"}
      # VLM model name
      APP_VLM_MODELNAME: ${APP_VLM_MODELNAME:-"nvidia/nemotron-nano-12b-v2-vl"}

      NVIDIA_API_KEY: ${NGC_API_KEY:?"NGC_API_KEY is required"}

      # ==== Service-Specific API Keys (Optional) ====
      # Set these to use different API keys for individual services.
      # If not set or empty, services will use NVIDIA_API_KEY as fallback.
      # Useful for multi-provider setups (e.g., OpenAI for LLM, NVIDIA for embeddings)
      APP_LLM_APIKEY: ${APP_LLM_APIKEY:-""}
      APP_EMBEDDINGS_APIKEY: ${APP_EMBEDDINGS_APIKEY:-""}
      APP_RANKING_APIKEY: ${APP_RANKING_APIKEY:-""}
      APP_QUERYREWRITER_APIKEY: ${APP_QUERYREWRITER_APIKEY:-""}
      APP_FILTEREXPRESSIONGENERATOR_APIKEY: ${APP_FILTEREXPRESSIONGENERATOR_APIKEY:-""}
      APP_VLM_APIKEY: ${APP_VLM_APIKEY:-""}
      SUMMARY_LLM_APIKEY: ${SUMMARY_LLM_APIKEY:-""}
      REFLECTION_LLM_APIKEY: ${REFLECTION_LLM_APIKEY:-""}

      # Number of document chunks to insert in LLM prompt, used only when ENABLE_RERANKER is set to True
      APP_RETRIEVER_TOPK: ${APP_RETRIEVER_TOPK:-10}

      # Log level for server, supported level NOTSET, DEBUG, INFO, WARN, ERROR, CRITICAL
      LOGLEVEL: ${LOGLEVEL:-INFO}

      # enable query rewriting for multiturn conversation in the rag chain.
      # This will improve accuracy of the retrieiver pipeline but increase latency due to an additional LLM call
      ENABLE_QUERYREWRITER: ${ENABLE_QUERYREWRITER:-False}

      # enable concatenating conversation history with current query for retrieval (when query rewriter is disabled)
      MULTITURN_RETRIEVER_SIMPLE: ${MULTITURN_RETRIEVER_SIMPLE:-False}

      # Choose whether to enable citations in the response
      ENABLE_CITATIONS: ${ENABLE_CITATIONS:-True}

      # Choose whether to enable/disable guardrails
      ENABLE_GUARDRAILS: ${ENABLE_GUARDRAILS:-False}

      # NeMo Guardrails URL when ENABLE_GUARDRAILS is true
      NEMO_GUARDRAILS_URL: ${NEMO_GUARDRAILS_URL:-nemo-guardrails-microservice:7331}

      # number of last n chat messages to consider from the provided conversation history (0 = no history)
      CONVERSATION_HISTORY: ${CONVERSATION_HISTORY:-0}

      # Tracing
      APP_TRACING_ENABLED: ${APP_TRACING_ENABLED:-"False"}
      # HTTP endpoint
      APP_TRACING_OTLPHTTPENDPOINT: http://otel-collector:4318/v1/traces
      # GRPC endpoint
      APP_TRACING_OTLPGRPCENDPOINT: grpc://otel-collector:4317
      # Prometheus multi-process metrics directory
      PROMETHEUS_MULTIPROC_DIR: "/tmp-data/prom_data"

      # Choose whether to enable source metadata in document content during generation
      ENABLE_SOURCE_METADATA: ${ENABLE_SOURCE_METADATA:-true}

      # Whether to filter content within <think></think> tags in model responses
      FILTER_THINK_TOKENS: ${FILTER_THINK_TOKENS:-true}

      # enable reflection (context relevance and response groundedness checking) in the rag chain
      ENABLE_REFLECTION: ${ENABLE_REFLECTION:-false}
      # Maximum number of context relevance loop iterations
      MAX_REFLECTION_LOOP: ${MAX_REFLECTION_LOOP:-3}
      # Minimum relevance score threshold (0-2)
      CONTEXT_RELEVANCE_THRESHOLD: ${CONTEXT_RELEVANCE_THRESHOLD:-1}
      # Minimum groundedness score threshold (0-2)
      RESPONSE_GROUNDEDNESS_THRESHOLD: ${RESPONSE_GROUNDEDNESS_THRESHOLD:-1}
      # reflection llm
      REFLECTION_LLM: ${REFLECTION_LLM:-"nvidia/llama-3.3-nemotron-super-49b-v1.5"}
      # reflection llm server url. If "", Nvidia hosted API is used
      REFLECTION_LLM_SERVERURL: ${REFLECTION_LLM_SERVERURL-"nim-llm:8000"}
      # enable iterative query decomposition
      ENABLE_QUERY_DECOMPOSITION: ${ENABLE_QUERY_DECOMPOSITION:-false}
      # maximum recursion depth for iterative query decomposition
      MAX_RECURSION_DEPTH: ${MAX_RECURSION_DEPTH:-3}

      # Redis configuration for summary status tracking
      REDIS_HOST: ${REDIS_HOST:-redis}
      REDIS_PORT: ${REDIS_PORT:-6379}
      REDIS_DB: ${REDIS_DB:-0}

    ports:
      - "8081:8081"
    expose:
      - "8081"
    shm_size: 5gb

  # Sample UI container which interacts with APIs exposed by rag-server container
  rag-frontend:
    container_name: rag-frontend
    image: nvcr.io/nvidia/blueprint/rag-frontend:${TAG:-2.5.0}
    build:
      # Set context to repo's root directory
      context: ../../frontend
      dockerfile: ./Dockerfile
      args:
        # Environment variables for Vite build
        VITE_API_CHAT_URL: ${VITE_API_CHAT_URL:-http://rag-server:8081/v1}
        VITE_API_VDB_URL: ${VITE_API_VDB_URL:-http://ingestor-server:8082/v1}
        VITE_MILVUS_URL: http://milvus:19530
        DOWNLOAD_LEGAL_COMPLIANCE: ${DOWNLOAD_LEGAL_COMPLIANCE:-false}
    ports:
      - "8090:3000"
    expose:
      - "3000"
    environment:
      # Runtime environment variables for Vite
      VITE_API_CHAT_URL: ${VITE_API_CHAT_URL:-http://rag-server:8081/v1}
      VITE_API_VDB_URL: ${VITE_API_VDB_URL:-http://ingestor-server:8082/v1}
      VITE_MILVUS_URL: http://milvus:19530
    depends_on:
      - rag-server

networks:
  default:
    name: nvidia-rag