openrag/docs/assets/compose_linux_gpu.yaml at e827d64fa3ddd1a39344e9c6067df57a8de39688 · linagora/openrag · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
include:
  - vdb/milvus.yaml
  - ${CHAINLIT_DATALAYER_COMPOSE:-extern/dummy.yaml}
  - extern/infinity.yaml

x-openrag: &openrag_template
  image: ghcr.io/linagora/openrag:dev-latest
  build:
    context: .
    dockerfile: Dockerfile
  volumes:
    - ${DATA_VOLUME:-./data}:/app/data
    - ${MODEL_WEIGHTS_VOLUME:-~/.cache/huggingface}:/app/model_weights # Model weights for RAG
    - ./openrag:/app/openrag # For dev mode
    - /$SHARED_ENV:/ray_mount/.env # Shared environment variables
    - ./ray_mount/logs:/app/logs
  ports:
    - ${APP_PORT:-8080}:${APP_iPORT:-8080}
    - 127.0.0.1:${RAY_DASHBOARD_PORT:-8265}:8265 # Localhost only: Ray dashboard/Jobs API is unauthenticated. Disable when in cluster mode
  networks:
    default:
      aliases:
        - openrag
  env_file:
    - ${SHARED_ENV:-.env}
  shm_size: 10.24gb

x-vllm: &vllm_template
  networks:
    default:
      aliases:
        - vllm
  restart: always
  environment:
    - HUGGING_FACE_HUB_TOKEN
  ipc: "host"
  volumes:
    - ${VLLM_CACHE:-/root/.cache/huggingface}:/root/.cache/huggingface # put ./vllm_cache if you want to have the weights on the vllm_cache folder in your project
  command: >
    --model ${EMBEDDER_MODEL_NAME:-jinaai/jina-embeddings-v3}
    --trust-remote-code
    --task embed
    --gpu_memory_utilization 0.3
    --max-num-seqs 1
  # --max-model-len ${MOX_MODEL_LEN:-2048}
  # gpu_memory_utilization, max-num-seqs et max-model-len can be tuned depending on your GPU memory

  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
    interval: 15s
    timeout: 5s
    retries: 3
    start_period: 60s
  ports:
    - ${VLLM_PORT:-8000}:8000
services:
  # GPU - default
  openrag:
    <<: *openrag_template
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [ gpu ]
    profiles:
      - ''
    depends_on:
      milvus:
        condition: service_healthy
      vllm-gpu:
        condition: service_healthy

  # No GPU
  openrag-cpu:
    <<: *openrag_template
    deploy: {}
    profiles:
      - 'cpu'
    depends_on:
      milvus:
        condition: service_healthy
      vllm-cpu:
        condition: service_healthy

  rdb:
    image: postgres:15
    environment:
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-root_password}
      - POSTGRES_USER=${POSTGRES_USER:-root}
    volumes:
      - ${DB_VOLUME:-./db}:/var/lib/postgresql/data

  vllm-gpu:
    <<: *vllm_template
    image: vllm/vllm-openai:v0.9.2
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    profiles:
      - '' # Empty string gives default behavior (but does not run when cpu requested)

  vllm-cpu:
    <<: *vllm_template
    build:
      context: extern/vllm
      dockerfile: Dockerfile.cpu
      target: vllm-openai
    image: openrag-vllm-openai-cpu
    deploy: {}
    profiles:
      - 'cpu'