feat: initial testing for translating with qwen 2.5

Author: M00dler

.gitignore

@@ -36,3 +36,6 @@ build/
 # Uploaded files (if saved locally during dev)
 uploads/
 data/
+
+models/
+.models

docker-compose.yml

@@ -70,5 +70,28 @@ services:
         exit 0;
       "
 
+  vllm:
+    container_name: qwen2.5
+    image: vllm/vllm-openai:latest
+    runtime: nvidia
+    ipc: host
+    volumes:
+      - ./models/Qwen2.5-14B-Instruct:/mnt/model/
+    #  - /home/ubuntu/.cache/hugging-face:/root/.cache/huggingface
+    ports:
+      - "80:8000"
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=0,1,2,3
+      - HF_HUB_OFFLINE=1
+    command: [
+      "--model", "/mnt/model/",
+      "--served-model-name","qwen2.5",
+      "--max-model-len", "4096",
+      "--tensor-parallel-size", "4",
+      "--gpu-memory-utilization", "0.8",
+      "--enable-auto-tool-choice",
+      "--tool-call-parser", "hermes"
+    ]
+
 volumes:
   minio-data:

docling_translation_service/main.py

@@ -0,0 +1,17 @@
+from fastapi import FastAPI
+from routers import health
+from docling_translation_service.routers import translation
+from routers import session
+import logging
+
+# Set up logger
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+)
+
+app = FastAPI()
+
+app.include_router(health.router)
+app.include_router(translation.router)
+app.include_router(session.router)