dgx-serve-vllm-gemma4/compose.yaml at main · anthonywu/dgx-serve-vllm-gemma4 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
services:
  gemma4-openai:
    image: ${VLLM_IMAGE}
    container_name: gemma4-openai
    restart: unless-stopped
    ipc: host
    shm_size: ${SHM_SIZE}
    gpus: all
    ports:
      - "${TAILSCALE_IP}:${VLLM_PORT}:8000"
      - "127.0.0.1:${VLLM_LOCAL_PORT}:8000"
    environment:
      HF_HUB_OFFLINE: ${HF_HUB_OFFLINE}
      TRANSFORMERS_OFFLINE: ${TRANSFORMERS_OFFLINE}
      VLLM_NO_USAGE_STATS: ${VLLM_NO_USAGE_STATS}
      DO_NOT_TRACK: ${DO_NOT_TRACK}
    volumes:
      - "${HF_CACHE_DIR}:/root/.cache/huggingface"
    command:
      - ${MODEL_ID}
      - --served-model-name
      - ${SERVED_MODEL_NAME}
      - --tensor-parallel-size
      - ${TENSOR_PARALLEL_SIZE}
      - --max-model-len
      - ${MAX_MODEL_LEN}
      - --gpu-memory-utilization
      - ${GPU_MEMORY_UTILIZATION}
      - --api-key
      - ${OPENAI_API_KEY}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --reasoning-parser
      - gemma4
      - --tool-call-parser
      - gemma4
      - --enable-auto-tool-choice
      - --async-scheduling
      - --kv-cache-dtype
      - fp8
      - --limit-mm-per-prompt
      - '{"image":4,"audio":1}'
      - --max-num-seqs
      - "64"