Skip to content

Commit b7e3401

Browse files
alvarobarttalvarobartt
authored
Bump Rust 1.92, CUDA 12.6, Ubuntu 24.04 and add Dockerfile-cuda-blackwell-all (#823)
1 parent b59f754 commit b7e3401

16 files changed

Lines changed: 234 additions & 118 deletions

‎.github/workflows/matrix.json‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"dockerfile": "Dockerfile-cuda"
1111
},
1212
{
13-
"name": "ampere",
13+
"name": "ampere-80",
1414
"imageNamePrefix": "",
1515
"runOn": "always",
1616
"sccache": true,
@@ -19,7 +19,7 @@
1919
"dockerfile": "Dockerfile-cuda"
2020
},
2121
{
22-
"name": "a10",
22+
"name": "ampere-86",
2323
"imageNamePrefix": "86-",
2424
"runOn": "always",
2525
"sccache": true,
@@ -28,7 +28,7 @@
2828
"dockerfile": "Dockerfile-cuda"
2929
},
3030
{
31-
"name": "RTX 4000",
31+
"name": "ada-lovelace",
3232
"imageNamePrefix": "89-",
3333
"runOn": "always",
3434
"sccache": true,
@@ -37,7 +37,7 @@
3737
"dockerfile": "Dockerfile-cuda"
3838
},
3939
{
40-
"name": "Hopper",
40+
"name": "hopper",
4141
"imageNamePrefix": "hopper-",
4242
"runOn": "always",
4343
"sccache": true,
@@ -46,7 +46,15 @@
4646
"dockerfile": "Dockerfile-cuda"
4747
},
4848
{
49-
"name": "Blackwell (10.0)",
49+
"name": "turing-ampere-ada-hopper",
50+
"imageNamePrefix": "cuda-",
51+
"runOn": "always",
52+
"sccache": false,
53+
"grpc": false,
54+
"dockerfile": "Dockerfile-cuda-all"
55+
},
56+
{
57+
"name": "blackwell-100",
5058
"imageNamePrefix": "100-",
5159
"runOn": "always",
5260
"sccache": true,
@@ -55,7 +63,7 @@
5563
"dockerfile": "Dockerfile-cuda-blackwell"
5664
},
5765
{
58-
"name": "Blackwell (12.0)",
66+
"name": "blackwell-120",
5967
"imageNamePrefix": "120-",
6068
"runOn": "always",
6169
"sccache": true,
@@ -64,12 +72,12 @@
6472
"dockerfile": "Dockerfile-cuda-blackwell"
6573
},
6674
{
67-
"name": "All",
68-
"imageNamePrefix": "cuda-",
75+
"name": "blackwell",
76+
"imageNamePrefix": "blackwell-",
6977
"runOn": "always",
7078
"sccache": false,
7179
"grpc": false,
72-
"dockerfile": "Dockerfile-cuda-all"
80+
"dockerfile": "Dockerfile-cuda-blackwell-all"
7381
},
7482
{
7583
"name": "cpu",

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM lukemathwalker/cargo-chef:latest-rust-1.85-bookworm AS chef
1+
FROM lukemathwalker/cargo-chef:latest-rust-1.92-bookworm AS chef
22
WORKDIR /usr/src
33

44
ENV SCCACHE=0.10.0

‎Dockerfile-cuda‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 AS base-builder
1+
FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 AS base-builder
22

33
ENV SCCACHE=0.10.0
44
ENV RUSTC_WRAPPER=/usr/local/bin/sccache
55
ENV PATH="/root/.cargo/bin:${PATH}"
6-
# aligned with `cargo-chef` version in `lukemathwalker/cargo-chef:latest-rust-1.85-bookworm`
7-
ENV CARGO_CHEF=0.1.71
6+
# aligned with `cargo-chef` version in `lukemathwalker/cargo-chef:latest-rust-1.92-bookworm`
7+
ENV CARGO_CHEF=0.1.73
88

99
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
1010
curl \
@@ -51,15 +51,15 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
5151
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
5252
if [ ${CUDA_COMPUTE_CAP} -ge 75 -a ${CUDA_COMPUTE_CAP} -lt 80 ]; \
5353
then \
54-
nvprune --generate-code code=sm_${CUDA_COMPUTE_CAP} /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
54+
nvprune --generate-code code=sm_${CUDA_COMPUTE_CAP} /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
5555
elif [ ${CUDA_COMPUTE_CAP} -ge 80 -a ${CUDA_COMPUTE_CAP} -lt 90 ]; \
5656
then \
57-
nvprune --generate-code code=sm_80 --generate-code code=sm_${CUDA_COMPUTE_CAP} /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
57+
nvprune --generate-code code=sm_80 --generate-code code=sm_${CUDA_COMPUTE_CAP} /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
5858
elif [ ${CUDA_COMPUTE_CAP} -eq 90 ]; \
5959
then \
60-
nvprune --generate-code code=sm_90 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
60+
nvprune --generate-code code=sm_90 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
6161
else \
62-
echo "cuda compute cap ${CUDA_COMPUTE_CAP} is not supported"; exit 1; \
62+
echo "cuda compute cap ${CUDA_COMPUTE_CAP} is not supported"; exit 1; \
6363
fi;
6464

6565
COPY --from=planner /usr/src/recipe.json recipe.json
@@ -68,9 +68,9 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
6868
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
6969
if [ ${CUDA_COMPUTE_CAP} -ge 75 -a ${CUDA_COMPUTE_CAP} -lt 80 ]; \
7070
then \
71-
cargo chef cook --release --features candle-cuda-turing --features static-linking --no-default-features --recipe-path recipe.json && sccache -s; \
71+
cargo chef cook --release --features candle-cuda-turing --features static-linking --no-default-features --recipe-path recipe.json && sccache -s; \
7272
else \
73-
cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s; \
73+
cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s; \
7474
fi;
7575

7676
COPY backends backends
@@ -85,9 +85,9 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
8585
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
8686
if [ ${CUDA_COMPUTE_CAP} -ge 75 -a ${CUDA_COMPUTE_CAP} -lt 80 ]; \
8787
then \
88-
cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F static-linking -F http --no-default-features && sccache -s; \
88+
cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F static-linking -F http --no-default-features && sccache -s; \
8989
else \
90-
cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && sccache -s; \
90+
cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && sccache -s; \
9191
fi;
9292

9393
FROM builder AS grpc-builder
@@ -108,23 +108,25 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
108108
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
109109
if [ ${CUDA_COMPUTE_CAP} -ge 75 -a ${CUDA_COMPUTE_CAP} -lt 80 ]; \
110110
then \
111-
cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F static-linking -F grpc --no-default-features && sccache -s; \
111+
cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F static-linking -F grpc --no-default-features && sccache -s; \
112112
else \
113-
cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F grpc --no-default-features && sccache -s; \
113+
cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F grpc --no-default-features && sccache -s; \
114114
fi;
115115

116-
FROM nvidia/cuda:12.2.0-base-ubuntu22.04 AS base
116+
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS base
117117

118118
ARG DEFAULT_USE_FLASH_ATTENTION=True
119119

120120
ENV HUGGINGFACE_HUB_CACHE=/data \
121121
PORT=80 \
122-
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION
122+
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
123+
LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
123124

124125
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
125126
ca-certificates \
126127
libssl-dev \
127128
curl \
129+
cuda-compat-12-6 \
128130
&& rm -rf /var/lib/apt/lists/*
129131

130132
FROM base AS grpc

‎Dockerfile-cuda-all‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 AS base-builder
1+
FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 AS base-builder
22

33
ENV SCCACHE=0.10.0
44
ENV RUSTC_WRAPPER=/usr/local/bin/sccache
55
ENV PATH="/root/.cargo/bin:${PATH}"
6-
# aligned with `cargo-chef` version in `lukemathwalker/cargo-chef:latest-rust-1.85-bookworm`
7-
ENV CARGO_CHEF=0.1.71
6+
# aligned with `cargo-chef` version in `lukemathwalker/cargo-chef:latest-rust-1.92-bookworm`
7+
ENV CARGO_CHEF=0.1.73
88

99
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
1010
curl \
@@ -88,18 +88,20 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
8888

8989
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-90
9090

91-
FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base
91+
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS base
9292

9393
ARG DEFAULT_USE_FLASH_ATTENTION=True
9494

9595
ENV HUGGINGFACE_HUB_CACHE=/data \
9696
PORT=80 \
97-
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION
97+
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
98+
LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
9899

99100
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
100101
ca-certificates \
101102
libssl-dev \
102103
curl \
104+
cuda-compat-12-6 \
103105
&& rm -rf /var/lib/apt/lists/*
104106

105107
COPY --from=builder /usr/src/target/release/text-embeddings-router-75 /usr/local/bin/text-embeddings-router-75

‎Dockerfile-cuda-blackwell-all‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
FROM nvidia/cuda:12.9.0-devel-ubuntu24.04 AS base-builder
2+
3+
ENV SCCACHE=0.10.0
4+
ENV RUSTC_WRAPPER=/usr/local/bin/sccache
5+
ENV PATH="/root/.cargo/bin:${PATH}"
6+
# aligned with `cargo-chef` version in `lukemathwalker/cargo-chef:latest-rust-1.92-bookworm`
7+
ENV CARGO_CHEF=0.1.73
8+
9+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
10+
curl \
11+
libssl-dev \
12+
pkg-config \
13+
&& rm -rf /var/lib/apt/lists/*
14+
15+
# Donwload and configure sccache
16+
RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v$SCCACHE/sccache-v$SCCACHE-x86_64-unknown-linux-musl.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin sccache-v$SCCACHE-x86_64-unknown-linux-musl/sccache && \
17+
chmod +x /usr/local/bin/sccache
18+
19+
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
20+
RUN cargo install cargo-chef --version $CARGO_CHEF --locked
21+
22+
FROM base-builder AS planner
23+
24+
WORKDIR /usr/src
25+
26+
COPY backends backends
27+
COPY core core
28+
COPY router router
29+
COPY Cargo.toml ./
30+
COPY Cargo.lock ./
31+
32+
RUN cargo chef prepare --recipe-path recipe.json
33+
34+
FROM base-builder AS builder
35+
36+
ARG GIT_SHA
37+
ARG DOCKER_LABEL
38+
39+
# sccache specific variables
40+
ARG SCCACHE_GHA_ENABLED
41+
42+
# Limit parallelism
43+
ARG RAYON_NUM_THREADS=4
44+
ARG CARGO_BUILD_JOBS
45+
ARG CARGO_BUILD_INCREMENTAL
46+
47+
WORKDIR /usr/src
48+
49+
COPY --from=planner /usr/src/recipe.json recipe.json
50+
51+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
52+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
53+
cargo chef cook --release --recipe-path recipe.json && sccache -s;
54+
55+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
56+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
57+
CUDA_COMPUTE_CAP=100 cargo chef cook --release --features candle-cuda --recipe-path recipe.json && sccache -s;
58+
59+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
60+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
61+
CUDA_COMPUTE_CAP=120 cargo chef cook --release --features candle-cuda --recipe-path recipe.json && sccache -s;
62+
63+
COPY backends backends
64+
COPY core core
65+
COPY router router
66+
COPY Cargo.toml ./
67+
COPY Cargo.lock ./
68+
69+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
70+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
71+
CUDA_COMPUTE_CAP=100 cargo build --release --bin text-embeddings-router -F candle-cuda && sccache -s;
72+
73+
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-100
74+
75+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
76+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
77+
CUDA_COMPUTE_CAP=120 cargo build --release --bin text-embeddings-router -F candle-cuda && sccache -s;
78+
79+
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-120
80+
81+
FROM nvidia/cuda:12.9.0-base-ubuntu24.04 AS base
82+
83+
ARG DEFAULT_USE_FLASH_ATTENTION=True
84+
85+
ENV HUGGINGFACE_HUB_CACHE=/data \
86+
PORT=80 \
87+
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
88+
LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
89+
90+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
91+
ca-certificates \
92+
libssl-dev \
93+
curl \
94+
cuda-compat-12-9 \
95+
&& rm -rf /var/lib/apt/lists/*
96+
97+
COPY --from=builder /usr/src/target/release/text-embeddings-router-100 /usr/local/bin/text-embeddings-router-100
98+
COPY --from=builder /usr/src/target/release/text-embeddings-router-120 /usr/local/bin/text-embeddings-router-120
99+
100+
COPY --chmod=775 cuda-blackwell-all-entrypoint.sh entrypoint.sh
101+
102+
ENTRYPOINT ["./entrypoint.sh"]
103+
CMD ["--json-output"]

‎Dockerfile-intel‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
ARG PLATFORM=cpu
2-
FROM lukemathwalker/cargo-chef:latest-rust-1.85-bookworm AS chef
2+
FROM lukemathwalker/cargo-chef:latest-rust-1.92-bookworm AS chef
33
WORKDIR /usr/src
44

55
ENV SCCACHE=0.10.0
@@ -64,12 +64,12 @@ ENV HUGGINGFACE_HUB_CACHE=/data \
6464
PORT=80
6565

6666
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
67-
build-essential \
68-
git \
69-
cmake \
70-
ninja-build \
71-
python3-dev &&\
72-
rm -rf /var/lib/apt/lists/*
67+
build-essential \
68+
git \
69+
cmake \
70+
ninja-build \
71+
python3-dev &&\
72+
rm -rf /var/lib/apt/lists/*
7373

7474
WORKDIR /usr/src
7575
COPY backends backends
@@ -88,12 +88,12 @@ ENV HUGGINGFACE_HUB_CACHE=/data \
8888
ENV PT_HPU_LAZY_MODE=1
8989

9090
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
91-
build-essential \
92-
git \
93-
cmake \
94-
ninja-build \
95-
python3-dev &&\
96-
rm -rf /var/lib/apt/lists/*
91+
build-essential \
92+
git \
93+
cmake \
94+
ninja-build \
95+
python3-dev &&\
96+
rm -rf /var/lib/apt/lists/*
9797

9898
WORKDIR /usr/src
9999
COPY backends backends
@@ -114,7 +114,7 @@ RUN wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.
114114
RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dearmor | tee /usr/share/keyrings/intel-graphics.gpg > /dev/null
115115

116116
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
117-
| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
117+
| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
118118

119119
RUN apt-get update && apt install -y intel-basekit cmake vim python3-dev ninja-build pciutils
120120
WORKDIR /usr/src

‎README.md‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -335,15 +335,17 @@ Options:
335335

336336
Text Embeddings Inference ships with multiple Docker images that you can use to target a specific backend:
337337

338-
| Architecture | Image |
339-
|-------------------------------------|-------------------------------------------------------------------------|
340-
| CPU | ghcr.io/huggingface/text-embeddings-inference:cpu-1.8 |
341-
| Volta | NOT SUPPORTED |
342-
| Turing (T4, RTX 2000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:turing-1.8 (experimental) |
343-
| Ampere 80 (A100, A30) | ghcr.io/huggingface/text-embeddings-inference:1.8 |
344-
| Ampere 86 (A10, A40, ...) | ghcr.io/huggingface/text-embeddings-inference:86-1.8 |
345-
| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-1.8 |
346-
| Hopper (H100) | ghcr.io/huggingface/text-embeddings-inference:hopper-1.8 (experimental) |
338+
| Architecture | Image |
339+
|----------------------------------------|------------------------------------------------------------------------------|
340+
| CPU | ghcr.io/huggingface/text-embeddings-inference:cpu-1.8 |
341+
| Volta | NOT SUPPORTED |
342+
| Turing (T4, RTX 2000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:turing-1.8 (experimental) |
343+
| Ampere 8.0 (A100, A30) | ghcr.io/huggingface/text-embeddings-inference:1.8 |
344+
| Ampere 8.6 (A10, A40, ...) | ghcr.io/huggingface/text-embeddings-inference:86-1.8 |
345+
| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-1.8 |
346+
| Hopper (H100) | ghcr.io/huggingface/text-embeddings-inference:hopper-1.8 |
347+
| Blackwell 10.0 (B200, GB200, ...) | ghcr.io/huggingface/text-embeddings-inference:100-sha-ac69b50 (experimental) |
348+
| Blackwell 12.0 (GeForce RTX 50X0, ...) | ghcr.io/huggingface/text-embeddings-inference:120-sha-ac69b50 (experimental) |
347349

348350
**Warning**: Flash Attention is turned off by default for the Turing image as it suffers from precision issues.
349351
You can turn Flash Attention v1 ON by using the `USE_FLASH_ATTENTION=True` environment variable.

0 commit comments

Comments
 (0)