Skip to content

Commit 1a5eeac

Browse files
alvarobarttalvarobartt
authored
Merge branch 'main' into feat-pplx-embed-1
2 parents 7577835 + c9902fd commit 1a5eeac

41 files changed

Lines changed: 376 additions & 465 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ on:
44
workflow_dispatch:
55
push:
66
branches:
7-
- 'main'
7+
- "main"
88
tags:
9-
- 'v*'
9+
- "v*"
1010
pull_request:
1111
paths:
1212
- ".github/workflows/build.yaml"
@@ -19,7 +19,7 @@ on:
1919
- "rust-toolchain.toml"
2020
- "Dockerfile"
2121
branches:
22-
- 'main'
22+
- "main"
2323

2424
jobs:
2525
matrix:
@@ -126,7 +126,7 @@ jobs:
126126
context: .
127127
file: ${{ matrix.dockerfile }}
128128
push: ${{ github.event_name != 'pull_request' }}
129-
platforms: 'linux/amd64'
129+
platforms: "linux/amd64"
130130
build-args: |
131131
SCCACHE_GHA_ENABLED=${{ matrix.sccache }}
132132
CUDA_COMPUTE_CAP=${{ matrix.cudaComputeCap }}
@@ -168,7 +168,7 @@ jobs:
168168
target: grpc
169169
file: ${{ matrix.dockerfile }}
170170
push: ${{ github.event_name != 'pull_request' }}
171-
platforms: 'linux/amd64'
171+
platforms: "linux/amd64"
172172
build-args: |
173173
SCCACHE_GHA_ENABLED=${{ matrix.sccache }}
174174
CUDA_COMPUTE_CAP=${{ matrix.cudaComputeCap }}

‎.github/workflows/integration-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Run integration tests
33
on:
44
workflow_dispatch:
55
schedule:
6-
- cron: '0 0 * * *' # Run the workflow nightly to check Gaudi integration is working
6+
- cron: "0 0 * * *" # Run the workflow nightly to check Gaudi integration is working
77

88
jobs:
99
tests:

.github/workflows/liniting.yaml renamed to .github/workflows/linting.yaml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,8 @@ jobs:
1919
steps:
2020
- uses: actions/checkout@v6
2121
- name: Install Rust
22-
uses: actions-rs/toolchain@v1
22+
uses: actions-rust-lang/setup-rust-toolchain@v1.15.2
2323
with:
24-
# Released on: 28 December, 2023
25-
# Branched from master on: 10 November, 2023
26-
# https://releases.rs/docs/1.85.0/
27-
toolchain: 1.85.0
28-
override: true
2924
components: rustfmt, clippy
3025
- name: Install Protoc
3126
uses: arduino/setup-protoc@v1

‎.github/workflows/matrix.json‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,22 +45,14 @@
4545
"grpc": true,
4646
"dockerfile": "Dockerfile-cuda"
4747
},
48-
{
49-
"name": "turing-ampere-ada-hopper",
50-
"imageNamePrefix": "cuda-",
51-
"runOn": "always",
52-
"sccache": false,
53-
"grpc": false,
54-
"dockerfile": "Dockerfile-cuda-all"
55-
},
5648
{
5749
"name": "blackwell-100",
5850
"imageNamePrefix": "100-",
5951
"runOn": "always",
6052
"sccache": true,
6153
"cudaComputeCap": 100,
6254
"grpc": true,
63-
"dockerfile": "Dockerfile-cuda-blackwell"
55+
"dockerfile": "Dockerfile-cuda"
6456
},
6557
{
6658
"name": "blackwell-120",
@@ -69,15 +61,15 @@
6961
"sccache": true,
7062
"cudaComputeCap": 120,
7163
"grpc": true,
72-
"dockerfile": "Dockerfile-cuda-blackwell"
64+
"dockerfile": "Dockerfile-cuda"
7365
},
7466
{
75-
"name": "blackwell",
76-
"imageNamePrefix": "blackwell-",
67+
"name": "all",
68+
"imageNamePrefix": "cuda-",
7769
"runOn": "always",
7870
"sccache": false,
7971
"grpc": false,
80-
"dockerfile": "Dockerfile-cuda-blackwell-all"
72+
"dockerfile": "Dockerfile-cuda-all"
8173
},
8274
{
8375
"name": "cpu",

‎.github/workflows/test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
steps:
3030
- name: Checkout repository
3131
uses: actions/checkout@v6
32-
- uses: actions-rust-lang/setup-rust-toolchain@v1
32+
- uses: actions-rust-lang/setup-rust-toolchain@v1.15.2
3333
- name: Run sccache-cache
3434
uses: mozilla-actions/sccache-action@v0.0.9
3535
with:

‎Cargo.lock‎

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ default-members = [
2222
resolver = "2"
2323

2424
[workspace.package]
25-
version = "1.8.3"
25+
version = "1.9.1"
2626
edition = "2021"
2727
authors = ["Olivier Dehaene", "Nicolas Patry", "Alvaro Bartolome"]
2828
homepage = "https://github.com/huggingface/text-embeddings-inference"

‎Dockerfile-cuda‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 AS base-builder
1+
FROM nvidia/cuda:12.9.1-devel-ubuntu24.04 AS base-builder
22

33
ENV SCCACHE=0.10.0
44
ENV RUSTC_WRAPPER=/usr/local/bin/sccache
@@ -58,6 +58,12 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
5858
elif [ ${CUDA_COMPUTE_CAP} -eq 90 ]; \
5959
then \
6060
nvprune --generate-code code=sm_90 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
61+
elif [ ${CUDA_COMPUTE_CAP} -eq 100 ]; \
62+
then \
63+
nvprune --generate-code code=sm_100 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
64+
elif [ ${CUDA_COMPUTE_CAP} -eq 120 ]; \
65+
then \
66+
nvprune --generate-code code=sm_120 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a; \
6167
else \
6268
echo "cuda compute cap ${CUDA_COMPUTE_CAP} is not supported"; exit 1; \
6369
fi;
@@ -113,32 +119,33 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
113119
cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F grpc --no-default-features && sccache -s; \
114120
fi;
115121

116-
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS base
122+
FROM nvidia/cuda:12.9.1-runtime-ubuntu24.04 AS base
117123

118124
ARG DEFAULT_USE_FLASH_ATTENTION=True
119125

120126
ENV HUGGINGFACE_HUB_CACHE=/data \
121127
PORT=80 \
122128
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
123-
LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
129+
LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
124130

125131
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
126132
ca-certificates \
127133
libssl-dev \
128134
curl \
129-
cuda-compat-12-6 \
135+
cuda-compat-12-9 \
130136
&& rm -rf /var/lib/apt/lists/*
131137

138+
COPY --chmod=775 cuda-entrypoint.sh entrypoint.sh
139+
132140
FROM base AS grpc
133141

134142
COPY --from=grpc-builder /usr/src/target/release/text-embeddings-router /usr/local/bin/text-embeddings-router
135143

136-
ENTRYPOINT ["text-embeddings-router"]
144+
ENTRYPOINT ["./entrypoint.sh"]
137145
CMD ["--json-output"]
138146

139147
FROM base
140148

141149
COPY --from=http-builder /usr/src/target/release/text-embeddings-router /usr/local/bin/text-embeddings-router
142-
143-
ENTRYPOINT ["text-embeddings-router"]
150+
ENTRYPOINT ["./entrypoint.sh"]
144151
CMD ["--json-output"]

‎Dockerfile-cuda-all‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 AS base-builder
1+
FROM nvidia/cuda:12.9.1-devel-ubuntu24.04 AS base-builder
22

33
ENV SCCACHE=0.10.0
44
ENV RUSTC_WRAPPER=/usr/local/bin/sccache
@@ -64,6 +64,14 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
6464
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
6565
CUDA_COMPUTE_CAP=90 cargo chef cook --release --features candle-cuda --recipe-path recipe.json && sccache -s;
6666

67+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
68+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
69+
CUDA_COMPUTE_CAP=100 cargo chef cook --release --features candle-cuda --recipe-path recipe.json && sccache -s;
70+
71+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
72+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
73+
CUDA_COMPUTE_CAP=120 cargo chef cook --release --features candle-cuda --recipe-path recipe.json && sccache -s;
74+
6775
COPY backends backends
6876
COPY core core
6977
COPY router router
@@ -88,25 +96,39 @@ RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
8896

8997
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-90
9098

91-
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS base
99+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
100+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
101+
CUDA_COMPUTE_CAP=100 cargo build --release --bin text-embeddings-router -F candle-cuda && sccache -s;
102+
103+
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-100
104+
105+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
106+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
107+
CUDA_COMPUTE_CAP=120 cargo build --release --bin text-embeddings-router -F candle-cuda && sccache -s;
108+
109+
RUN mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-120
110+
111+
FROM nvidia/cuda:12.9.1-runtime-ubuntu24.04 AS base
92112

93113
ARG DEFAULT_USE_FLASH_ATTENTION=True
94114

95115
ENV HUGGINGFACE_HUB_CACHE=/data \
96116
PORT=80 \
97117
USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
98-
LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
118+
LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
99119

100120
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
101121
ca-certificates \
102122
libssl-dev \
103123
curl \
104-
cuda-compat-12-6 \
124+
cuda-compat-12-9 \
105125
&& rm -rf /var/lib/apt/lists/*
106126

107127
COPY --from=builder /usr/src/target/release/text-embeddings-router-75 /usr/local/bin/text-embeddings-router-75
108128
COPY --from=builder /usr/src/target/release/text-embeddings-router-80 /usr/local/bin/text-embeddings-router-80
109129
COPY --from=builder /usr/src/target/release/text-embeddings-router-90 /usr/local/bin/text-embeddings-router-90
130+
COPY --from=builder /usr/src/target/release/text-embeddings-router-100 /usr/local/bin/text-embeddings-router-100
131+
COPY --from=builder /usr/src/target/release/text-embeddings-router-120 /usr/local/bin/text-embeddings-router-120
110132

111133
COPY --chmod=775 cuda-all-entrypoint.sh entrypoint.sh
112134

0 commit comments

Comments
 (0)