Skip to content

Commit b66a056

Browse files
alvarobarttalvarobartt
committed
Back to dynamic-linking + add 8.9 target (temporarily)
1 parent b9ea648 commit b66a056

2 files changed

Lines changed: 33 additions & 21 deletions

File tree

‎Dockerfile-cuda-all‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,9 @@ COPY --from=planner /usr/src/recipe.json recipe.json
5151

5252
FROM builder-base AS builder-75
5353

54-
RUN nvprune --generate-code code=sm_75 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a
55-
5654
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
5755
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
58-
CUDA_COMPUTE_CAP=75 cargo chef cook --release --features candle-cuda-turing --features static-linking --no-default-features --recipe-path recipe.json && sccache -s
56+
CUDA_COMPUTE_CAP=75 cargo chef cook --release --features candle-cuda-turing --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
5957

6058
COPY backends backends
6159
COPY core core
@@ -65,17 +63,15 @@ COPY Cargo.lock Cargo.lock
6563

6664
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
6765
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
68-
CUDA_COMPUTE_CAP=75 cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F static-linking -F http --no-default-features && \
66+
CUDA_COMPUTE_CAP=75 cargo build --release --bin text-embeddings-router -F candle-cuda-turing -F dynamic-linking -F http --no-default-features && \
6967
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-75 && \
7068
sccache -s
7169

7270
FROM builder-base AS builder-80
7371

74-
RUN nvprune --generate-code code=sm_80 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a
75-
7672
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
7773
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
78-
CUDA_COMPUTE_CAP=80 cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s
74+
CUDA_COMPUTE_CAP=80 cargo chef cook --release --features candle-cuda --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
7975

8076
COPY backends backends
8177
COPY core core
@@ -85,17 +81,33 @@ COPY Cargo.lock Cargo.lock
8581

8682
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
8783
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
88-
CUDA_COMPUTE_CAP=80 cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && \
84+
CUDA_COMPUTE_CAP=80 cargo build --release --bin text-embeddings-router -F candle-cuda -F dynamic-linking -F http --no-default-features && \
8985
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-80 && \
9086
sccache -s
9187

92-
FROM builder-base AS builder-90
88+
FROM builder-base AS builder-89
9389

94-
RUN nvprune --generate-code code=sm_90 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a
90+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
91+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
92+
CUDA_COMPUTE_CAP=89 cargo chef cook --release --features candle-cuda --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
93+
94+
COPY backends backends
95+
COPY core core
96+
COPY router router
97+
COPY Cargo.toml Cargo.toml
98+
COPY Cargo.lock Cargo.lock
9599

96100
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
97101
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
98-
CUDA_COMPUTE_CAP=90 cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s
102+
CUDA_COMPUTE_CAP=89 cargo build --release --bin text-embeddings-router -F candle-cuda -F dynamic-linking -F http --no-default-features && \
103+
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-89 && \
104+
sccache -s
105+
106+
FROM builder-base AS builder-90
107+
108+
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
109+
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
110+
CUDA_COMPUTE_CAP=90 cargo chef cook --release --features candle-cuda --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
99111

100112
COPY backends backends
101113
COPY core core
@@ -105,17 +117,15 @@ COPY Cargo.lock Cargo.lock
105117

106118
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
107119
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
108-
CUDA_COMPUTE_CAP=90 cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && \
120+
CUDA_COMPUTE_CAP=90 cargo build --release --bin text-embeddings-router -F candle-cuda -F dynamic-linking -F http --no-default-features && \
109121
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-90 && \
110122
sccache -s
111123

112124
FROM builder-base AS builder-100
113125

114-
RUN nvprune --generate-code code=sm_100 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a
115-
116126
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
117127
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
118-
CUDA_COMPUTE_CAP=100 cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s
128+
CUDA_COMPUTE_CAP=100 cargo chef cook --release --features candle-cuda --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
119129

120130
COPY backends backends
121131
COPY core core
@@ -125,17 +135,15 @@ COPY Cargo.lock Cargo.lock
125135

126136
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
127137
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
128-
CUDA_COMPUTE_CAP=100 cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && \
138+
CUDA_COMPUTE_CAP=100 cargo build --release --bin text-embeddings-router -F candle-cuda -F dynamic-linking -F http --no-default-features && \
129139
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-100 && \
130140
sccache -s
131141

132142
FROM builder-base AS builder-120
133143

134-
RUN nvprune --generate-code code=sm_120 /usr/local/cuda/lib64/libcublas_static.a -o /usr/local/cuda/lib64/libcublas_static.a
135-
136144
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
137145
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
138-
CUDA_COMPUTE_CAP=120 cargo chef cook --release --features candle-cuda --features static-linking --no-default-features --recipe-path recipe.json && sccache -s
146+
CUDA_COMPUTE_CAP=120 cargo chef cook --release --features candle-cuda --features dynamic-linking --no-default-features --recipe-path recipe.json && sccache -s
139147

140148
COPY backends backends
141149
COPY core core
@@ -145,7 +153,7 @@ COPY Cargo.lock Cargo.lock
145153

146154
RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \
147155
--mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \
148-
CUDA_COMPUTE_CAP=120 cargo build --release --bin text-embeddings-router -F candle-cuda -F static-linking -F http --no-default-features && \
156+
CUDA_COMPUTE_CAP=120 cargo build --release --bin text-embeddings-router -F candle-cuda -F dynamic-linking -F http --no-default-features && \
149157
mv /usr/src/target/release/text-embeddings-router /usr/src/target/release/text-embeddings-router-120 && \
150158
sccache -s
151159

@@ -163,10 +171,12 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
163171
libssl-dev \
164172
curl \
165173
cuda-compat-12-9 \
174+
libcublas-12-9 \
166175
&& rm -rf /var/lib/apt/lists/*
167176

168177
COPY --from=builder-75 /usr/src/target/release/text-embeddings-router-75 /usr/local/bin/text-embeddings-router-75
169178
COPY --from=builder-80 /usr/src/target/release/text-embeddings-router-80 /usr/local/bin/text-embeddings-router-80
179+
COPY --from=builder-89 /usr/src/target/release/text-embeddings-router-89 /usr/local/bin/text-embeddings-router-89
170180
COPY --from=builder-90 /usr/src/target/release/text-embeddings-router-90 /usr/local/bin/text-embeddings-router-90
171181
COPY --from=builder-100 /usr/src/target/release/text-embeddings-router-100 /usr/local/bin/text-embeddings-router-100
172182
COPY --from=builder-120 /usr/src/target/release/text-embeddings-router-120 /usr/local/bin/text-embeddings-router-120

‎cuda-all-entrypoint.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@ compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv | sed -n '2p' | se
3030

3131
if [ ${compute_cap} -eq 75 ]; then
3232
exec text-embeddings-router-75 "$@"
33-
elif [ ${compute_cap} -ge 80 -a ${compute_cap} -lt 90 ]; then
33+
elif [ ${compute_cap} -ge 80 -a ${compute_cap} -lt 89 ]; then
3434
exec text-embeddings-router-80 "$@"
35+
elif [ ${compute_cap} -eq 89 ]; then
36+
exec text-embeddings-router-89 "$@"
3537
elif [ ${compute_cap} -eq 90 ]; then
3638
exec text-embeddings-router-90 "$@"
3739
elif [ ${compute_cap} -eq 100 ]; then

0 commit comments

Comments
 (0)