Skip to content

Commit ac05d39

Browse files
prekshivyasprekshivyas
committed
fix(inference): bound Spark llama.cpp request bodies
1 parent 01e0b92 commit ac05d39

12 files changed

Lines changed: 279 additions & 38 deletions

‎managed-inference/images/llama-cpp/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ RUN apt-get update \
2929
build-essential=12.10ubuntu1 \
3030
ca-certificates=20260601~24.04.1 \
3131
cmake=3.28.3-1build7 \
32-
curl=8.5.0-2ubuntu10.11 \
32+
curl=8.5.0-2ubuntu10.12 \
3333
g++-14=14.2.0-4ubuntu2~24.04.1 \
3434
gcc-14=14.2.0-4ubuntu2~24.04.1 \
3535
golang-go=2:1.22~2build1 \
36-
libcurl4-openssl-dev=8.5.0-2ubuntu10.11 \
36+
libcurl4-openssl-dev=8.5.0-2ubuntu10.12 \
3737
libssl-dev=3.0.13-0ubuntu3.12 \
3838
&& rm -rf /var/lib/apt/lists/*
3939

@@ -123,7 +123,7 @@ RUN test -n "${CUDA_DEV_IMAGE}" \
123123
RUN apt-get update \
124124
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
125125
ca-certificates=20260601~24.04.1 \
126-
libcurl4t64=8.5.0-2ubuntu10.11 \
126+
libcurl4t64=8.5.0-2ubuntu10.12 \
127127
libgomp1=14.2.0-4ubuntu2~24.04.1 \
128128
&& groupadd --gid "${RUNTIME_GID}" nemoclaw-llama \
129129
&& useradd --uid "${RUNTIME_UID}" --gid "${RUNTIME_GID}" \

‎managed-inference/images/llama-cpp/image.yaml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ spec:
7272
- context-window
7373
- authentication
7474
- malformed-request
75+
- request-body-limit
7576
- cancellation
7677
- client-timeout
7778
- log-redaction
@@ -89,8 +90,8 @@ spec:
8990

9091
source:
9192
repository: https://github.com/ggml-org/llama.cpp
92-
revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55
93-
archiveSha256: sha256:975f70723e053785e894f4e1d9cf770f2f1a7bc762fd3af174ff5635014108b6
93+
revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f
94+
archiveSha256: sha256:45a24299e7a24410624489d19924d492bc71a120fa17d9b7cb32f6d5c4f1aed0
9495

9596
cuda:
9697
developmentBase: docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52
@@ -115,11 +116,11 @@ spec:
115116
build-essential: 12.10ubuntu1
116117
ca-certificates: 20260601~24.04.1
117118
cmake: 3.28.3-1build7
118-
curl: 8.5.0-2ubuntu10.11
119+
curl: 8.5.0-2ubuntu10.12
119120
g++-14: 14.2.0-4ubuntu2~24.04.1
120121
gcc-14: 14.2.0-4ubuntu2~24.04.1
121122
golang-go: 2:1.22~2build1
122-
libcurl4-openssl-dev: 8.5.0-2ubuntu10.11
123+
libcurl4-openssl-dev: 8.5.0-2ubuntu10.12
123124
libssl-dev: 3.0.13-0ubuntu3.12
124125
cmake:
125126
ggmlBackendDl: true
@@ -162,7 +163,7 @@ spec:
162163
- /usr/bin/sh
163164
packages:
164165
ca-certificates: 20260601~24.04.1
165-
libcurl4t64: 8.5.0-2ubuntu10.11
166+
libcurl4t64: 8.5.0-2ubuntu10.12
166167
libgomp1: 14.2.0-4ubuntu2~24.04.1
167168
writablePaths:
168169
- /tmp

‎managed-inference/images/llama-cpp/request-guard/main.go‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -97,27 +97,29 @@ func requireExactCommandMarker(command []string, option string) error {
9797

9898
func validateSupportedCommandOptions(command []string) error {
9999
allowed := map[string]bool{
100-
"--alias": true,
101-
"--api-key-file": true,
102-
"--batch-size": true,
103-
"--cache-type-k": true,
104-
"--cache-type-v": true,
105-
"--ctx-size": true,
106-
"--flash-attn": true,
107-
"--gpu-layers": true,
108-
"--host": true,
109-
"--metrics": false,
110-
"--model": true,
111-
"--no-agent": false,
112-
"--no-mmproj": false,
113-
"--no-slots": false,
114-
"--no-ui": false,
115-
"--n-predict": true,
116-
"--parallel": true,
117-
"--port": true,
118-
"--sleep-idle-seconds": true,
119-
"--timeout": true,
120-
"--ubatch-size": true,
100+
"--alias": true,
101+
"--api-key-file": true,
102+
"--batch-size": true,
103+
"--cache-type-k": true,
104+
"--cache-type-v": true,
105+
"--chat-template-kwargs": true,
106+
"--ctx-size": true,
107+
"--flash-attn": true,
108+
"--gpu-layers": true,
109+
"--host": true,
110+
"--jinja": false,
111+
"--metrics": false,
112+
"--model": true,
113+
"--no-agent": false,
114+
"--no-mmproj": false,
115+
"--no-slots": false,
116+
"--no-ui": false,
117+
"--n-predict": true,
118+
"--parallel": true,
119+
"--port": true,
120+
"--sleep-idle-seconds": true,
121+
"--timeout": true,
122+
"--ubatch-size": true,
121123
}
122124
seen := make(map[string]bool, len(allowed))
123125
for index := 0; index < len(command); index++ {

‎managed-inference/images/llama-cpp/request-guard/main_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ func TestParseConfigRequiresEveryDeclaredValue(t *testing.T) {
119119
"--port", "8082",
120120
"--api-key-file", llamaServerAPIKeyPath,
121121
"--n-predict", "4096",
122+
"--jinja",
123+
"--chat-template-kwargs", `{"reasoning_strength":"low"}`,
122124
"--no-ui",
123125
"--no-slots",
124126
"--no-mmproj",

‎managed-inference/recipes/llama-cpp.muse-glimmer-30b.spark-single.v1.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ spec:
8888
value: f16
8989
speculativeDecoding: disabled
9090
limits:
91-
maxRequestBodyBytes: 1048576
91+
maxRequestBodyBytes: 16384
9292
maxRequestHeaderBytes: 32768
9393
maxOutputTokens: 4096
9494
requestTimeoutSeconds: 900

‎managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ spec:
1616
technology: llama.cpp
1717
source:
1818
repository: ggml-org/llama.cpp
19-
revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55
19+
revision: 8e7f22b67ef4667b4ddd50230771287f328cfb3f
2020

2121
model:
2222
id: unsloth/Nemotron-3-Nano-30B-A3B-GGUF
@@ -86,7 +86,7 @@ spec:
8686
value: f16
8787
speculativeDecoding: disabled
8888
limits:
89-
maxRequestBodyBytes: 1048576
89+
maxRequestBodyBytes: 32768
9090
maxRequestHeaderBytes: 32768
9191
maxOutputTokens: 4096
9292
requestTimeoutSeconds: 900

‎scripts/checks/export-llama-cpp-image-config.mts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -625,11 +625,11 @@ export function loadLlamaCppImageConfig(
625625
"build-essential": "12.10ubuntu1",
626626
"ca-certificates": "20260601~24.04.1",
627627
cmake: "3.28.3-1build7",
628-
curl: "8.5.0-2ubuntu10.11",
628+
curl: "8.5.0-2ubuntu10.12",
629629
"g++-14": "14.2.0-4ubuntu2~24.04.1",
630630
"gcc-14": "14.2.0-4ubuntu2~24.04.1",
631631
"golang-go": "2:1.22~2build1",
632-
"libcurl4-openssl-dev": "8.5.0-2ubuntu10.11",
632+
"libcurl4-openssl-dev": "8.5.0-2ubuntu10.12",
633633
"libssl-dev": "3.0.13-0ubuntu3.12",
634634
};
635635
const expectedCompiler = {
@@ -639,7 +639,7 @@ export function loadLlamaCppImageConfig(
639639
};
640640
const expectedRuntimePackages = {
641641
"ca-certificates": "20260601~24.04.1",
642-
libcurl4t64: "8.5.0-2ubuntu10.11",
642+
libcurl4t64: "8.5.0-2ubuntu10.12",
643643
libgomp1: "14.2.0-4ubuntu2~24.04.1",
644644
};
645645
const expectedRequiredPaths = [

‎scripts/checks/llama-cpp-dgx-spark-protocol-qualification.mts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import {
55
LLAMA_CPP_DGX_SPARK_PROTOCOL_PROBES,
6+
LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES,
67
LLAMA_CPP_DGX_SPARK_REQUIRED_METRIC_SERIES,
78
type LlamaCppDgxSparkExecutionPlan,
89
type LlamaCppDgxSparkQualificationReceipt,
@@ -164,6 +165,51 @@ function jsonRequest(
164165
};
165166
}
166167

168+
function exactSizeChatRequest(model: string, targetBytes: number): string {
169+
const request = {
170+
max_tokens: 1,
171+
messages: [{ content: "", role: "user" }],
172+
model,
173+
temperature: 0,
174+
};
175+
const emptyBody = JSON.stringify(request);
176+
const contentBytes = targetBytes - new TextEncoder().encode(emptyBody).byteLength;
177+
if (contentBytes < 0) throw new Error("request-body probe target is too small");
178+
request.messages[0].content = "x".repeat(contentBytes);
179+
const body = JSON.stringify(request);
180+
if (new TextEncoder().encode(body).byteLength !== targetBytes) {
181+
throw new Error("request-body probe did not construct the exact declared size");
182+
}
183+
return body;
184+
}
185+
186+
function exactSizeJsonRequest(
187+
authorization: string,
188+
body: string,
189+
timeoutMilliseconds: number,
190+
): RequestInit {
191+
return {
192+
body,
193+
headers: {
194+
Authorization: authorization,
195+
"Content-Type": "application/json",
196+
},
197+
method: "POST",
198+
signal: requestSignal(timeoutMilliseconds),
199+
};
200+
}
201+
202+
function validateRequestBodyLimitError(value: unknown): void {
203+
if (
204+
!isRecord(value) ||
205+
!isRecord(value.error) ||
206+
value.error.code !== "request_body_too_large" ||
207+
value.error.type !== "invalid_request_error"
208+
) {
209+
throw new Error("request-body limit probe did not return the declared error contract");
210+
}
211+
}
212+
167213
function usageFrom(value: unknown): ProtocolEvidence["usage"] {
168214
if (!isRecord(value)) throw new Error("chat usage was not returned");
169215
const promptTokens = value.prompt_tokens;
@@ -727,6 +773,73 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: {
727773
await expectStatus(malformedResponse, 400, bounds.maxResponseBytes, "malformed-request probe");
728774
executedProbes.add("malformed-request");
729775

776+
const acceptedRequestBytes = plan.recipe.serve.limits.maxRequestBodyBytes;
777+
const acceptedResponse = await fetchImpl(
778+
chatUrl,
779+
exactSizeJsonRequest(
780+
authorization,
781+
exactSizeChatRequest(model, acceptedRequestBytes),
782+
timeoutMilliseconds,
783+
),
784+
);
785+
await expectStatus(
786+
acceptedResponse,
787+
200,
788+
bounds.maxResponseBytes,
789+
"request-body boundary probe",
790+
);
791+
792+
const rejectedResponse = await fetchImpl(
793+
chatUrl,
794+
exactSizeJsonRequest(
795+
authorization,
796+
exactSizeChatRequest(model, LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES),
797+
timeoutMilliseconds,
798+
),
799+
);
800+
validateRequestBodyLimitError(
801+
await readJson(
802+
rejectedResponse,
803+
413,
804+
bounds.maxResponseBytes,
805+
"oversized request-body probe",
806+
),
807+
);
808+
809+
const continuationHealthResponse = await fetchImpl(`${baseUrl}/health`, {
810+
headers: { Authorization: authorization },
811+
signal: requestSignal(bounds.clientTimeoutMilliseconds),
812+
});
813+
await expectStatus(
814+
continuationHealthResponse,
815+
200,
816+
bounds.maxResponseBytes,
817+
"request-body continuation health probe",
818+
);
819+
const bodyLimitContinuationResponse = await fetchImpl(
820+
chatUrl,
821+
jsonRequest(
822+
authorization,
823+
{
824+
max_tokens: bounds.maxTokens.synchronousChat,
825+
messages: [{ content: "Return one short continuation token.", role: "user" }],
826+
model,
827+
temperature: 0,
828+
},
829+
timeoutMilliseconds,
830+
),
831+
);
832+
validateChatCompletionResponse(
833+
await readJson(
834+
bodyLimitContinuationResponse,
835+
200,
836+
bounds.maxResponseBytes,
837+
"request-body continuation completion probe",
838+
),
839+
model,
840+
);
841+
executedProbes.add("request-body-limit");
842+
730843
const synchronousResponse = await fetchImpl(
731844
chatUrl,
732845
jsonRequest(
@@ -914,6 +1027,17 @@ export async function runLlamaCppDgxSparkProtocolQualification(options: {
9141027
},
9151028
health: { httpStatus: 200, ok: true },
9161029
malformedRequest: { httpStatus: 400, ok: true },
1030+
requestBodyLimit: {
1031+
acceptedBytes: acceptedRequestBytes,
1032+
acceptedHttpStatus: 200,
1033+
continuationHealthHttpStatus: 200,
1034+
continuationHttpStatus: 200,
1035+
errorCode: "request_body_too_large",
1036+
errorType: "invalid_request_error",
1037+
ok: true,
1038+
rejectedBytes: LLAMA_CPP_DGX_SPARK_REJECTED_REQUEST_BODY_BYTES,
1039+
rejectedHttpStatus: 413,
1040+
},
9171041
models: { httpStatus: 200, model, ok: true },
9181042
metrics,
9191043
properties: propertiesEvidence.properties,

0 commit comments

Comments
 (0)