Skip to content

Commit 9492933

Browse files
committed
Update Nemotron 3 Nano Omni documentation and layout for Jetson Thor support
- Added build URL for Nemotron 3 Nano Omni. - Updated minimum Jetson model from Orin AGX to Thor. - Removed unsupported AGX Orin commands and references from the documentation. - Enhanced layout logic to manage module commands based on active inference engine. - Introduced SGLang support in the inference engines configuration.
1 parent f8673a9 commit 9492933

3 files changed

Lines changed: 44 additions & 41 deletions

File tree

src/content/models/nemotron-3-nano-omni.md

Lines changed: 5 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ license: "NVIDIA Open Model License"
1717
model_size: "21GB"
1818
hf_checkpoint: "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4"
1919
huggingface_url: "https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4"
20-
minimum_jetson: "Orin AGX"
20+
build_nvidia_url: "https://build.nvidia.com/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
21+
minimum_jetson: "Thor"
2122
supported_inference_engines:
2223
- engine: "vLLM"
2324
type: "Container"
@@ -60,20 +61,6 @@ supported_inference_engines:
6061
modules_supported:
6162
- thor_t5000
6263
- thor_t4000
63-
- orin_agx_64
64-
serve_command_orin: |-
65-
sudo docker run -it --rm --pull always \
66-
--runtime=nvidia --network host \
67-
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
68-
ghcr.io/nvidia-ai-iot/llama_cpp:latest-jetson-orin \
69-
llama-server \
70-
--hf-repo ggml-org/NVIDIA-Nemotron-3-Nano-Omni \
71-
--hf-file nemotron-3-nano-omni-ga_v1.0-Q4_K_M.gguf \
72-
--host 0.0.0.0 \
73-
--port 8080 \
74-
-c 8192 \
75-
-ngl 999 \
76-
--alias my_model
7764
serve_command_thor: |-
7865
sudo docker run -it --rm --pull always \
7966
--runtime=nvidia --network host \
@@ -110,8 +97,7 @@ Nemotron Nano 3 Omni is NVIDIA's multimodal reasoning model combining language,
11097

11198
## Supported Platforms
11299

113-
- Jetson Thor (NVFP4, FP8, BF16 — vLLM, SGLang, llama.cpp)
114-
- Jetson Orin AGX 64GB (llama.cpp)
100+
- Jetson Thor
115101

116102
## Running with vLLM
117103

@@ -173,28 +159,9 @@ curl -s http://${JETSON_HOST}:30000/v1/chat/completions \
173159

174160
<div class="device-tabs">
175161
<div class="device-tab-bar">
176-
<button class="device-tab active" data-target="orin-llama">Jetson Orin</button>
177-
<button class="device-tab" data-target="thor-llama">Jetson Thor</button>
178-
</div>
179-
<div class="device-panel" data-panel="orin-llama">
180-
181-
```bash
182-
sudo docker run -it --rm --pull always \
183-
--runtime=nvidia --network host \
184-
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
185-
ghcr.io/nvidia-ai-iot/llama_cpp:latest-jetson-orin \
186-
llama-server \
187-
--hf-repo ggml-org/NVIDIA-Nemotron-3-Nano-Omni \
188-
--hf-file nemotron-3-nano-omni-ga_v1.0-Q4_K_M.gguf \
189-
--host 0.0.0.0 \
190-
--port 8080 \
191-
-c 8192 \
192-
-ngl 999 \
193-
--alias my_model
194-
```
195-
162+
<button class="device-tab active" data-target="thor-llama">Jetson Thor</button>
196163
</div>
197-
<div class="device-panel hidden" data-panel="thor-llama">
164+
<div class="device-panel" data-panel="thor-llama">
198165

199166
```bash
200167
sudo docker run -it --rm --pull always \
@@ -231,8 +198,6 @@ curl -s http://127.0.0.1:8080/v1/chat/completions \
231198

232199
> **Note:** `--hf-repo ggml-org/NVIDIA-Nemotron-3-Nano-Omni` and `--hf-file nemotron-3-nano-omni-ga_v1.0-Q4_K_M.gguf` download the official GGUF checkpoint from Hugging Face. `-ngl 999` offloads all layers to GPU. `--alias my_model` sets the model name used in API requests. `chat_template_kwargs: {"enable_thinking": true}` activates chain-of-thought reasoning.
233200
234-
-->
235-
236201
## Running with TensorRT Edge-LLM
237202

238203
TensorRT Edge-LLM support for this model is currently Jetson Thor only. It requires exporting the model to ONNX and building TensorRT engines before running inference. See the [TensorRT Edge-LLM GitHub repository](https://github.com/NVIDIA/TensorRT-Edge-LLM) and the [export/build quick start](https://nvidia.github.io/TensorRT-Edge-LLM/user_guide/getting_started/quick-start-guide.html) for the full setup flow.

src/layouts/Layout.astro

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -741,9 +741,38 @@ const canonicalURL = new URL(Astro.url.pathname, Astro.site);
741741
syncRunModalMatrixEngineRail(modal, meta);
742742
return;
743743
}
744+
var activeEngine = modal.getAttribute('data-active-engine') || meta.defaultEngineId || '';
745+
var engineSupportMap = meta.modulesSupportedByEngine || {};
746+
var allowedForActiveEngine = null;
747+
if (Object.prototype.hasOwnProperty.call(engineSupportMap, activeEngine)) {
748+
allowedForActiveEngine = engineSupportMap[activeEngine];
749+
} else {
750+
for (var ek in engineSupportMap) {
751+
if (Object.prototype.hasOwnProperty.call(engineSupportMap, ek) && normalizeInferenceEngineKey(ek) === normalizeInferenceEngineKey(activeEngine)) {
752+
allowedForActiveEngine = engineSupportMap[ek];
753+
break;
754+
}
755+
}
756+
}
757+
function commandForModuleAndEngine(moduleId, engineName) {
758+
var commands = meta.moduleCommandMatrix && meta.moduleCommandMatrix[moduleId];
759+
if (!commands) return '';
760+
for (var ck in commands) {
761+
if (Object.prototype.hasOwnProperty.call(commands, ck) && normalizeInferenceEngineKey(ck) === normalizeInferenceEngineKey(engineName)) {
762+
return commands[ck] || '';
763+
}
764+
}
765+
return '';
766+
}
744767
modal.querySelectorAll('[data-run-module-tab]').forEach(function (btn) {
745768
var global = btn.getAttribute('data-run-module-global-disabled') === '1';
746-
var dis = global;
769+
var moduleId = btn.getAttribute('data-run-module-tab') || '';
770+
var unsupportedByEngine = false;
771+
if (allowedForActiveEngine != null && Array.isArray(allowedForActiveEngine)) {
772+
unsupportedByEngine = allowedForActiveEngine.length === 0 || allowedForActiveEngine.indexOf(moduleId) === -1;
773+
}
774+
var hasCmd = !!String(commandForModuleAndEngine(moduleId, activeEngine)).trim();
775+
var dis = global || unsupportedByEngine || !hasCmd;
747776
btn.disabled = dis;
748777
if (dis) {
749778
applyMatrixModuleTabDisabledVisual(btn);

src/lib/engines.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,15 @@ export const INFERENCE_ENGINES: Record<string, InferenceEngine> = {
3333
return `vllm serve ${checkpoint}`;
3434
}
3535
},
36+
sglang: {
37+
id: 'sglang',
38+
label: 'SGLang',
39+
supports: ['Text', 'Multimodal'],
40+
buildCommand: ({ options }) => {
41+
const checkpoint = options?.checkpoint || 'model/checkpoint';
42+
return `sglang serve ${checkpoint}`;
43+
}
44+
},
3645
edgemllm: {
3746
id: 'edgemllm',
3847
label: 'EdgeLLM',

0 commit comments

Comments
 (0)