|
| 1 | +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +# NIM Service — credential-free inference via a Hugging Face model. |
| 16 | +# Serves an ungated Hugging Face model through NIM without any NGC credential: |
| 17 | +# no NGC_API_KEY and no registry pull secret. |
| 18 | +# |
| 19 | +# How it avoids credentials: |
| 20 | +# - NIM_MODEL_NAME uses an hf:// URI, which puts the operator on its Hugging |
| 21 | +# Face path. There it marks NGC_API_KEY optional and injects HF_TOKEN from |
| 22 | +# the same authSecret, so the secret need not carry an NGC key at all. |
| 23 | +# - This repository (nim/meta/llama-3.1-8b-instruct) serves anonymous |
| 24 | +# registry tokens, so no pullSecrets are required. Availability is |
| 25 | +# per-repository, not namespace-wide: verify any other image before |
| 26 | +# assuming it pulls anonymously. The generic Multi-LLM image |
| 27 | +# nvcr.io/nim/nvidia/llm-nim is gated and does need a pull secret. |
| 28 | +# |
| 29 | +# spec.authSecret is required by the NIMService schema, so the secret must |
| 30 | +# exist and must contain an HF_TOKEN key. An empty value is sufficient for an |
| 31 | +# ungated model; a gated Hugging Face repository needs a real token. |
| 32 | +# |
| 33 | +# Off-label combination: this is a Llama-specific NIM image serving a Hugging |
| 34 | +# Face model it was not built for. The image keeps its own profile identity |
| 35 | +# (logs report model_tag meta/llama-3.1-8b-instruct) while vLLM serves the |
| 36 | +# downloaded Hugging Face weights. It works, but it is not the image's |
| 37 | +# advertised use, and "verified" below means the service loads and returns a |
| 38 | +# completion -- a liveness check. An off-label pairing can still apply the |
| 39 | +# wrong chat template or tokenizer and degrade output quality silently, so |
| 40 | +# spot-check generations before relying on this for anything but a smoke |
| 41 | +# test. The |
| 42 | +# image intended for arbitrary hf:// models is nvcr.io/nim/nvidia/llm-nim, |
| 43 | +# which is a gated repository and therefore needs an NGC pull secret -- |
| 44 | +# choosing it trades the credential-free property for a supported pairing. |
| 45 | +# Pin the tag (not latest) so the pairing you validated is the one you ship. |
| 46 | +# The hf:// capability is version-dependent: 2.0.10 honours it, while 2.0.9 |
| 47 | +# ignores NIM_MODEL_NAME, falls back to its own NGC profile, and fails with an |
| 48 | +# authentication error. Re-verify before moving this pin. |
| 49 | +# |
| 50 | +# Prerequisites: |
| 51 | +# - k8s-nim-operator deployed (via AICR NIM recipe); no credential needed |
| 52 | +# |
| 53 | +# Setup: |
| 54 | +# kubectl create ns nim-workload |
| 55 | +# kubectl create secret generic hf-secret \ |
| 56 | +# --from-literal=HF_TOKEN="" -n nim-workload |
| 57 | +# |
| 58 | +# Deploy: |
| 59 | +# kubectl apply -f nimservice-hf-nocred.yaml |
| 60 | +# |
| 61 | +# Test: |
| 62 | +# kubectl port-forward svc/qwen3-nocred 8000:8000 -n nim-workload |
| 63 | +# curl http://localhost:8000/v1/models |
| 64 | +# curl http://localhost:8000/v1/chat/completions \ |
| 65 | +# -H "Content-Type: application/json" \ |
| 66 | +# -d '{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"Hello!"}],"max_tokens":30}' |
| 67 | +# |
| 68 | +# Node scheduling below targets an AKS GPU pool. Adjust nodeSelector and |
| 69 | +# tolerations for other platforms (EKS/GKE use different pool labels and taints). |
| 70 | + |
| 71 | +apiVersion: v1 |
| 72 | +kind: Namespace |
| 73 | +metadata: |
| 74 | + name: nim-workload |
| 75 | +--- |
| 76 | +apiVersion: apps.nvidia.com/v1alpha1 |
| 77 | +kind: NIMService |
| 78 | +metadata: |
| 79 | + name: qwen3-nocred |
| 80 | + namespace: nim-workload |
| 81 | +spec: |
| 82 | + # Required by the schema. Holds only HF_TOKEN — no NGC_API_KEY. |
| 83 | + authSecret: hf-secret |
| 84 | + image: |
| 85 | + repository: nvcr.io/nim/meta/llama-3.1-8b-instruct |
| 86 | + tag: "2.0.10" |
| 87 | + pullPolicy: IfNotPresent |
| 88 | + # No pullSecrets: this repository serves anonymous registry tokens. |
| 89 | + env: |
| 90 | + - name: NIM_MODEL_NAME |
| 91 | + value: hf://Qwen/Qwen3-0.6B |
| 92 | + - name: NIM_SERVED_MODEL_NAME |
| 93 | + value: Qwen/Qwen3-0.6B |
| 94 | + storage: |
| 95 | + emptyDir: |
| 96 | + sizeLimit: 60Gi |
| 97 | + replicas: 1 |
| 98 | + resources: |
| 99 | + limits: |
| 100 | + nvidia.com/gpu: 1 |
| 101 | + nodeSelector: |
| 102 | + # Replace gpuworker1 with your own GPU pool's agentpool label. |
| 103 | + agentpool: gpuworker1 |
| 104 | + tolerations: |
| 105 | + - key: nvidia.com/gpu |
| 106 | + operator: Equal |
| 107 | + value: present |
| 108 | + effect: NoSchedule |
| 109 | + expose: |
| 110 | + service: |
| 111 | + type: ClusterIP |
| 112 | + port: 8000 |
0 commit comments