Skip to content

Commit ced14ea

Browse files
lalitadithyalalitadithya
committed
chore: restore incorrectly deleted files
1 parent e60eace commit ced14ea

2 files changed

Lines changed: 191 additions & 0 deletions

File tree

‎distros/kubernetes/nvsentinel/values-preflight-kube.yaml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# --------------------------------------------------------------------------
16+
# Preflight, for clusters using Kubernetes' native gang scheduling.
17+
#
18+
# Preflight is different from monitoring and remediation: it's an active
19+
# check that runs as an init container in the workload pod's own spec,
20+
# right before the pod starts, to confirm the node is actually ready to
21+
# take that workload.
22+
#
23+
# Multi-node checks (verifying GPU-to-GPU bandwidth across the whole job,
24+
# not just within one node) need to know which pods belong together. This
25+
# file uses Kubernetes' own gang scheduling APIs to find out, so it works
26+
# with no extra scheduler, PROVIDED your cluster has them turned on:
27+
# - Kubernetes 1.36: scheduling.k8s.io/v1alpha2 PodGroup
28+
# - Kubernetes 1.35: scheduling.k8s.io/v1alpha1 Workload, used as a
29+
# fallback if PodGroup isn't available
30+
#
31+
# Both are alpha and disabled by default. You need the GenericWorkload
32+
# feature gate on the API server and scheduler (to get the API itself),
33+
# and the scheduler's GangScheduling gate (to get all-or-nothing gang
34+
# scheduling behavior; folded into GenericWorkload from Kubernetes 1.37).
35+
# A cluster admin has to enable these; they aren't a Helm value, and some
36+
# managed Kubernetes offerings won't let you turn on alpha gates at all.
37+
# Check what your cluster actually exposes before relying on this:
38+
# kubectl api-resources --api-group=scheduling.k8s.io
39+
#
40+
# On a different scheduler (KAI Scheduler, Volcano, etc.), use that
41+
# scheduler's values file instead of this one. See the gang
42+
# discovery guide for others:
43+
# https://docs.nvidia.com/nvsentinel/configuration/preflight/#gang-discovery
44+
#
45+
# Usage, layered on top of whatever stage you're already running:
46+
#
47+
# helm upgrade --install nvsentinel oci://ghcr.io/nvidia/nvsentinel \
48+
# --version "$NVSENTINEL_VERSION" \
49+
# --namespace nvsentinel --create-namespace \
50+
# -f distros/kubernetes/nvsentinel/values-remediation.yaml \
51+
# -f distros/kubernetes/nvsentinel/values-preflight-kube.yaml
52+
#
53+
# Then label each namespace that should run preflight checks:
54+
#
55+
# kubectl label namespace <your-namespace> nvsentinel.nvidia.com/preflight=enabled
56+
# --------------------------------------------------------------------------
57+
58+
global:
59+
preflight:
60+
enabled: true
61+
62+
# preflight:
63+
# gangDiscovery:
64+
# name: "osmo-with-kai"
65+
# labelKeys:
66+
# - "osmo.group_uuid"
67+
# podGroupGVR:
68+
# group: "scheduling.run.ai"
69+
# version: "v2alpha2"
70+
# resource: "podgroups"
71+
# minCountExpr: "podGroup.spec.minMember"

‎distros/kubernetes/nvsentinel/values-remediation.yaml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# --------------------------------------------------------------------------
16+
# Monitoring + remediation.
17+
#
18+
# Layer this on top of the default install once you trust what NVSentinel
19+
# is reporting. It turns on the rest of the pipeline (quarantine/cordon,
20+
# drain, and remediation) and deploys the in-cluster MongoDB event store
21+
# those modules read from. When a node reports a fatal GPU error, NVSentinel
22+
# will cordon it, evict its workloads, and then repair it automatically:
23+
#
24+
# - Recoverable GPU faults -> in-place GPU reset (nvidia-smi), seconds
25+
# - Everything else -> node reboot (chroot /host reboot), minutes
26+
#
27+
# Both actions run as privileged Kubernetes Jobs on the node itself, so this
28+
# works out of the box on any cluster (bare-metal, on-prem, or any cloud)
29+
# with no cloud credentials required. If you'd rather have NVSentinel call
30+
# your cloud provider's reboot/terminate API instead of running these Jobs
31+
# locally, see "Using CSP APIs instead" below.
32+
#
33+
# helm upgrade --install nvsentinel oci://ghcr.io/nvidia/nvsentinel \
34+
# --version "$NVSENTINEL_VERSION" \
35+
# --namespace nvsentinel --create-namespace \
36+
# -f values-remediation.yaml
37+
#
38+
# `helm upgrade` (without --reuse-values) only applies the chart's own
39+
# defaults plus whatever -f/--set flags you pass on that command, so this
40+
# file is self-contained: it doesn't depend on flags used in a previous
41+
# install.
42+
#
43+
# Review docs/fault-quarantine.md and docs/node-drainer.md before enabling
44+
# this in a production cluster. The default ruleset cordons any node that
45+
# reports a fatal GPU error.
46+
# --------------------------------------------------------------------------
47+
48+
global:
49+
faultQuarantine:
50+
enabled: true
51+
nodeDrainer:
52+
enabled: true
53+
faultRemediation:
54+
enabled: true
55+
janitor:
56+
enabled: true
57+
janitorProvider:
58+
enabled: true
59+
60+
# Deploys an in-cluster MongoDB (3-node replica set) event store.
61+
# fault-quarantine, node-drainer, and fault-remediation all watch it for
62+
# events, so it's required from this stage on.
63+
mongodbStore:
64+
enabled: true
65+
66+
janitor-provider:
67+
auth:
68+
# The bundled janitor is derived from the release namespace, so nothing is
69+
# needed here. List only an out-of-tree janitor, if you run one.
70+
allowedUsernames: []
71+
csp:
72+
# "generic" performs remediation locally on the node (a privileged Job
73+
# running `chroot /host reboot`, equivalent to an operator running
74+
# `sudo reboot` on the box) instead of calling a cloud provider API.
75+
provider: "generic"
76+
77+
fault-remediation:
78+
maintenance:
79+
actions:
80+
# Recoverable GPU faults: reset the GPU in place instead of rebooting
81+
# the whole node. Keeps the rest of the node's GPUs online.
82+
COMPONENT_RESET:
83+
apiGroup: "janitor.dgxc.nvidia.com"
84+
version: "v1alpha1"
85+
kind: "GPUReset"
86+
scope: "Cluster"
87+
completeConditionType: "Complete"
88+
templateFileName: "gpureset-template.yaml"
89+
equivalenceGroup: "reset"
90+
impactedEntityScope: "GPU_UUID"
91+
supersedingEquivalenceGroups: ["restart"]
92+
templates:
93+
"gpureset-template.yaml": |
94+
apiVersion: {{ .ApiGroup }}/{{ .Version }}
95+
kind: GPUReset
96+
metadata:
97+
name: maintenance-{{ .HealthEvent.NodeName }}-{{ .HealthEventID }}
98+
spec:
99+
nodeName: {{ .HealthEvent.NodeName }}
100+
selector:
101+
uuids:
102+
- {{ .ImpactedEntityScopeValue }}
103+
104+
# --------------------------------------------------------------------------
105+
# Using CSP APIs instead
106+
#
107+
# To have the janitor call your cloud provider's API instead of running the
108+
# local reboot Job, replace the janitor-provider.csp block above. See:
109+
# - docs/configuration/janitor-provider.md: per-provider Helm reference
110+
# - docs/csp-health-monitor-iam.md: IAM / Workload Identity setup
111+
#
112+
# Example for AWS:
113+
# janitor-provider:
114+
# csp:
115+
# provider: "aws"
116+
# aws:
117+
# region: "us-west-2"
118+
# accountId: "123456789012"
119+
# iamRoleName: "nvsentinel-janitor-provider"
120+
# --------------------------------------------------------------------------

0 commit comments

Comments
 (0)