Merge remote-tracking branch 'upstream/main' into feature/jsc-92-ef-adr

Author: jtschelling

.github/dependabot.yml

@@ -41,6 +41,7 @@ updates:
       - "/health-monitors/slurm-drain-monitor"
       - "/health-monitors/syslog-health-monitor"
       - "/janitor"
+      - "/lifecycle-manager"
       - "/janitor-provider"
       - "/labeler"
       - "/metadata-collector"

.github/workflows/cleanup-untagged-images.yml

@@ -79,6 +79,7 @@ jobs:
           - nvsentinel/log-collector
           - nvsentinel/file-server-cleanup
           - nvsentinel/janitor
+          - nvsentinel/lifecycle-manager
           - nvsentinel/health-events-analyzer
           - nvsentinel/maintenance-notifier
           - nvsentinel/event-exporter

.github/workflows/container-build-test.yml

@@ -145,6 +145,8 @@ jobs:
             path: .
           - module: preflight
             path: .
+          - module: lifecycle-manager
+            path: .
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
 

.github/workflows/e2e-test.yml

@@ -107,7 +107,7 @@ jobs:
         uses: ./.github/actions/setup-ci-env
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c  # v4.2.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
 
       - name: Prep system for multi-node Kind cluster
         run: |

.github/workflows/fern-docs-preview-build.yml

@@ -69,10 +69,27 @@ jobs:
             if git show-ref --verify --quiet "refs/tags/${version}"; then
               mkdir -p "fern/versions/${version}-content"
               git archive "refs/tags/${version}" -- docs/ | tar -x --strip-components=1 -C "fern/versions/${version}-content"
-              find "fern/versions/${version}-content" -name '*.md' -print0 | xargs -0 -r sed -i \
-                -e 's/{/\\{/g' \
-                -e 's/}/\\}/g' \
-                -e 's/</\&lt;/g'
+              # Escape {, }, < for MDX — but only outside fenced code blocks and inline code spans
+              find "fern/versions/${version}-content" -name '*.md' -print0 | while IFS= read -r -d '' f; do
+                awk '
+                  /^````*/ || /^~~~~*/ { fence = !fence; print; next }
+                  fence { print; next }
+                  {
+                    n = split($0, p, "`")
+                    out = ""
+                    for (i = 1; i <= n; i++) {
+                      if (i % 2 == 1) {
+                        gsub(/{/, "\\{", p[i])
+                        gsub(/}/, "\\}", p[i])
+                        gsub(/</, "\\&lt;", p[i])
+                      }
+                      out = out p[i]
+                      if (i < n) out = out "`"
+                    }
+                    print out
+                  }
+                ' "$f" > "${f}.tmp" && mv "${f}.tmp" "$f"
+              done
               echo "Extracted docs from $version"
             else
               echo "::warning::Tag $version not found — skipping content checkout"

.github/workflows/lint-test.yml

@@ -179,6 +179,7 @@ jobs:
           - node-drainer
           - fault-remediation
           - janitor
+          - lifecycle-manager
           - preflight
           - tests
     steps:

.github/workflows/publish-fern-docs.yml

@@ -164,10 +164,27 @@ jobs:
             if git show-ref --verify --quiet "refs/tags/${version}"; then
               mkdir -p "fern/versions/${version}-content"
               git archive "refs/tags/${version}" -- docs/ | tar -x --strip-components=1 -C "fern/versions/${version}-content"
-              find "fern/versions/${version}-content" -name '*.md' -print0 | xargs -0 -r sed -i \
-                -e 's/{/\\{/g' \
-                -e 's/}/\\}/g' \
-                -e 's/</\&lt;/g'
+              # Escape {, }, < for MDX — but only outside fenced code blocks and inline code spans
+              find "fern/versions/${version}-content" -name '*.md' -print0 | while IFS= read -r -d '' f; do
+                awk '
+                  /^````*/ || /^~~~~*/ { fence = !fence; print; next }
+                  fence { print; next }
+                  {
+                    n = split($0, p, "`")
+                    out = ""
+                    for (i = 1; i <= n; i++) {
+                      if (i % 2 == 1) {
+                        gsub(/{/, "\\{", p[i])
+                        gsub(/}/, "\\}", p[i])
+                        gsub(/</, "\\&lt;", p[i])
+                      }
+                      out = out p[i]
+                      if (i < n) out = out "`"
+                    }
+                    print out
+                  }
+                ' "$f" > "${f}.tmp" && mv "${f}.tmp" "$f"
+              done
               echo "Extracted docs from $version"
             else
               echo "::error::Tag $version not found — cannot pin frozen docs content"

.gitignore

@@ -450,3 +450,7 @@ tests/scale-tests/results/*.csv
 tests/scale-tests/cmd/fqm-scale-test/results/
 preflight-checks/dcgm-diag/dcgm-diag
 preflight/preflight
+
+# Build output of tilt/simple-health-client (an 18MB binary that sits next to its
+# own source, so a bare `git add tilt/` would stage it).
+/tilt/simple-health-client/simple-health-client

.ko.yaml

@@ -288,3 +288,20 @@ builds:
       org.opencontainers.image.version: "{{.Env.VERSION}}"
       org.opencontainers.image.revision: "{{.Env.GIT_COMMIT}}"
       org.opencontainers.image.created: "{{.Env.BUILD_DATE}}"
+
+  - id: lifecycle-manager
+    dir: lifecycle-manager
+    main: .
+    ldflags:
+      - "-s -w"
+      - "-X main.version={{.Env.VERSION}} -X main.commit={{.Env.GIT_COMMIT}} -X main.date={{.Env.BUILD_DATE}}"
+    annotations:
+      org.opencontainers.image.description: "Lifecycle manager controller for NVSentinel"
+    labels:
+      org.opencontainers.image.source: "https://github.com/nvidia/nvsentinel"
+      org.opencontainers.image.licenses: "Apache-2.0"
+      org.opencontainers.image.title: "NVSentinel Lifecycle Manager"
+      org.opencontainers.image.description: "Lifecycle manager for GPU node validation and maintenance in NVSentinel"
+      org.opencontainers.image.version: "{{.Env.VERSION}}"
+      org.opencontainers.image.revision: "{{.Env.GIT_COMMIT}}"
+      org.opencontainers.image.created: "{{.Env.BUILD_DATE}}"

Makefile

@@ -51,6 +51,7 @@ GO_MODULES := \
 	node-drainer \
 	fault-remediation \
 	janitor \
+	lifecycle-manager \
 	metadata-collector \
 	event-exporter \
 	store-client \
@@ -79,7 +80,8 @@ PRIVATE_MODULES := \
 # Modules requiring kubebuilder for tests
 KUBEBUILDER_MODULES := \
 	node-drainer \
-	fault-remediation
+	fault-remediation \
+	lifecycle-manager
 
 # Default target
 .PHONY: all
@@ -455,6 +457,11 @@ lint-test-janitor:
 	@echo "Linting and testing janitor (using standardized Makefile)..."
 	$(MAKE) -C janitor lint-test
 
+.PHONY: lint-test-lifecycle-manager
+lint-test-lifecycle-manager:
+	@echo "Linting and testing lifecycle-manager (using standardized Makefile)..."
+	$(MAKE) -C lifecycle-manager lint-test
+
 .PHONY: lint-test-store-client
 lint-test-store-client:
 	@echo "Linting and testing store-client..."
@@ -485,6 +492,12 @@ kubernetes-distro-lint:
 	$(MAKE) -C distros/kubernetes lint
 
 # Helm chart validation
+# Subcharts that cannot render standalone: they reference umbrella helpers or
+# globals that only exist when rendered as part of the parent chart.
+UMBRELLA_ONLY_CHARTS := csp-health-monitor event-exporter fault-quarantine \
+                       fault-remediation health-events-analyzer mongodb-store \
+                       node-drainer
+
 .PHONY: helm-lint
 helm-lint:
 	@echo "🎯 Validating Helm charts..."
@@ -501,17 +514,38 @@ helm-lint:
 	@echo ""
 	@# Individual component charts
 	@echo "Validating component charts..."
+	@# Recipes run under /bin/sh, so this uses [ rather than [[.
+	@# UMBRELLA_ONLY_CHARTS reference umbrella helpers (for example
+	@# nvsentinel.mongodb.certVolume) or umbrella globals, so they cannot render
+	@# standalone. They are skipped by name and reported, rather than being
+	@# silently included in a success message they did not earn. They are still
+	@# covered by the umbrella lint above and by helm-test.
 	@for chart_dir in distros/kubernetes/nvsentinel/charts/*/; do \
-		if [[ -f "$$chart_dir/Chart.yaml" ]]; then \
+		if [ -f "$$chart_dir/Chart.yaml" ]; then \
 			chart_name=$$(basename "$$chart_dir"); \
+			case " $(UMBRELLA_ONLY_CHARTS) " in \
+				*" $$chart_name "*) \
+					echo "Skipping $$chart_name (needs umbrella context)"; \
+					continue;; \
+			esac; \
 			echo "Validating chart: $$chart_name"; \
 			helm lint "$$chart_dir" -f distros/kubernetes/nvsentinel/values.yaml || exit 1; \
 			echo "Testing template rendering for: $$chart_name"; \
 			helm template "$$chart_name" "$$chart_dir" -f distros/kubernetes/nvsentinel/values.yaml >/dev/null || exit 1; \
 			echo ""; \
 		fi; \
 	done
-	@echo "✅ All Helm charts validated successfully"
+	@# The charts in UMBRELLA_ONLY_CHARTS ship disabled, so neither the parent
+	@# lint above (which renders defaults) nor the loop above exercises them.
+	@# Render once with every dependency condition turned on so they are
+	@# validated with the parent chart context they need. The conditions are
+	@# read from Chart.yaml so a newly added chart is picked up automatically.
+	@echo "Validating disabled-by-default charts with full context..."
+	helm template nvsentinel distros/kubernetes/nvsentinel \
+		$$(grep -oE 'condition: *[a-zA-Z0-9_.]+' distros/kubernetes/nvsentinel/Chart.yaml \
+			| awk '{print "--set "$$2"=true"}' | tr '\n' ' ') >/dev/null
+	@echo ""
+	@echo "✅ Parent chart, standalone-renderable component charts, and all charts with every dependency enabled validated"
 
 # Helm chart unit tests
 .PHONY: helm-test
@@ -521,8 +555,17 @@ helm-test:
 		echo "❌ Error: helm-unittest plugin not found. Install with: helm plugin install https://github.com/helm-unittest/helm-unittest"; \
 		exit 1; \
 	fi
+	@# The umbrella's own suite covers the cross-chart consistency checks, which
+	@# no subchart can see. --with-subchart=false because the subchart suites
+	@# address their templates by chart-relative path and are run separately
+	@# below.
+	@if [ -d "distros/kubernetes/nvsentinel/tests" ]; then \
+		echo "Running unit tests for chart: nvsentinel (umbrella)"; \
+		helm unittest --with-subchart=false distros/kubernetes/nvsentinel || exit 1; \
+		echo ""; \
+	fi
 	@for chart_dir in distros/kubernetes/nvsentinel/charts/*/; do \
-		if [[ -d "$$chart_dir/tests" ]]; then \
+		if [ -d "$$chart_dir/tests" ]; then \
 			chart_name=$$(basename "$$chart_dir"); \
 			echo "Running unit tests for chart: $$chart_name"; \
 			helm unittest "$$chart_dir" || exit 1; \