demos: drop daemon-mode cuda check in favor of preflight-checks/cuda-validation

The demo re-bundled CUDA context/memory validation as a polling daemon
check — the exact model deesharma24 rejected on #891, where a long-running
daemon contends for GPU memory with active workloads. That review is why
CUDA validation re-landed as a preflight init-container (#1384), which runs
once before workloads schedule.

Remove the daemon check from this demo to avoid contradicting the accepted
design: delete checks/cuda_validation.py and drop all references from
monitor.py, config.py, metrics.py, the k8s ConfigMap/PrometheusRule, and the
tests. README now points at preflight-checks/cuda-validation/ (#1384) as the
supported form.

Also note in the DaemonSet that the system-services-monitor:0.1.0 tag is a
demo-local pin, independent of the Helm chart appVersion.

Author: dmvevents

demos/system-services-monitor/README.md

@@ -17,7 +17,8 @@ NVIDIA Fabric Manager can fail and stay broken for weeks undetected. NVSentinel'
 | 1 | **Fabric Manager Service** | FM not running, flapping, error state | `nsenter` + `systemctl` |
 | 2 | **Critical GPU Services** | persistenced dead | `nsenter` + `systemctl` |
 | 3 | **Per-GPU Fabric State** | FM_NOT_STARTED, FM_REGISTRATION_STUCK, FM_FABRIC_ERROR | `nsenter` + `nvidia-smi` |
-| 4 | **CUDA Validation** | Context failures, memory errors | PyTorch subprocess |
+
+> **CUDA validation is not part of this monitor.** Polling CUDA context/memory tests from a long-running daemon contends for GPU memory with active workloads (see the [#891 review](https://github.com/NVIDIA/NVSentinel/pull/891)). The supported form is a preflight init-container that runs once before workloads schedule — see [`preflight-checks/cuda-validation/`](../../preflight-checks/cuda-validation/) (#1384).
 
 ## Quick Start
 
@@ -60,7 +61,6 @@ The ServiceMonitor includes PrometheusRule with alerts:
 - `FabricManagerFlapping` (warning, 5m)
 - `FabricStateUnhealthy` (critical, 5m) -- per-GPU fabric orchestration failure
 - `GPUServiceDown` (critical, 3m)
-- `CUDAValidationFailed` (critical, 5m)
 
 ## Validated On
 

demos/system-services-monitor/checks/cuda_validation.py

@@ -27,10 +27,6 @@ class MonitorConfig:
 
     # Check toggles
     enable_fabric_check: bool = True
-    enable_cuda_validation: bool = False  # off by default (resource intensive)
-
-    # CUDA validation runs at a slower cadence
-    cuda_validation_interval: int = 600  # seconds
 
     # Services to monitor (besides fabric manager).
     # nv-hostengine is monitored by gpu-health-monitor via
@@ -55,8 +51,6 @@ def _bool(val: str) -> bool:
             flap_window=int(os.environ.get("FLAP_WINDOW", "600")),
             flap_threshold=int(os.environ.get("FLAP_THRESHOLD", "3")),
             enable_fabric_check=_bool(os.environ.get("ENABLE_FABRIC_CHECK", "true")),
-            enable_cuda_validation=_bool(os.environ.get("ENABLE_CUDA_VALIDATION", "false")),
-            cuda_validation_interval=int(os.environ.get("CUDA_VALIDATION_INTERVAL", "600")),
         )
 
         services_env = os.environ.get("GPU_SERVICES")

demos/system-services-monitor/config.py

@@ -14,5 +14,4 @@ data:
   FLAP_WINDOW: "600"
   FLAP_THRESHOLD: "3"
   ENABLE_FABRIC_CHECK: "true"
-  ENABLE_CUDA_VALIDATION: "false"
   LOG_LEVEL: "INFO"

demos/system-services-monitor/k8s/configmap.yaml

@@ -32,6 +32,7 @@ spec:
         - name: monitor
           # Pin to a specific semver tag (or @sha256:<digest> in production).
           # Demo image; replace with your registry path when deploying.
+          # 0.1.0 is a demo-local pin, independent of the Helm chart appVersion.
           image: system-services-monitor:0.1.0
           imagePullPolicy: IfNotPresent
           envFrom:

demos/system-services-monitor/k8s/daemonset.yaml

@@ -94,16 +94,3 @@ spec:
             description: >-
               NVIDIA service {{ $labels.service_name }} has been down for more
               than 3 minutes on {{ $labels.node }}.
-
-        # Critical: CUDA validation failed
-        - alert: CUDAValidationFailed
-          expr: cuda_validation_passed == 0
-          for: 5m
-          labels:
-            severity: critical
-            component: system-services-monitor
-          annotations:
-            summary: "CUDA validation failed on {{ $labels.node }}"
-            description: >-
-              CUDA context creation or memory test failed on {{ $labels.node }}.
-              One or more GPUs may be in a bad state requiring driver reload or reboot.

demos/system-services-monitor/k8s/servicemonitor.yaml

@@ -38,13 +38,6 @@
     ["node", "service_name"],
 )
 
-# --- CUDA validation ---
-cuda_validation_passed = Gauge(
-    "cuda_validation_passed",
-    "CUDA validation result (1=passed, 0=failed)",
-    ["node"],
-)
-
 # --- Check infrastructure ---
 health_check_duration_seconds = Histogram(
     "health_check_duration_seconds",

demos/system-services-monitor/metrics.py

@@ -24,12 +24,10 @@
     fabric_manager_restarts_total,
     fabric_manager_last_healthy_seconds,
     nvidia_service_up,
-    cuda_validation_passed,
     health_check_duration_seconds,
     health_check_errors_total,
 )
 from checks.service_check import ServiceChecker
-from checks.cuda_validation import CUDAValidator
 
 logger = logging.getLogger(__name__)
 
@@ -41,14 +39,12 @@ def __init__(self, config: MonitorConfig):
         self.config = config
         self._shutdown = Event()
         self._start_time = time.monotonic()
-        self._last_cuda_check = 0.0
 
         # Initialize checkers
         self._service_checker = ServiceChecker(
             flap_window=config.flap_window,
             flap_threshold=config.flap_threshold,
         )
-        self._cuda_validator = CUDAValidator()
 
         # Track state for cross-check correlation
         self._fabric_manager_down = False
@@ -129,22 +125,9 @@ def run_check_cycle(self):
                     logger.exception("Service check failed")
                     health_check_errors_total.labels("services").inc()
 
-        # --- Check 3: CUDA validation (slower cadence) ---
-        if self.config.enable_cuda_validation:
-            now = time.monotonic()
-            if (now - self._last_cuda_check) >= self.config.cuda_validation_interval:
-                self._last_cuda_check = now
-                with health_check_duration_seconds.labels("cuda").time():
-                    try:
-                        cuda_result = self._cuda_validator.check()
-                        cuda_validation_passed.labels(node).set(1 if cuda_result.passed else 0)
-                        if not cuda_result.passed:
-                            logger.error("CUDA validation FAILED on %s: %s",
-                                         node, cuda_result.errors or cuda_result.error)
-                            overall_healthy = False
-                    except Exception:
-                        logger.exception("CUDA validation failed")
-                        health_check_errors_total.labels("cuda").inc()
+        # GPU context/memory validation is intentionally not polled from this
+        # daemon (it would contend for GPU memory with running workloads). It
+        # runs as a preflight init-container instead — see the demo README.
 
         # --- Overall health ---
         if self._in_grace_period():

demos/system-services-monitor/monitor.py

@@ -20,7 +20,6 @@ def test_defaults(self):
         assert config.metrics_port == 9101
         assert config.boot_grace_period == 300
         assert config.enable_fabric_check is True
-        assert config.enable_cuda_validation is False
         assert "nvidia-fabricmanager" in config.gpu_services
         # nv-hostengine is covered by gpu-health-monitor / DCGM and must
         # not appear in the default demo service list.
@@ -36,8 +35,6 @@ def test_from_env(self):
             "FLAP_WINDOW": "300",
             "FLAP_THRESHOLD": "5",
             "ENABLE_FABRIC_CHECK": "true",
-            "ENABLE_CUDA_VALIDATION": "true",
-            "CUDA_VALIDATION_INTERVAL": "1200",
         }
         with patch.dict(os.environ, env, clear=False):
             config = MonitorConfig.from_env()
@@ -48,8 +45,7 @@ def test_from_env(self):
         assert config.node_name == "test-node"
         assert config.boot_grace_period == 120
         assert config.flap_threshold == 5
-        assert config.enable_cuda_validation is True
-        assert config.cuda_validation_interval == 1200
+        assert config.enable_fabric_check is True
 
     def test_custom_gpu_services(self):
         with patch.dict(os.environ, {"GPU_SERVICES": "svc-a, svc-b , svc-c"}, clear=False):
@@ -58,14 +54,14 @@ def test_custom_gpu_services(self):
 
     def test_bool_parsing(self):
         for truthy in ("true", "True", "TRUE", "1", "yes", "Yes"):
-            with patch.dict(os.environ, {"ENABLE_CUDA_VALIDATION": truthy}, clear=False):
+            with patch.dict(os.environ, {"ENABLE_FABRIC_CHECK": truthy}, clear=False):
                 config = MonitorConfig.from_env()
-                assert config.enable_cuda_validation is True
+                assert config.enable_fabric_check is True
 
         for falsy in ("false", "False", "0", "no", ""):
-            with patch.dict(os.environ, {"ENABLE_CUDA_VALIDATION": falsy}, clear=False):
+            with patch.dict(os.environ, {"ENABLE_FABRIC_CHECK": falsy}, clear=False):
                 config = MonitorConfig.from_env()
-                assert config.enable_cuda_validation is False
+                assert config.enable_fabric_check is False
 
 
 class TestSystemServicesMonitor:
@@ -78,7 +74,6 @@ def _make_monitor(self, **overrides):
             "node_name": "test-node",
             "boot_grace_period": 0,  # no grace period in tests
             "enable_fabric_check": True,
-            "enable_cuda_validation": False,
         }
         defaults.update(overrides)
         config = MonitorConfig(**defaults)