fix: bind gpu-health-monitor metrics server for IPv6-only clusters

Add a --metrics-addr flag to gpu_health_monitor and pass it to
start_http_server so the Prometheus metrics endpoint can bind dual-stack.
The flag defaults to 0.0.0.0 (no change for existing invocations). Wire a
new global.metricsAddress Helm value (default "::") into the dcgm 3.x/4.x
daemonsets so a freshly installed chart binds dual-stack and kubelet
liveness/readiness probes reach the endpoint over IPv6.

Refs #1407

Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>

Author: mvanhorn

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-3.x.yaml

@@ -53,6 +53,8 @@ spec:
             - {{ include "gpu-health-monitor.dcgmCliMode" . | quote }}
             - --port
             - "{{ .Values.global.metricsPort }}"
+            - --metrics-addr
+            - "{{ .Values.global.metricsAddress | default "0.0.0.0" }}"
             - --verbose
             - {{ .Values.verbose | quote }}
             - --state-file

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/templates/daemonset-dcgm-4.x.yaml

@@ -53,6 +53,8 @@ spec:
             - {{ include "gpu-health-monitor.dcgmCliMode" . | quote }}
             - --port
             - "{{ .Values.global.metricsPort }}"
+            - --metrics-addr
+            - "{{ .Values.global.metricsAddress | default "0.0.0.0" }}"
             - --verbose
             - {{ .Values.verbose | quote }}
             - --state-file

distros/kubernetes/nvsentinel/values.yaml

@@ -21,6 +21,10 @@ global:
     tag: "12-debian-12-r30"
     pullPolicy: IfNotPresent
   metricsPort: 2112
+  # IPv4 bind address for the GPU health monitor metrics server. Set to "::"
+  # to enable IPv6 / dual-stack binding. This requires an IPv6-capable kernel;
+  # nodes booted with ipv6.disable=1 fail with EAFNOSUPPORT and CrashLoop.
+  metricsAddress: "0.0.0.0"
   # Default Kubernetes API client limits for core controllers. Components can
   # override these with their own qps and burst values.
   # qps: positive values throttle requests, 0 uses the client-go default,

health-monitors/gpu-health-monitor/gpu_health_monitor/cli.py

@@ -74,6 +74,14 @@ def _init_event_processor(
 )
 @click.option("--config-file", type=click.Path(), help="Path to config file", required=True)
 @click.option("--port", type=int, help="Port to use for metrics server", required=True)
+@click.option(
+    "--metrics-addr",
+    type=str,
+    default="0.0.0.0",
+    show_default=True,
+    help="Address the metrics server binds to. Use '::' for IPv6 / dual-stack clusters.",
+    required=False,
+)
 @click.option("--verbose", type=bool, default=False, help="Enable debug logging", required=False)
 @click.option("--state-file", type=click.Path(), help="gpu health monitor state file path", required=True)
 @click.option("--dcgm-k8s-service-enabled", type=bool, help="Is DCGM K8s service Enabled", required=True)
@@ -124,6 +132,7 @@ def cli(
     dcgm_error_mapping_config_file,
     config_file,
     port,
+    metrics_addr,
     verbose,
     state_file,
     dcgm_k8s_service_enabled,
@@ -261,7 +270,7 @@ def cli(
     # expose a documented fixed RPC timeout, so fleets should validate this
     # deadline in STORE_ONLY mode before enabling remediation. Set to 0 to disable.
     probe_deadline_seconds = dcgm_config.getfloat("ProbeDeadlineSeconds", fallback=poll_interval * 3)
-    prom_server, t = start_health_server(port, staleness_seconds=poll_interval * 3)
+    prom_server, t = start_health_server(port, staleness_seconds=poll_interval * 3, addr=metrics_addr)
 
     def process_exit_signal(signum, frame):
         exit.set()

health-monitors/gpu-health-monitor/gpu_health_monitor/healthz.py

@@ -20,6 +20,7 @@
 false positives from NTP clock adjustments.
 """
 
+import socket
 import threading
 import time
 from http.server import ThreadingHTTPServer
@@ -80,13 +81,17 @@ def do_GET(self) -> None:
             super().do_GET()
 
 
-def start_server(port: int, staleness_seconds: float = 300.0) -> tuple[ThreadingHTTPServer, threading.Thread]:
+def start_server(
+    port: int, staleness_seconds: float = 300.0, addr: str = "0.0.0.0"
+) -> tuple[ThreadingHTTPServer, threading.Thread]:
     """Start an HTTP server serving /metrics and /healthz.
 
     Args:
         port: TCP port to listen on.
         staleness_seconds: Maximum seconds between reconcile iterations
             before /healthz returns 503.
+        addr: Address to bind. Defaults to 0.0.0.0 (IPv4). Pass "::" to bind
+            IPv6, which also accepts IPv4-mapped clients on dual-stack nodes.
 
     Returns:
         (ThreadingHTTPServer, daemon Thread) — same interface as
@@ -98,7 +103,13 @@ def start_server(port: int, staleness_seconds: float = 300.0) -> tuple[Threading
     # Reset the grace period to start from server startup, not module import.
     _last_reconcile.mark_alive()
 
-    httpd = ThreadingHTTPServer(("", port), _HealthMetricsHandler)
+    server_cls = ThreadingHTTPServer
+    if ":" in addr:
+        # ThreadingHTTPServer defaults to AF_INET, which cannot bind an IPv6 literal.
+        server_cls = type(
+            "_ThreadingHTTPServerV6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6}
+        )
+    httpd = server_cls((addr, port), _HealthMetricsHandler)
     t = threading.Thread(target=httpd.serve_forever, daemon=True)
     t.start()
 

health-monitors/gpu-health-monitor/gpu_health_monitor/tests/test_cli.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the gpu-health-monitor CLI, focused on the metrics server binding."""
+
+from unittest.mock import MagicMock, patch
+
+from click.testing import CliRunner
+
+from gpu_health_monitor.cli import cli
+
+
+def _find_option(param_name):
+    for param in cli.params:
+        if param.name == param_name:
+            return param
+    return None
+
+
+def test_metrics_addr_option_defaults_to_ipv4():
+    """--metrics-addr exists and defaults to 0.0.0.0 (no behavior change by default)."""
+    option = _find_option("metrics_addr")
+    assert option is not None
+    assert option.default == "0.0.0.0"
+    assert option.required is False
+
+
+def _write_config(tmp_path):
+    config_file = tmp_path / "config.ini"
+    config_file.write_text(
+        "[logging]\n"
+        "[dcgm]\n"
+        "PollIntervalSeconds = 60\n"
+        "[cli]\n"
+        "EnabledEventProcessors = PlatformConnectorEventProcessor\n"
+        "[eventprocessors.platformconnector]\n"
+        "SocketPath = /tmp/does-not-matter.sock\n"
+    )
+    mapping_file = tmp_path / "dcgmerrors.csv"
+    mapping_file.write_text("0,DCGM_FR_UNKNOWN\n")
+    return config_file, mapping_file
+
+
+def _run_cli(tmp_path, extra_args):
+    config_file, mapping_file = _write_config(tmp_path)
+    args = [
+        "--dcgm-addr",
+        "localhost:5555",
+        "--dcgm-error-mapping-config-file",
+        str(mapping_file),
+        "--config-file",
+        str(config_file),
+        "--port",
+        "2112",
+        "--state-file",
+        str(tmp_path / "statefile"),
+        "--dcgm-k8s-service-enabled",
+        "false",
+        *extra_args,
+    ]
+    with patch("gpu_health_monitor.cli.start_health_server") as mock_start, patch(
+        "gpu_health_monitor.cli._init_event_processor"
+    ), patch("gpu_health_monitor.cli.dcgm.DCGMWatcher") as mock_watcher:
+        mock_start.return_value = (MagicMock(), MagicMock())
+        runner = CliRunner()
+        result = runner.invoke(cli, args, env={"NODE_NAME": "test-node"})
+    return result, mock_start, mock_watcher
+
+
+def test_health_server_binds_explicit_metrics_addr(tmp_path):
+    """--metrics-addr :: is passed through to the health server as addr='::'."""
+    result, mock_start, _ = _run_cli(tmp_path, ["--metrics-addr", "::"])
+    assert result.exit_code == 0, result.output
+    mock_start.assert_called_once()
+    assert mock_start.call_args.args[0] == 2112
+    assert mock_start.call_args.kwargs["addr"] == "::"
+
+
+def test_health_server_defaults_to_ipv4(tmp_path):
+    """Without --metrics-addr the server still binds 0.0.0.0 (backward compatible)."""
+    result, mock_start, _ = _run_cli(tmp_path, [])
+    assert result.exit_code == 0, result.output
+    mock_start.assert_called_once()
+    assert mock_start.call_args.args[0] == 2112
+    assert mock_start.call_args.kwargs["addr"] == "0.0.0.0"