feat(monitor): system-services-monitor implementation + unit tests

.github/workflows/container-build-test.yml

@@ -60,6 +60,8 @@ jobs:
             make_command: 'make -C health-monitors/gpu-health-monitor docker-build-dcgm4'
           - component: syslog-health-monitor
             make_command: 'make -C health-monitors/syslog-health-monitor docker-build'
+          - component: system-services-monitor
+            make_command: 'make -C health-monitors/system-services-monitor docker-build'
           # Log Collection (Docker-based)
           - component: log-collector
             make_command: 'make -C log-collector docker-build-log-collector'

health-monitors/system-services-monitor/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG VERSION="0.1.0"
+
+FROM public.ecr.aws/docker/library/python:3.13-bookworm AS build
+
+ARG VERSION
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install poetry==2.3.3 poetry-plugin-export==1.10.0
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+COPY health-monitors/system-services-monitor/ .
+# Set package version from build arg (strip 'v' prefix for PEP 440 compliance)
+RUN poetry version $(echo "${VERSION}" | sed 's/^v//')
+RUN --mount=type=cache,target=/tmp/poetry_cache \
+    poetry build --format wheel
+RUN poetry export --format requirements.txt --output constraints.txt --without-hashes
+
+
+FROM public.ecr.aws/docker/library/python:3.13-slim-bookworm AS runtime
+
+# util-linux provides nsenter for inspecting host systemd services
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        util-linux
+
+ENV PYTHONUNBUFFERED=1
+
+COPY --from=build /app/dist/*.whl ./
+COPY --from=build /app/constraints.txt ./
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install ./system_services_monitor*.whl --constraint constraints.txt
+
+ENTRYPOINT ["system_services_monitor"]

health-monitors/system-services-monitor/Makefile

@@ -0,0 +1,119 @@
+# System Services Monitor Makefile
+# Individual module build and test targets (Python module)
+
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =============================================================================
+# MODULE-SPECIFIC CONFIGURATION
+# =============================================================================
+
+# Python module configuration
+IS_GO_MODULE := 0
+HAS_DOCKER := 1
+
+# Python package name (with underscores, not hyphens)
+PYTHON_PACKAGE_NAME := system_services_monitor
+# Coverage in make/python.mk uses --source=$(MODULE_NAME); common.mk defaults
+# MODULE_NAME to the (hyphenated) directory basename, which does not match the
+# importable package, yielding 0% coverage. Pin it to the package name.
+MODULE_NAME := $(PYTHON_PACKAGE_NAME)
+
+# system-services-monitor specific settings (Python module)
+CLEAN_EXTRA_FILES := system_services_monitor.egg-info
+DOCKER_EXTRA_ARGS :=
+
+# =============================================================================
+# INCLUDE SHARED DEFINITIONS
+# =============================================================================
+
+include ../../make/common.mk
+include ../../make/python.mk
+include ../../make/docker.mk
+
+# =============================================================================
+# DEFAULT TARGET
+# =============================================================================
+
+.PHONY: all
+all: lint-test
+
+# =============================================================================
+# PYTHON-SPECIFIC DEVELOPMENT TARGETS
+# =============================================================================
+
+.PHONY: setup shell deps deps-show format
+setup:
+	@echo "Setting up Poetry environment for $(MODULE_NAME)..."
+	poetry config virtualenvs.in-project true
+	poetry install
+
+shell:
+	@echo "Opening Poetry shell for $(MODULE_NAME)..."
+	poetry shell
+
+deps:
+	@echo "Updating dependencies for $(MODULE_NAME)..."
+	poetry update
+
+deps-show:
+	@echo "Showing dependencies for $(MODULE_NAME)..."
+	poetry show
+
+format:
+	@echo "Running Black formatter on $(MODULE_NAME)..."
+	poetry run black .
+
+# =============================================================================
+# MODULE HELP
+# =============================================================================
+
+.PHONY: help
+help:
+	@echo "System Services Monitor Makefile - Using nvsentinel make/*.mk standards (Python)"
+	@echo ""
+	@echo "This is a Python module that monitors non-DCGM service health on GPU nodes."
+	@echo ""
+	@echo "Configuration (environment variables):"
+	@echo "  MODULE_NAME=$(MODULE_NAME)"
+	@echo "  REPO_ROOT=$(REPO_ROOT)"
+	@echo "  CONTAINER_REGISTRY=$(CONTAINER_REGISTRY)"
+	@echo "  CONTAINER_ORG=$(CONTAINER_ORG)"
+	@echo "  SAFE_REF_NAME=$(SAFE_REF_NAME)"
+	@echo "  PLATFORMS=$(PLATFORMS)"
+	@echo "  HAS_DOCKER=$(HAS_DOCKER)"
+	@echo ""
+	@echo "Main targets:"
+	@echo "  all        - Run lint-test (standardized default)"
+	@echo "  lint-test  - Run full lint and test suite (Python with Poetry)"
+	@echo ""
+	@echo "Individual targets:"
+	@echo "  setup      - Set up Poetry environment"
+	@echo "  lint       - Run Black formatter check"
+	@echo "  format     - Run Black formatter"
+	@echo "  test       - Run tests with coverage"
+	@echo "  coverage   - Generate coverage reports"
+	@echo "  build      - Build Python package"
+	@echo ""
+	@echo "Docker targets:"
+	@echo "  docker-build   - Build Docker image (local)"
+	@echo "  docker-publish - Build and publish Docker image (CI/production)"
+	@echo "  setup-buildx   - Setup Docker buildx builder"
+	@echo ""
+	@echo "Development targets:"
+	@echo "  shell      - Open Poetry shell"
+	@echo "  deps       - Update dependencies"
+	@echo "  deps-show  - Show dependencies"
+	@echo "  clean      - Clean build artifacts and reports"
+	@echo "  help       - Show this help message"

health-monitors/system-services-monitor/README.md

@@ -0,0 +1,21 @@
+# System Services Monitor
+
+Health monitor for non-DCGM infrastructure failures on NVIDIA GPU nodes: Fabric Manager service health, per-GPU fabric state, and GPU service lifecycle.
+
+Documentation has been moved to the repo-wide `docs/` tree per project convention:
+
+- **Component overview, scope, architecture, check categories** --
+  [`docs/system-services-monitor.md`](../../docs/system-services-monitor.md)
+- **CLI flags, environment variables, deployment recipes** --
+  [`docs/configuration/system-services-monitor.md`](../../docs/configuration/system-services-monitor.md)
+- **Architectural decision (split from `gpu-health-monitor`)** --
+  [`docs/designs/049-system-services-monitor-scope.md`](../../docs/designs/049-system-services-monitor-scope.md)
+- **Pod-level orphan detection for gpu-operator DaemonSets** --
+  [`docs/monitoring-critical-operators.md`](../../docs/monitoring-critical-operators.md) (owned by `kubernetes-object-monitor`)
+
+## Build & Test
+
+```sh
+make -C health-monitors/system-services-monitor lint-test
+make -C health-monitors/system-services-monitor docker-build
+```

health-monitors/system-services-monitor/pyproject.toml

@@ -0,0 +1,69 @@
+[tool.poetry]
+name = "system-services-monitor"
+version = "0.1.0"
+description = "GPU node health monitor for Fabric Manager and infrastructure failures"
+authors = ["Community Contributors"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+click = "^8.3.1"
+grpcio = "^1.78.0"
+prometheus-client = "^0.24.1"
+protobuf = ">=6.31.1,<7.0.0"
+googleapis-common-protos = ">=1.56.0"
+structlog = "^25.1.0"
+
+[tool.poetry.group.dev.dependencies]
+black = "^26.1.0"
+coverage = "^7.13.4"
+grpcio-tools = "^1.78.0"
+pytest = "^9.0.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+system_services_monitor = "system_services_monitor.cli:cli"
+
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+exclude = '''
+
+(
+  /(
+      \.eggs         # exclude a few common directories in the
+    | \.git          # root of the project
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+  )/
+  | system_services_monitor/protos
+
+)
+'''
+
+[tool.coverage.report]
+exclude_also = [
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+omit = [
+    "tests/*",
+    "system_services_monitor/protos/*"
+]

health-monitors/system-services-monitor/system_services_monitor/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

health-monitors/system-services-monitor/system_services_monitor/checkers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.