Skip to content

Commit a8402b1

Browse files
dmveventsdmvevents
committed
feat(monitor): system-services-monitor implementation + unit tests
Implements the system-services-monitor package per ADR-030 (#1380): - service_check.py for systemd service health - watcher.py + event_processor.py with thread-safe entity_cache - cli.py with --verbose flag and version handling - logger.py with explicit warning on unknown log levels - Dockerfile (python:3.13 base, apt cache mount per CR review) - Makefile with real lint-test / test / docker-build / docker-publish targets - Unit tests under tests/ covering service_check + event_processor Lands as #891 split (3 of 5) on top of #1380 (ADR) and the CI PR. Excludes cuda_validation.py — that checker is being moved to preflight-checks/cuda-validation/ in a follow-up PR per @XRFXLP review on the umbrella PR. The runtime GPU-allocation concern raised in his review is resolved by removing it from the daemon-poll path entirely.
1 parent 94a3d83 commit a8402b1

22 files changed

Lines changed: 2021 additions & 24 deletions

‎health-monitors/system-services-monitor/Dockerfile‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ARG VERSION="0.1.0"
16+
17+
FROM public.ecr.aws/docker/library/python:3.13-bookworm AS build
18+
19+
ARG VERSION
20+
21+
RUN --mount=type=cache,target=/root/.cache/pip \
22+
pip install poetry==1.8.2
23+
24+
ENV POETRY_NO_INTERACTION=1 \
25+
POETRY_VIRTUALENVS_IN_PROJECT=1 \
26+
POETRY_VIRTUALENVS_CREATE=1 \
27+
POETRY_CACHE_DIR=/tmp/poetry_cache
28+
29+
WORKDIR /app
30+
31+
COPY health-monitors/system-services-monitor/ .
32+
# Set package version from build arg (strip 'v' prefix for PEP 440 compliance)
33+
RUN poetry version $(echo "${VERSION}" | sed 's/^v//')
34+
RUN --mount=type=cache,target=/tmp/poetry_cache \
35+
poetry build --format wheel
36+
RUN poetry export --format requirements.txt --output constraints.txt --without-hashes
37+
38+
39+
FROM public.ecr.aws/docker/library/python:3.13-slim-bookworm AS runtime
40+
41+
# util-linux provides nsenter for inspecting host systemd services
42+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
43+
--mount=type=cache,target=/var/lib/apt,sharing=locked \
44+
echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries && \
45+
apt-get update && \
46+
apt-get install -y --no-install-recommends \
47+
util-linux
48+
49+
ENV PYTHONUNBUFFERED=1
50+
51+
COPY --from=build /app/dist/*.whl ./
52+
COPY --from=build /app/constraints.txt ./
53+
RUN --mount=type=cache,target=/root/.cache/pip \
54+
pip install ./system_services_monitor*.whl --constraint constraints.txt
55+
56+
ENTRYPOINT ["system_services_monitor"]

‎health-monitors/system-services-monitor/Makefile‎

Lines changed: 89 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
# System Services Monitor Makefile (STUB — real impl lands in PR #3)
1+
# System Services Monitor Makefile
2+
# Individual module build and test targets (Python module)
3+
24
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
35
#
46
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,37 +16,100 @@
1416
# limitations under the License.
1517

1618
# =============================================================================
17-
# STUB MAKEFILE
19+
# MODULE-SPECIFIC CONFIGURATION
1820
# =============================================================================
19-
# This is the CI-wiring half of the system-services-monitor split (#891 of 5).
20-
# It registers the matrix entry in container-build-test.yml so that the
21-
# follow-up implementation PR's first CI run lands on the real matrix row
22-
# rather than being backfilled afterwards.
23-
#
24-
# The real Makefile (with make/python.mk + make/docker.mk includes, Poetry
25-
# setup, lint/test/docker-publish targets, etc.) lands together with the
26-
# Python implementation in PR #3.
2721

28-
.PHONY: all lint-test docker-build docker-publish help
22+
# Python module configuration
23+
IS_GO_MODULE := 0
24+
HAS_DOCKER := 1
25+
26+
# Python package name (with underscores, not hyphens)
27+
PYTHON_PACKAGE_NAME := system_services_monitor
28+
29+
# system-services-monitor specific settings (Python module)
30+
CLEAN_EXTRA_FILES := system_services_monitor.egg-info
31+
DOCKER_EXTRA_ARGS :=
32+
33+
# =============================================================================
34+
# INCLUDE SHARED DEFINITIONS
35+
# =============================================================================
36+
37+
include ../../make/common.mk
38+
include ../../make/python.mk
39+
include ../../make/docker.mk
40+
41+
# =============================================================================
42+
# DEFAULT TARGET
43+
# =============================================================================
2944

45+
.PHONY: all
3046
all: lint-test
3147

32-
lint-test:
33-
@echo "system-services-monitor: lint-test stub — real impl in PR #3 of #891 split"
48+
# =============================================================================
49+
# PYTHON-SPECIFIC DEVELOPMENT TARGETS
50+
# =============================================================================
51+
52+
.PHONY: setup shell deps deps-show format
53+
setup:
54+
@echo "Setting up Poetry environment for $(MODULE_NAME)..."
55+
poetry config virtualenvs.in-project true
56+
poetry install
57+
58+
shell:
59+
@echo "Opening Poetry shell for $(MODULE_NAME)..."
60+
poetry shell
3461

35-
docker-build:
36-
@echo "system-services-monitor: docker-build stub — real impl in PR #3 of #891 split"
62+
deps:
63+
@echo "Updating dependencies for $(MODULE_NAME)..."
64+
poetry update
3765

38-
docker-publish:
39-
@echo "system-services-monitor: docker-publish stub — real impl in PR #3 of #891 split"
66+
deps-show:
67+
@echo "Showing dependencies for $(MODULE_NAME)..."
68+
poetry show
4069

70+
format:
71+
@echo "Running Black formatter on $(MODULE_NAME)..."
72+
poetry run black .
73+
74+
# =============================================================================
75+
# MODULE HELP
76+
# =============================================================================
77+
78+
.PHONY: help
4179
help:
42-
@echo "system-services-monitor — STUB Makefile"
80+
@echo "System Services Monitor Makefile - Using nvsentinel make/*.mk standards (Python)"
81+
@echo ""
82+
@echo "This is a Python module that monitors non-DCGM service health on GPU nodes."
83+
@echo ""
84+
@echo "Configuration (environment variables):"
85+
@echo " MODULE_NAME=$(MODULE_NAME)"
86+
@echo " REPO_ROOT=$(REPO_ROOT)"
87+
@echo " CONTAINER_REGISTRY=$(CONTAINER_REGISTRY)"
88+
@echo " CONTAINER_ORG=$(CONTAINER_ORG)"
89+
@echo " SAFE_REF_NAME=$(SAFE_REF_NAME)"
90+
@echo " PLATFORMS=$(PLATFORMS)"
91+
@echo " HAS_DOCKER=$(HAS_DOCKER)"
92+
@echo ""
93+
@echo "Main targets:"
94+
@echo " all - Run lint-test (standardized default)"
95+
@echo " lint-test - Run full lint and test suite (Python with Poetry)"
96+
@echo ""
97+
@echo "Individual targets:"
98+
@echo " setup - Set up Poetry environment"
99+
@echo " lint - Run Black formatter check"
100+
@echo " format - Run Black formatter"
101+
@echo " test - Run tests with coverage"
102+
@echo " coverage - Generate coverage reports"
103+
@echo " build - Build Python package"
43104
@echo ""
44-
@echo "This Makefile contains stub targets only. The real targets land"
45-
@echo "together with the Python implementation in PR #3 of the #891 split."
105+
@echo "Docker targets:"
106+
@echo " docker-build - Build Docker image (local)"
107+
@echo " docker-publish - Build and publish Docker image (CI/production)"
108+
@echo " setup-buildx - Setup Docker buildx builder"
46109
@echo ""
47-
@echo "Available stub targets:"
48-
@echo " lint-test - returns 0 (stub)"
49-
@echo " docker-build - returns 0 (stub)"
50-
@echo " docker-publish - returns 0 (stub)"
110+
@echo "Development targets:"
111+
@echo " shell - Open Poetry shell"
112+
@echo " deps - Update dependencies"
113+
@echo " deps-show - Show dependencies"
114+
@echo " clean - Clean build artifacts and reports"
115+
@echo " help - Show this help message"

‎health-monitors/system-services-monitor/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# System Services Monitor
2+
3+
Health monitor for non-DCGM infrastructure failures on NVIDIA GPU nodes: Fabric Manager service health, per-GPU fabric state, and GPU service lifecycle.
4+
5+
Documentation has been moved to the repo-wide `docs/` tree per project convention:
6+
7+
- **Component overview, scope, architecture, check categories** --
8+
[`docs/system-services-monitor.md`](../../docs/system-services-monitor.md)
9+
- **CLI flags, environment variables, deployment recipes** --
10+
[`docs/configuration/system-services-monitor.md`](../../docs/configuration/system-services-monitor.md)
11+
- **Architectural decision (split from `gpu-health-monitor`)** --
12+
[`docs/designs/030-fabric-manager-monitor-scope.md`](../../docs/designs/030-fabric-manager-monitor-scope.md)
13+
- **Pod-level orphan detection for gpu-operator DaemonSets** --
14+
[`docs/monitoring-critical-operators.md`](../../docs/monitoring-critical-operators.md) (owned by `kubernetes-object-monitor`)
15+
16+
## Build & Test
17+
18+
```sh
19+
make -C health-monitors/system-services-monitor lint-test
20+
make -C health-monitors/system-services-monitor docker-build
21+
```

‎health-monitors/system-services-monitor/pyproject.toml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
[tool.poetry]
2+
name = "system-services-monitor"
3+
version = "0.1.0"
4+
description = "GPU node health monitor for Fabric Manager and infrastructure failures"
5+
authors = ["Community Contributors"]
6+
readme = "README.md"
7+
8+
[tool.poetry.dependencies]
9+
python = "^3.13"
10+
click = "^8.3.1"
11+
grpcio = "^1.78.0"
12+
prometheus-client = "^0.24.1"
13+
protobuf = ">=6.31.1,<7.0.0"
14+
googleapis-common-protos = ">=1.56.0"
15+
structlog = "^25.1.0"
16+
17+
[tool.poetry.group.dev.dependencies]
18+
black = "^26.1.0"
19+
coverage = "^7.13.4"
20+
grpcio-tools = "^1.78.0"
21+
pytest = "^9.0.2"
22+
23+
[build-system]
24+
requires = ["poetry-core"]
25+
build-backend = "poetry.core.masonry.api"
26+
27+
[tool.poetry.scripts]
28+
system_services_monitor = "system_services_monitor.cli:cli"
29+
30+
[tool.black]
31+
line-length = 120
32+
include = '\.pyi?$'
33+
exclude = '''
34+
35+
(
36+
/(
37+
\.eggs # exclude a few common directories in the
38+
| \.git # root of the project
39+
| \.hg
40+
| \.mypy_cache
41+
| \.tox
42+
| \.venv
43+
| _build
44+
| buck-out
45+
| build
46+
| dist
47+
)/
48+
| system_services_monitor/protos
49+
50+
)
51+
'''
52+
53+
[tool.coverage.report]
54+
exclude_also = [
55+
"def __repr__",
56+
"if self.debug:",
57+
"if settings.DEBUG",
58+
"raise AssertionError",
59+
"raise NotImplementedError",
60+
"if 0:",
61+
"if __name__ == .__main__.:",
62+
"if TYPE_CHECKING:",
63+
"class .*\\bProtocol\\):",
64+
"@(abc\\.)?abstractmethod",
65+
]
66+
omit = [
67+
"tests/*",
68+
"system_services_monitor/protos/*"
69+
]

‎health-monitors/system-services-monitor/system_services_monitor/__init__.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.

‎health-monitors/system-services-monitor/system_services_monitor/checkers/__init__.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.

0 commit comments

Comments
 (0)