Skip to content

Commit 42672ae

Browse files
feat(auth): authenticate health event publishers and bind events to the reporting node (#1636)
1 parent d45cc2c commit 42672ae

111 files changed

Lines changed: 7792 additions & 219 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,3 +450,7 @@ tests/scale-tests/results/*.csv
450450
tests/scale-tests/cmd/fqm-scale-test/results/
451451
preflight-checks/dcgm-diag/dcgm-diag
452452
preflight/preflight
453+
454+
# Build output of tilt/simple-health-client (an 18MB binary that sits next to its
455+
# own source, so a bare `git add tilt/` would stage it).
456+
/tilt/simple-health-client/simple-health-client

Makefile

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,12 @@ kubernetes-distro-lint:
492492
$(MAKE) -C distros/kubernetes lint
493493

494494
# Helm chart validation
495+
# Subcharts that cannot render standalone: they reference umbrella helpers or
496+
# globals that only exist when rendered as part of the parent chart.
497+
UMBRELLA_ONLY_CHARTS := csp-health-monitor event-exporter fault-quarantine \
498+
fault-remediation health-events-analyzer mongodb-store \
499+
node-drainer
500+
495501
.PHONY: helm-lint
496502
helm-lint:
497503
@echo "🎯 Validating Helm charts..."
@@ -508,17 +514,38 @@ helm-lint:
508514
@echo ""
509515
@# Individual component charts
510516
@echo "Validating component charts..."
517+
@# Recipes run under /bin/sh, so this uses [ rather than [[.
518+
@# UMBRELLA_ONLY_CHARTS reference umbrella helpers (for example
519+
@# nvsentinel.mongodb.certVolume) or umbrella globals, so they cannot render
520+
@# standalone. They are skipped by name and reported, rather than being
521+
@# silently included in a success message they did not earn. They are still
522+
@# covered by the umbrella lint above and by helm-test.
511523
@for chart_dir in distros/kubernetes/nvsentinel/charts/*/; do \
512-
if [[ -f "$$chart_dir/Chart.yaml" ]]; then \
524+
if [ -f "$$chart_dir/Chart.yaml" ]; then \
513525
chart_name=$$(basename "$$chart_dir"); \
526+
case " $(UMBRELLA_ONLY_CHARTS) " in \
527+
*" $$chart_name "*) \
528+
echo "Skipping $$chart_name (needs umbrella context)"; \
529+
continue;; \
530+
esac; \
514531
echo "Validating chart: $$chart_name"; \
515532
helm lint "$$chart_dir" -f distros/kubernetes/nvsentinel/values.yaml || exit 1; \
516533
echo "Testing template rendering for: $$chart_name"; \
517534
helm template "$$chart_name" "$$chart_dir" -f distros/kubernetes/nvsentinel/values.yaml >/dev/null || exit 1; \
518535
echo ""; \
519536
fi; \
520537
done
521-
@echo "✅ All Helm charts validated successfully"
538+
@# The charts in UMBRELLA_ONLY_CHARTS ship disabled, so neither the parent
539+
@# lint above (which renders defaults) nor the loop above exercises them.
540+
@# Render once with every dependency condition turned on so they are
541+
@# validated with the parent chart context they need. The conditions are
542+
@# read from Chart.yaml so a newly added chart is picked up automatically.
543+
@echo "Validating disabled-by-default charts with full context..."
544+
helm template nvsentinel distros/kubernetes/nvsentinel \
545+
$$(grep -oE 'condition: *[a-zA-Z0-9_.]+' distros/kubernetes/nvsentinel/Chart.yaml \
546+
| awk '{print "--set "$$2"=true"}' | tr '\n' ' ') >/dev/null
547+
@echo ""
548+
@echo "✅ Parent chart, standalone-renderable component charts, and all charts with every dependency enabled validated"
522549

523550
# Helm chart unit tests
524551
.PHONY: helm-test
@@ -528,8 +555,17 @@ helm-test:
528555
echo "❌ Error: helm-unittest plugin not found. Install with: helm plugin install https://github.com/helm-unittest/helm-unittest"; \
529556
exit 1; \
530557
fi
558+
@# The umbrella's own suite covers the cross-chart consistency checks, which
559+
@# no subchart can see. --with-subchart=false because the subchart suites
560+
@# address their templates by chart-relative path and are run separately
561+
@# below.
562+
@if [ -d "distros/kubernetes/nvsentinel/tests" ]; then \
563+
echo "Running unit tests for chart: nvsentinel (umbrella)"; \
564+
helm unittest --with-subchart=false distros/kubernetes/nvsentinel || exit 1; \
565+
echo ""; \
566+
fi
531567
@for chart_dir in distros/kubernetes/nvsentinel/charts/*/; do \
532-
if [[ -d "$$chart_dir/tests" ]]; then \
568+
if [ -d "$$chart_dir/tests" ]; then \
533569
chart_name=$$(basename "$$chart_dir"); \
534570
echo "Running unit tests for chart: $$chart_name"; \
535571
helm unittest "$$chart_dir" || exit 1; \

commons/go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ toolchain go1.26.3
66

77
require (
88
github.com/BurntSushi/toml v1.6.0
9+
github.com/hashicorp/golang-lru/v2 v2.0.7
910
github.com/nvidia/nvsentinel/data-models v0.0.0
1011
github.com/nvidia/nvsentinel/store-client v0.0.0
1112
github.com/prometheus/client_golang v1.24.1
@@ -16,6 +17,7 @@ require (
1617
go.opentelemetry.io/otel/sdk v1.45.0
1718
go.opentelemetry.io/otel/trace v1.45.0
1819
golang.org/x/sync v0.22.0
20+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea
1921
google.golang.org/grpc v1.83.0
2022
google.golang.org/protobuf v1.36.12
2123
gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -77,7 +79,6 @@ require (
7779
golang.org/x/text v0.40.0 // indirect
7880
golang.org/x/time v0.15.0 // indirect
7981
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
80-
google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect
8182
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
8283
gopkg.in/inf.v0 v0.9.1 // indirect
8384
gopkg.in/yaml.v3 v3.0.1 // indirect

commons/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
7272
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
7373
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
7474
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
75+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
76+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
7577
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
7678
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
7779
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=

commons/pkg/grpcauth/cache.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package grpcauth
16+
17+
import (
18+
"crypto/sha256"
19+
"fmt"
20+
"time"
21+
22+
lru "github.com/hashicorp/golang-lru/v2"
23+
)
24+
25+
const (
26+
// cacheMaxEntries bounds the cache. Each entry is one live publisher's
27+
// token, so this is far above the number of pods that can address a single
28+
// node's platform-connector; the bound exists so a caller presenting many
29+
// distinct tokens cannot grow the map without limit.
30+
cacheMaxEntries = 4096
31+
32+
// cacheTTL is how long a positive verdict is remembered, for every caller.
33+
//
34+
// It is a fixed value rather than the token's own expiry. TokenReview is the
35+
// only check that notices the bound pod being deleted, so the TTL is the
36+
// window in which a deleted pod's token still works. Honouring the token's
37+
// exp would make that window the token lifetime — an hour at the chart's
38+
// default — for every ordinary publisher. A flat two minutes bounds it for
39+
// everyone, costs about one TokenReview per token per two minutes, and
40+
// needs no assumption about what kind of credential TokenReview accepted.
41+
// It matches the API server's own webhook-authenticator cache default.
42+
cacheTTL = 2 * time.Minute
43+
)
44+
45+
// verdictCache remembers successful authentication verdicts keyed by the token
46+
// that produced them, so a steady stream of requests from the same caller
47+
// costs one TokenReview per cacheTTL instead of one per request.
48+
//
49+
// Only positive verdicts are cached. A rejected token is re-reviewed every
50+
// time: rejections are rare in a healthy system, and remembering them would
51+
// add a second expiry policy for no measurable saving.
52+
type verdictCache struct {
53+
entries *lru.Cache[[sha256.Size]byte, cacheEntry]
54+
}
55+
56+
type cacheEntry struct {
57+
identity Identity
58+
expires time.Time
59+
}
60+
61+
func newVerdictCache() (*verdictCache, error) {
62+
entries, err := lru.New[[sha256.Size]byte, cacheEntry](cacheMaxEntries)
63+
if err != nil {
64+
return nil, fmt.Errorf("failed to build verdict cache: %w", err)
65+
}
66+
67+
return &verdictCache{entries: entries}, nil
68+
}
69+
70+
func (c *verdictCache) get(token string, now time.Time) (*Identity, bool) {
71+
key := sha256.Sum256([]byte(token))
72+
73+
entry, ok := c.entries.Get(key)
74+
if !ok {
75+
return nil, false
76+
}
77+
78+
if now.After(entry.expires) {
79+
c.entries.Remove(key)
80+
81+
return nil, false
82+
}
83+
84+
// Identity is all value fields, so this copy fully isolates the caller from
85+
// the stored entry.
86+
identity := entry.identity
87+
88+
return &identity, true
89+
}
90+
91+
func (c *verdictCache) put(token string, identity *Identity, now time.Time) {
92+
c.entries.Add(sha256.Sum256([]byte(token)), cacheEntry{
93+
identity: *identity,
94+
expires: now.Add(cacheTTL),
95+
})
96+
}

0 commit comments

Comments
 (0)