feat(validator): AKS H100 NCCL all-reduce performance validation (NVIDIA#1695)

Signed-off-by: Yuan Chen <yuanchen97@gmail.com>

Author: yuanchen8911

docs/integrator/aks-gpu-setup.md

@@ -173,12 +173,24 @@ time. See
 `Standard_ND96isr_H100_v5` is the 8-GPU ND H100 v5 SKU. The AKS Dynamo
 inference throughput gate (`inference-throughput`) is a fixed absolute
 **full-node** floor calibrated on an 8-GPU H100 node, so this SKU is the
-supported happy path for that gate. Smaller NCads H100 SKUs
-(`Standard_NC80adis_H100_v5` = 2 GPUs, `Standard_NC40ads_H100_v5` = 1 GPU) run
-fine for deployment but will false-fail the throughput floor; gate on
-`inference-ttft-p99` only on those until the per-GPU normalization in
+supported happy path for that gate. The same applies to the AKS H100 training
+NCCL gate (`nccl-all-reduce-bw >= 150`): its floor is calibrated on full
+ND96isr nodes using the Network Operator's RDMA shared device pool
+(`rdma/hca_shared_devices_a`) over the SKU's multi-HCA InfiniBand fabric.
+Smaller NCads H100 SKUs (`Standard_NC80adis_H100_v5` = 2 GPUs,
+`Standard_NC40ads_H100_v5` = 1 GPU) run fine for deployment but will
+false-fail both full-node floors — they lack the GPU count and IB fabric the
+calibrations assume; gate on `inference-ttft-p99` only on those until the
+per-GPU normalization in
 [#1254](https://github.com/NVIDIA/aicr/issues/1254) lands.
 
+The NCCL gate's benchmark pods pull the `nccl-tests` image from
+`public.ecr.aws` (AWS's public registry) — a cross-cloud pull when running on
+Azure. Private or egress-restricted AKS clusters must allow registry egress to
+`public.ecr.aws` or mirror the image into an Azure-reachable registry (e.g.
+ACR) before running the performance phase; otherwise the benchmark workers
+fail at image pull and the check fails without measuring anything.
+
 ### Alternative: Use the AKS Driver-Only Profile
 
 If you prefer AKS to install the driver (e.g., for driver version pinning by

docs/user/validation.md

@@ -47,7 +47,7 @@ ones) that match the target fabric:
 
 | Check | Transport | When it's selected |
 |---|---|---|
-| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | H100/H200 on EKS, H100 on GKE, and B200/GB200 on self-managed clusters (`service=any`). Preserves the pre-variant behavior. |
+| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | H100/H200 on EKS, H100 on GKE, H100 on AKS (ND-series InfiniBand — NCCL's built-in IB/verbs transport over the `rdma/hca_shared_devices_a` shared device pool), and B200/GB200 on self-managed clusters (`service=any`). Preserves the pre-variant behavior. |
 | `nccl-all-reduce-bw-net` | NET (EFA on EKS by default; ConnectX RoCE via `AICR_NCCL_FABRIC=roce`) | GB200 + EKS. Asserts EFA actually carried traffic — catches silent fallback to Socket when the NVIDIA driver is missing `NVreg_GrdmaPciTopoCheckOverride=1`. |
 | `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS, and GB200 + OKE. Asserts the NVLS communicator actually initialized — catches silent fallback to EFA (EKS) or Socket (OKE) when the IMEX domain is misconfigured. |
 

examples/recipes/aks-training.yaml

@@ -168,6 +168,14 @@ validation:
       - expected-resources
       - gpu-operator-version
       - check-nvidia-smi
+  performance:
+    checks:
+      - nccl-all-reduce-bw
+    constraints:
+      # Provisional floor pending calibration on an ND96isr_H100_v5 testbed —
+      # see recipes/overlays/h100-aks-training.yaml.
+      - name: nccl-all-reduce-bw
+        value: '>= 150'
   conformance:
     checks:
       - platform-health

recipes/overlays/a100-aks-training.yaml

@@ -63,12 +63,13 @@ spec:
   # here.
   #
   # Performance gating is intentionally omitted until an empirical A100-on-Azure
-  # NCCL baseline is established. AKS has no validator NCCL runtime template or
-  # scheduling path (it is not in the support table), so even the H100 AKS
-  # training overlay leaves the performance phase undeclared rather than gate on
-  # an unmeasured floor (see issue #1337). A100 on Azure is likewise unmeasured
-  # and would need its own fabric-aware baseline (#1256), so carrying any floor
-  # here would only false-fail healthy runs.
+  # NCCL baseline is established. The validator's AKS NCCL runtime path covers
+  # H100 only (supportedNCCLCombinations in
+  # validators/performance/nccl_all_reduce_bw_constraint.go; h100-aks-training
+  # declares nccl-all-reduce-bw with a floor calibrated on ND96isr_H100_v5).
+  # A100 on Azure is unmeasured and needs its own fabric-aware baseline (#1256)
+  # plus an aks/a100 support-table entry (#1337), so carrying any floor here
+  # would only false-fail healthy runs.
   validation:
     conformance:
       checks:

recipes/overlays/h100-aks-training.yaml

@@ -66,13 +66,23 @@ spec:
       constraints:
         - name: Deployment.gpu-operator.version
           value: ">= v24.6.0"
-    # No performance phase: the nccl-all-reduce-bw check is intentionally not
-    # declared here. The validator has no AKS NCCL runtime template or
-    # scheduling path (AKS is not in the support table), so a declared check
-    # would silently skip and report the phase as passed without measuring
-    # anything — false assurance. Wire it (testbed-validated AKS runtime
-    # template + scheduling path + support-table entry + real threshold) once
-    # an H100 AKS testbed is characterized. See issue #1337.
+    performance:
+      checks:
+        - nccl-all-reduce-bw
+      # PROVISIONAL floor, set at half the calibrated EKS H100 value (>= 300,
+      # p5.48xlarge: 8x H100 SXM, 32x EFA). Live baseline: 157.24 GB/s busbw
+      # measured UNTUNED (16 GiB all-reduce, NCCL built-in IB/verbs with
+      # NCCL_NET_PLUGIN=none over the network-operator shared HCA pool) on
+      # aicr-test1, 2x Standard_ND96isr_H100_v5 (8x H100 SXM, 8x 400Gb NDR
+      # InfiniBand per node), 2026-07-09. There is substantial tuning headroom
+      # — 8x 400Gb NDR line rate is ~400 GB/s — so recalibrate this floor
+      # upward once the IB path is tuned rather than treating 157 as the
+      # fabric's ceiling. See issue #1337; the general fabric-class-aware
+      # floor work is tracked in
+      # https://github.com/NVIDIA/aicr/issues/1256.
+      constraints:
+        - name: nccl-all-reduce-bw
+          value: ">= 150"
     conformance:
       checks:
         - platform-health

validators/performance/nccl_aks_utils.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"log/slog"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// aksRdmaSharedResource is the extended resource published by the network
+// operator's rdma-shared-device-plugin on AICR AKS clusters (selector
+// vendors=15b3 drivers=mlx5_core). The resource name hca_shared_devices_a is
+// pinned by recipes/components/network-operator/manifests/
+// nic-cluster-policy-aks.yaml for workload compatibility — keep the two in
+// sync. It is a *shared* device pool: a pod that requests one unit is granted
+// access to every InfiniBand HCA on the node (/dev/infiniband/*), so NCCL
+// workers request exactly 1 regardless of the HCA count.
+const aksRdmaSharedResource = "rdma/hca_shared_devices_a"
+
+// discoverAKSRdmaCount reads the allocatable rdma-shared-device-plugin
+// resource count from a GPU node. 0 is valid (network-operator RDMA shared
+// device plugin not installed) — NCCL falls back to TCP over the pod network,
+// mirroring the EKS zero-EFA behavior. On ND-series InfiniBand SKUs
+// (e.g. Standard_ND96isr_H100_v5) the plugin advertises a large shared pool
+// (allocatable "1k" = 1000).
+func discoverAKSRdmaCount(node v1.Node) int {
+	rdmaQuantity := node.Status.Allocatable[v1.ResourceName(aksRdmaSharedResource)]
+	return int(rdmaQuantity.Value())
+}
+
+// applyAKSTemplateData populates the AKS-specific runtime template variables:
+// the ${RDMA_RESOURCE_LIMITS}/${RDMA_RESOURCE_REQUESTS} worker resource lines
+// discovered from the first target node, with the same TCP fallback the EKS
+// zero-EFA path uses (reduced max message size) when the RDMA shared device
+// plugin is absent.
+func applyAKSTemplateData(config *gpuConfiguration, templateData map[string]string) {
+	warnIfHeterogeneousNodes(config.Nodes)
+	rdmaCount := discoverAKSRdmaCount(config.Nodes[0])
+	// Indentation matches the resource block position in runtime.yaml.
+	const rdmaIndent = "                      "
+	templateData["RDMA_RESOURCE_LIMITS"] = buildAKSRdmaResourceLine(rdmaCount, rdmaIndent)
+	templateData["RDMA_RESOURCE_REQUESTS"] = buildAKSRdmaResourceLine(rdmaCount, rdmaIndent)
+	if rdmaCount == 0 {
+		templateData["MAX_MESSAGE_SIZE"] = maxMessageSizeTCP
+		slog.Warn("No shared RDMA devices found — NCCL will use TCP (reduced bandwidth)",
+			"resource", aksRdmaSharedResource, "maxMessageSize", maxMessageSizeTCP)
+	} else {
+		slog.Info("Discovered AKS shared RDMA device pool", "resource", aksRdmaSharedResource, "allocatable", rdmaCount)
+	}
+}
+
+// buildAKSRdmaResourceLine returns the YAML line requesting one unit of the
+// shared RDMA resource at the correct indentation, or an empty string when
+// the node advertises none (TCP fallback — the placeholder line is dropped
+// from the rendered runtime). Requesting "1" (not the pool size) is the
+// rdma-shared-device-plugin contract: any positive request mounts every
+// shared IB device into the container.
+func buildAKSRdmaResourceLine(rdmaCount int, indent string) string {
+	if rdmaCount == 0 {
+		return ""
+	}
+	return fmt.Sprintf("%s%s: \"1\"", indent, aksRdmaSharedResource)
+}

validators/performance/nccl_aks_utils_test.go

@@ -0,0 +1,166 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestDiscoverAKSRdmaCount(t *testing.T) {
+	tests := []struct {
+		name string
+		node v1.Node
+		want int
+	}{
+		{
+			// Standard_ND96isr_H100_v5 with the network-operator
+			// rdma-shared-device-plugin: allocatable is advertised as "1k"
+			// (the shared pool size), which resource.Quantity parses as 1000.
+			name: "ND96isr_H100_v5 with shared RDMA pool",
+			node: v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"node.kubernetes.io/instance-type": "Standard_ND96isr_H100_v5",
+					},
+				},
+				Status: v1.NodeStatus{
+					Allocatable: v1.ResourceList{
+						v1.ResourceName("nvidia.com/gpu"):      resource.MustParse("8"),
+						v1.ResourceName(aksRdmaSharedResource): resource.MustParse("1k"),
+					},
+				},
+			},
+			want: 1000,
+		},
+		{
+			name: "no RDMA shared device plugin (falls back to TCP)",
+			node: v1.Node{
+				Status: v1.NodeStatus{
+					Allocatable: v1.ResourceList{
+						v1.ResourceName("nvidia.com/gpu"): resource.MustParse("8"),
+					},
+				},
+			},
+			want: 0,
+		},
+		{
+			name: "no allocatable at all",
+			node: v1.Node{},
+			want: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := discoverAKSRdmaCount(tt.node); got != tt.want {
+				t.Errorf("discoverAKSRdmaCount() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestApplyAKSTemplateData(t *testing.T) {
+	const rdmaLine = `                      rdma/hca_shared_devices_a: "1"`
+	tests := []struct {
+		name           string
+		allocatable    v1.ResourceList
+		wantLine       string
+		wantMaxMsgSize string
+	}{
+		{
+			name: "shared RDMA pool present keeps IB message size",
+			allocatable: v1.ResourceList{
+				v1.ResourceName("nvidia.com/gpu"):      resource.MustParse("8"),
+				v1.ResourceName(aksRdmaSharedResource): resource.MustParse("1k"),
+			},
+			wantLine:       rdmaLine,
+			wantMaxMsgSize: maxMessageSize,
+		},
+		{
+			name: "no RDMA pool falls back to TCP with reduced message size",
+			allocatable: v1.ResourceList{
+				v1.ResourceName("nvidia.com/gpu"): resource.MustParse("8"),
+			},
+			wantLine:       "",
+			wantMaxMsgSize: maxMessageSizeTCP,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := &gpuConfiguration{
+				WorkerCount:     2,
+				GPUCountPerNode: 8,
+				TotalGPUCount:   16,
+				Nodes: []v1.Node{
+					{Status: v1.NodeStatus{Allocatable: tt.allocatable}},
+					{Status: v1.NodeStatus{Allocatable: tt.allocatable}},
+				},
+			}
+			templateData := map[string]string{"MAX_MESSAGE_SIZE": maxMessageSize}
+			applyAKSTemplateData(config, templateData)
+			if got := templateData["RDMA_RESOURCE_LIMITS"]; got != tt.wantLine {
+				t.Errorf("RDMA_RESOURCE_LIMITS = %q, want %q", got, tt.wantLine)
+			}
+			if got := templateData["RDMA_RESOURCE_REQUESTS"]; got != tt.wantLine {
+				t.Errorf("RDMA_RESOURCE_REQUESTS = %q, want %q", got, tt.wantLine)
+			}
+			if got := templateData["MAX_MESSAGE_SIZE"]; got != tt.wantMaxMsgSize {
+				t.Errorf("MAX_MESSAGE_SIZE = %q, want %q", got, tt.wantMaxMsgSize)
+			}
+		})
+	}
+}
+
+func TestBuildAKSRdmaResourceLine(t *testing.T) {
+	tests := []struct {
+		name      string
+		rdmaCount int
+		indent    string
+		want      string
+	}{
+		{
+			// A worker always requests exactly 1 unit, never the pool size:
+			// the rdma-shared-device-plugin grants access to every shared IB
+			// device per unit requested.
+			name:      "shared pool of 1000 requests a single unit",
+			rdmaCount: 1000,
+			indent:    "                      ",
+			want:      `                      rdma/hca_shared_devices_a: "1"`,
+		},
+		{
+			name:      "single device still requests one unit",
+			rdmaCount: 1,
+			indent:    "                      ",
+			want:      `                      rdma/hca_shared_devices_a: "1"`,
+		},
+		{
+			name:      "no RDMA — empty string",
+			rdmaCount: 0,
+			indent:    "                      ",
+			want:      "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := buildAKSRdmaResourceLine(tt.rdmaCount, tt.indent)
+			if got != tt.want {
+				t.Errorf("buildAKSRdmaResourceLine(%d) = %q, want %q", tt.rdmaCount, got, tt.want)
+			}
+		})
+	}
+}