feat(node-drainer): reduce node-drainer informer cache footprint by removing unused fields from node/pod (#1615)

Signed-off-by: Ajay Mishra <ajmishra@nvidia.com>

Author: XRFXLP

node-drainer/pkg/evaluator/evaluator_integration_test.go

@@ -108,7 +108,14 @@ func setupDirectTest(t *testing.T, userNamespaces []config.UserNamespace, dryRun
 		PartialDrainEnabled:       partialDrainEnabled,
 	}
 
-	informersInstance, err := informers.NewInformers(client, 1*time.Minute, ptr.To(2), false, dryRun)
+	informersInstance, err := informers.NewInformers(
+		client,
+		1*time.Minute,
+		ptr.To(2),
+		false,
+		dryRun,
+		tomlConfig.SystemNamespaces,
+	)
 	require.NoError(t, err)
 	go func() { _ = informersInstance.Run(ctx) }()
 	require.Eventually(t, informersInstance.HasSynced, 30*time.Second, 1*time.Second)

node-drainer/pkg/informers/informers.go

@@ -31,13 +31,15 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/utils/ptr"
 
 	"github.com/nvidia/nvsentinel/data-models/pkg/model"
 	"github.com/nvidia/nvsentinel/data-models/pkg/protos"
+	"github.com/nvidia/nvsentinel/fault-quarantine/pkg/common"
 	"github.com/nvidia/nvsentinel/node-drainer/pkg/metrics"
 )
 
@@ -59,15 +61,24 @@ type Informers struct {
 }
 
 func NewInformers(clientset kubernetes.Interface, resyncPeriod time.Duration,
-	notReadyTimeoutMinutes *int, drainGPUPods bool, dryRun bool) (*Informers, error) {
+	notReadyTimeoutMinutes *int, drainGPUPods bool, dryRun bool, systemNamespaces string) (*Informers, error) {
 	informerFactory := informers.NewSharedInformerFactoryWithOptions(
 		clientset,
 		resyncPeriod,
 	)
 
 	podInformer := informerFactory.Core().V1().Pods().Informer()
 
-	err := podInformer.GetIndexer().AddIndexers(
+	systemNamespacesRegex, err := compileExcludePattern(systemNamespaces)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compile system namespaces regex: %w", err)
+	}
+
+	if err := podInformer.SetTransform(excludedPodTransform(systemNamespacesRegex)); err != nil {
+		return nil, fmt.Errorf("failed to set pod informer transform: %w", err)
+	}
+
+	err = podInformer.GetIndexer().AddIndexers(
 		cache.Indexers{
 			NodeIndex:          NodeIndexFunc,
 			NamespaceNodeIndex: NamespaceNodeIndexFunc,
@@ -93,6 +104,9 @@ func NewInformers(clientset kubernetes.Interface, resyncPeriod time.Duration,
 	}
 
 	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
+	if err := nodeInformer.SetTransform(nodeTransform); err != nil {
+		return nil, fmt.Errorf("failed to set node informer transform: %w", err)
+	}
 
 	dryRunMode := []string{}
 	if dryRun {
@@ -111,6 +125,153 @@ func NewInformers(clientset kubernetes.Interface, resyncPeriod time.Duration,
 	}, nil
 }
 
+func excludedPodTransform(systemNamespacesRegex *regexp.Regexp) cache.TransformFunc {
+	return func(obj any) (any, error) {
+		pod, ok := obj.(*v1.Pod)
+		if !ok {
+			return obj, nil
+		}
+
+		isSystemNamespace := systemNamespacesRegex != nil && systemNamespacesRegex.MatchString(pod.Namespace)
+		if !isSystemNamespace && !isDaemonSetOwned(pod.OwnerReferences) {
+			return drainEligiblePodCacheObject(pod), nil
+		}
+
+		return &v1.Pod{
+			ObjectMeta: identityObjectMeta(
+				pod.Name,
+				pod.Namespace,
+				pod.UID,
+				pod.ResourceVersion,
+				nil,
+			),
+		}, nil
+	}
+}
+
+// drainEligiblePodCacheObject retains only fields used by pod indexes and drain decisions.
+// Keep this contract in sync with the cached Pod reads in this package.
+func drainEligiblePodCacheObject(pod *v1.Pod) *v1.Pod {
+	var annotations map[string]string
+	if devices, exists := pod.Annotations[model.PodDeviceAnnotationName]; exists {
+		annotations = map[string]string{model.PodDeviceAnnotationName: devices}
+	}
+
+	ownerReferences := make([]metav1.OwnerReference, len(pod.OwnerReferences))
+	for idx, owner := range pod.OwnerReferences {
+		ownerReferences[idx] = metav1.OwnerReference{Kind: owner.Kind}
+	}
+
+	var deletionTimestamp *metav1.Time
+	if pod.DeletionTimestamp != nil {
+		deletionTimestamp = pod.DeletionTimestamp.DeepCopy()
+	}
+
+	var terminationGracePeriodSeconds *int64
+	if pod.Spec.TerminationGracePeriodSeconds != nil {
+		terminationGracePeriodSeconds = ptr.To(*pod.Spec.TerminationGracePeriodSeconds)
+	}
+
+	return &v1.Pod{
+		TypeMeta: pod.TypeMeta,
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              pod.Name,
+			Namespace:         pod.Namespace,
+			UID:               pod.UID,
+			ResourceVersion:   pod.ResourceVersion,
+			Annotations:       annotations,
+			OwnerReferences:   ownerReferences,
+			DeletionTimestamp: deletionTimestamp,
+		},
+		Spec: v1.PodSpec{
+			NodeName:                      pod.Spec.NodeName,
+			TerminationGracePeriodSeconds: terminationGracePeriodSeconds,
+			Containers:                    trimContainers(pod.Spec.Containers),
+			InitContainers:                trimContainers(pod.Spec.InitContainers),
+		},
+		Status: v1.PodStatus{
+			Phase:      pod.Status.Phase,
+			Conditions: trimPodReadyConditions(pod.Status.Conditions),
+		},
+	}
+}
+
+func trimContainers(containers []v1.Container) []v1.Container {
+	cached := make([]v1.Container, len(containers))
+	for idx, container := range containers {
+		limits := make(v1.ResourceList, len(container.Resources.Limits))
+		for resourceName, quantity := range container.Resources.Limits {
+			limits[resourceName] = quantity.DeepCopy()
+		}
+
+		cached[idx].Resources.Limits = limits
+	}
+
+	return cached
+}
+
+func trimPodReadyConditions(conditions []v1.PodCondition) []v1.PodCondition {
+	var cached []v1.PodCondition
+	for _, condition := range conditions {
+		if condition.Type != v1.PodReady {
+			continue
+		}
+
+		cached = append(cached, v1.PodCondition{
+			Type:               condition.Type,
+			Status:             condition.Status,
+			LastTransitionTime: condition.LastTransitionTime,
+		})
+	}
+
+	return cached
+}
+
+func nodeTransform(obj any) (any, error) {
+	node, ok := obj.(*v1.Node)
+	if !ok {
+		return obj, nil
+	}
+
+	var annotations map[string]string
+	if quarantineHealthEvent, exists := node.Annotations[common.QuarantineHealthEventAnnotationKey]; exists {
+		annotations = map[string]string{
+			common.QuarantineHealthEventAnnotationKey: quarantineHealthEvent,
+		}
+	}
+
+	return &v1.Node{
+		ObjectMeta: identityObjectMeta(
+			node.Name,
+			"",
+			node.UID,
+			node.ResourceVersion,
+			annotations,
+		),
+	}, nil
+}
+
+func identityObjectMeta(name, namespace string, uid types.UID, resourceVersion string,
+	annotations map[string]string) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Name:            name,
+		Namespace:       namespace,
+		UID:             uid,
+		ResourceVersion: resourceVersion,
+		Annotations:     annotations,
+	}
+}
+
+func isDaemonSetOwned(ownerReferences []metav1.OwnerReference) bool {
+	for _, owner := range ownerReferences {
+		if owner.Kind == "DaemonSet" {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (i *Informers) HasSynced() bool {
 	return i.podInformer.HasSynced() && i.eventInformer.HasSynced() && i.nodeInformer.HasSynced()
 }
@@ -383,15 +544,13 @@ func (i *Informers) filterPodsWithGPURequests(pods []*v1.Pod) []*v1.Pod {
 }
 
 func (i *Informers) isDaemonSetPod(pod *v1.Pod) bool {
-	for _, owner := range pod.OwnerReferences {
-		if owner.Kind == "DaemonSet" {
-			slog.Info("Ignoring DaemonSet pod in namespace on node during eviction check",
-				"pod", pod.Name,
-				"namespace", pod.Namespace,
-				"node", pod.Spec.NodeName)
+	if isDaemonSetOwned(pod.OwnerReferences) {
+		slog.Info("Ignoring DaemonSet pod in namespace on node during eviction check",
+			"pod", pod.Name,
+			"namespace", pod.Namespace,
+			"node", pod.Spec.NodeName)
 
-			return true
-		}
+		return true
 	}
 
 	return false
@@ -798,6 +957,10 @@ func (i *Informers) GetNamespacesMatchingPattern(ctx context.Context,
 }
 
 func (i *Informers) compileExcludePattern(excludePattern string) (*regexp.Regexp, error) {
+	return compileExcludePattern(excludePattern)
+}
+
+func compileExcludePattern(excludePattern string) (*regexp.Regexp, error) {
 	if excludePattern == "" {
 		return nil, nil
 	}