feat: add max remediation attempts per equivalence group

This addresses issue #1543 by implementing a configurable limit
on the number of remediation attempts per equivalence group.

Changes:
- Add MaxRetryAttempts config field to TomlConfig
- Add RetryCount field to EquivalenceGroupState (persists in node annotation)
- Increment retry count on each remediation attempt
- Skip remediation when retry limit is exceeded
- Add tests for retry count tracking and persistence

The retry counter survives pod restarts because it's stored in the
node's annotation rather than in-memory state.

Fixes #1543

Signed-off-by: Billard <82095453+iacker@users.noreply.github.com>

Author: iacker

fault-remediation/pkg/annotation/annotation.go

@@ -104,11 +104,19 @@ func (m *NodeAnnotationManager) UpdateRemediationState(ctx context.Context, node
 			return err
 		}
 
+		// Increment retry count if this group already exists
+		existingGroup, exists := state.EquivalenceGroups[group]
+		retryCount := 0
+		if exists {
+			retryCount = existingGroup.RetryCount + 1
+		}
+
 		// Update state for the group
 		state.EquivalenceGroups[group] = EquivalenceGroupState{
 			MaintenanceCR: crName,
 			CreatedAt:     time.Now().UTC(),
 			ActionName:    actionName,
+			RetryCount:    retryCount,
 		}
 
 		// Marshal to JSON
@@ -131,7 +139,8 @@ func (m *NodeAnnotationManager) UpdateRemediationState(ctx context.Context, node
 		slog.InfoContext(ctx, "Updated remediation state annotation for node",
 			"node", nodeName,
 			"group", group,
-			"crName", crName)
+			"crName", crName,
+			"retryCount", retryCount)
 
 		return nil
 	})

fault-remediation/pkg/annotation/annotation_interface.go

@@ -47,4 +47,8 @@ type EquivalenceGroupState struct {
 	// Action that created the CR (e.g., "RESTART_BM")
 	// Required to look up the corresponding MaintenanceResource from the TomlConfig
 	ActionName string `json:"actionName"`
+
+	// RetryCount tracks how many remediation attempts have been made for this equivalence group.
+	// Survives pod restarts as it's persisted in the node annotation.
+	RetryCount int `json:"retryCount,omitempty"`
 }

fault-remediation/pkg/annotation/retry_count_test.go

@@ -0,0 +1,128 @@
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package annotation
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestRetryCountIncrementsOnUpdate(t *testing.T) {
+	ctx := context.Background()
+	nodeName := "test-node"
+	groupName := "test-group"
+
+	node := &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        nodeName,
+			Annotations: map[string]string{},
+		},
+	}
+
+	client := fake.NewClientBuilder().WithObjects(node).Build()
+	annotationManager := NodeAnnotationManager{client: client}
+
+	// First update - retry count should be 0
+	err := annotationManager.UpdateRemediationState(ctx, nodeName, groupName, "cr-1", "RESTART_BM")
+	require.NoError(t, err)
+
+	state, _, err := annotationManager.GetRemediationState(ctx, nodeName)
+	require.NoError(t, err)
+	assert.Equal(t, 0, state.EquivalenceGroups[groupName].RetryCount, "First attempt should have retry count 0")
+
+	// Second update - retry count should be 1
+	err = annotationManager.UpdateRemediationState(ctx, nodeName, groupName, "cr-2", "RESTART_BM")
+	require.NoError(t, err)
+
+	state, _, err = annotationManager.GetRemediationState(ctx, nodeName)
+	require.NoError(t, err)
+	assert.Equal(t, 1, state.EquivalenceGroups[groupName].RetryCount, "Second attempt should have retry count 1")
+
+	// Third update - retry count should be 2
+	err = annotationManager.UpdateRemediationState(ctx, nodeName, groupName, "cr-3", "RESTART_BM")
+	require.NoError(t, err)
+
+	state, _, err = annotationManager.GetRemediationState(ctx, nodeName)
+	require.NoError(t, err)
+	assert.Equal(t, 2, state.EquivalenceGroups[groupName].RetryCount, "Third attempt should have retry count 2")
+}
+
+func TestRetryCountIndependentPerGroup(t *testing.T) {
+	ctx := context.Background()
+	nodeName := "test-node"
+
+	node := &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        nodeName,
+			Annotations: map[string]string{},
+		},
+	}
+
+	client := fake.NewClientBuilder().WithObjects(node).Build()
+	annotationManager := NodeAnnotationManager{client: client}
+
+	// Create group-1 twice
+	err := annotationManager.UpdateRemediationState(ctx, nodeName, "group-1", "cr-1", "RESTART_BM")
+	require.NoError(t, err)
+	err = annotationManager.UpdateRemediationState(ctx, nodeName, "group-1", "cr-2", "RESTART_BM")
+	require.NoError(t, err)
+
+	// Create group-2 once
+	err = annotationManager.UpdateRemediationState(ctx, nodeName, "group-2", "cr-3", "COMPONENT_RESET")
+	require.NoError(t, err)
+
+	state, _, err := annotationManager.GetRemediationState(ctx, nodeName)
+	require.NoError(t, err)
+
+	assert.Equal(t, 1, state.EquivalenceGroups["group-1"].RetryCount, "group-1 should have retry count 1")
+	assert.Equal(t, 0, state.EquivalenceGroups["group-2"].RetryCount, "group-2 should have retry count 0")
+}
+
+func TestRetryCountPersistsAcrossPodRestarts(t *testing.T) {
+	ctx := context.Background()
+	nodeName := "test-node"
+	groupName := "test-group"
+
+	node := &corev1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        nodeName,
+			Annotations: map[string]string{},
+		},
+	}
+
+	client := fake.NewClientBuilder().WithObjects(node).Build()
+
+	// Simulate first pod session
+	manager1 := NodeAnnotationManager{client: client}
+	err := manager1.UpdateRemediationState(ctx, nodeName, groupName, "cr-1", "RESTART_BM")
+	require.NoError(t, err)
+
+	// Simulate pod restart - create new annotation manager instance
+	manager2 := NodeAnnotationManager{client: client}
+	err = manager2.UpdateRemediationState(ctx, nodeName, groupName, "cr-2", "RESTART_BM")
+	require.NoError(t, err)
+
+	// Verify retry count persisted
+	state, _, err := manager2.GetRemediationState(ctx, nodeName)
+	require.NoError(t, err)
+	assert.Equal(t, 1, state.EquivalenceGroups[groupName].RetryCount,
+		"Retry count should persist across pod restarts")
+}

fault-remediation/pkg/config/config.go

@@ -85,6 +85,10 @@ type TomlConfig struct {
 
 	// Common configuration
 	UpdateRetry UpdateRetry `toml:"updateRetry"`
+
+	// MaxRetryAttempts is the maximum number of remediation attempts per equivalence group
+	// before marking the remediation as failed. Zero means unlimited retries (legacy behavior).
+	MaxRetryAttempts int `toml:"maxRetryAttempts"`
 }
 
 // Validate checks the configuration for consistency and completeness.

fault-remediation/pkg/reconciler/reconciler.go

@@ -1065,6 +1065,19 @@ func (r *FaultRemediationReconciler) handleRemediationEvent(
 			nodeName)
 	}
 
+	// Check if we've exceeded the maximum retry attempts
+	maxRetries := r.Config.RemediationClient.GetConfig().MaxRetryAttempts
+	if maxRetries > 0 {
+		res, err, done := r.trySkipMaxRetriesExceeded(ctx, nodeName, groupConfig.EffectiveEquivalenceGroup,
+			maxRetries, eventWithToken, watcherInstance, healthEventStore)
+		if done {
+			span.SetAttributes(
+				attribute.String("fault_remediation.status", "max_retries_exceeded"),
+			)
+			return res, err
+		}
+	}
+
 	result, err := r.runLogCollectorAndRemediate(ctx, healthEvent, healthEventWithStatus, eventWithToken,
 		watcherInstance, healthEventStore, groupConfig, nodeName)
 	if err != nil {
@@ -1161,6 +1174,56 @@ func (r *FaultRemediationReconciler) trySkipResolvedEvent(
 	return result, err, true
 }
 
+// trySkipMaxRetriesExceeded returns (result, err, true) when the retry count for this
+// equivalence group has reached or exceeded the configured maximum; otherwise (zero, nil, false).
+func (r *FaultRemediationReconciler) trySkipMaxRetriesExceeded(
+	ctx context.Context,
+	nodeName string,
+	effectiveEquivalenceGroup string,
+	maxRetries int,
+	eventWithToken datastore.EventWithToken,
+	watcherInstance datastore.ChangeStreamWatcher,
+	healthEventStore datastore.HealthEventStore,
+) (ctrl.Result, error, bool) {
+	remediationState, _, err := r.annotationManager.GetRemediationState(ctx, nodeName)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// Node does not exist, not a retry limit issue
+			return ctrl.Result{}, nil, false
+		}
+		slog.ErrorContext(ctx, "Failed to get remediation state for retry check",
+			"node", nodeName,
+			"error", err)
+		return ctrl.Result{}, fmt.Errorf("failed to get remediation state for retry check: %w", err), true
+	}
+
+	groupState, exists := remediationState.EquivalenceGroups[effectiveEquivalenceGroup]
+	if !exists {
+		// First attempt for this group
+		return ctrl.Result{}, nil, false
+	}
+
+	if groupState.RetryCount >= maxRetries {
+		slog.WarnContext(ctx, "Maximum retry attempts exceeded for equivalence group",
+			"node", nodeName,
+			"group", effectiveEquivalenceGroup,
+			"retryCount", groupState.RetryCount,
+			"maxRetries", maxRetries)
+
+		metrics.EventsProcessed.WithLabelValues(metrics.CRStatusSkipped, nodeName).Inc()
+
+		// Mark as failed since we cannot remediate
+		if err := r.updateNodeRemediatedStatus(ctx, healthEventStore, eventWithToken, false); err != nil {
+			return ctrl.Result{}, err, true
+		}
+
+		result, err := r.markProcessedOrError(ctx, watcherInstance, eventWithToken, nodeName)
+		return result, err, true
+	}
+
+	return ctrl.Result{}, nil, false
+}
+
 // handleEventCoveredByExistingCR routes a shouldCreate=false decision: an event behind a
 // still-in-progress CR is requeued, an event covered by a terminal CR is finalized.
 func (r *FaultRemediationReconciler) handleEventCoveredByExistingCR(