Skip to content

Commit 50e7673

Browse files
committed
test: cover stored gvk existing cr lookup
Signed-off-by: Alex Jun <aljun@nvidia.com>
1 parent 44d3344 commit 50e7673

1 file changed

Lines changed: 121 additions & 0 deletions

File tree

fault-remediation/pkg/reconciler/reconciler_e2e_test.go

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ import (
2929
"github.com/stretchr/testify/require"
3030
"google.golang.org/protobuf/types/known/timestamppb"
3131
corev1 "k8s.io/api/core/v1"
32+
apierrors "k8s.io/apimachinery/pkg/api/errors"
3233
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3335
"k8s.io/apimachinery/pkg/runtime/schema"
3436
"k8s.io/client-go/dynamic"
3537
"k8s.io/client-go/kubernetes"
@@ -440,6 +442,11 @@ func TestCRBasedDeduplication_Integration(t *testing.T) {
440442
require.NoError(t, err)
441443
assert.Contains(t, state.EquivalenceGroups, "restart")
442444
assert.NotEmpty(t, state.EquivalenceGroups["restart"].MaintenanceCR)
445+
assert.Equal(t, protos.RecommendedAction_RESTART_BM.String(), state.EquivalenceGroups["restart"].ActionName)
446+
assert.Equal(t, "janitor.dgxc.nvidia.com", state.EquivalenceGroups["restart"].ApiGroup)
447+
assert.Equal(t, "v1alpha1", state.EquivalenceGroups["restart"].Version)
448+
assert.Equal(t, "RebootNode", state.EquivalenceGroups["restart"].Kind)
449+
assert.Empty(t, state.EquivalenceGroups["restart"].Namespace)
443450
assert.WithinDuration(t, time.Now(), state.EquivalenceGroups["restart"].CreatedAt, 5*time.Second)
444451

445452
// Verify CR was actually created
@@ -1131,6 +1138,120 @@ func TestEventSequenceWithSupersedingGroup(t *testing.T) {
11311138
_ = testDynamic.Resource(gpuResetGVR).Delete(ctx, crName3, metav1.DeleteOptions{})
11321139
}
11331140

1141+
func TestExistingCRStatusUsesStoredGVKAnnotation_E2E(t *testing.T) {
1142+
ctx := testContext
1143+
1144+
nodeName := "test-node-stored-gvk-e2e"
1145+
createTestNode(ctx, nodeName, nil, map[string]string{"test": "label"})
1146+
defer func() {
1147+
_ = testClient.CoreV1().Nodes().Delete(ctx, nodeName, metav1.DeleteOptions{})
1148+
}()
1149+
1150+
rebootNodeGVR := schema.GroupVersionResource{
1151+
Group: "janitor.dgxc.nvidia.com",
1152+
Version: "v1alpha1",
1153+
Resource: "rebootnodes",
1154+
}
1155+
gpuResetGVR := schema.GroupVersionResource{
1156+
Group: "janitor.dgxc.nvidia.com",
1157+
Version: "v1alpha1",
1158+
Resource: "gpuresets",
1159+
}
1160+
existingCRName := "maintenance-" + nodeName + "-existing"
1161+
1162+
existingRebootNode := &unstructured.Unstructured{Object: map[string]interface{}{
1163+
"apiVersion": "janitor.dgxc.nvidia.com/v1alpha1",
1164+
"kind": "RebootNode",
1165+
"metadata": map[string]interface{}{
1166+
"name": existingCRName,
1167+
},
1168+
"spec": map[string]interface{}{
1169+
"nodeName": nodeName,
1170+
"force": false,
1171+
},
1172+
}}
1173+
_, err := testDynamic.Resource(rebootNodeGVR).Create(ctx, existingRebootNode, metav1.CreateOptions{})
1174+
require.NoError(t, err)
1175+
defer func() {
1176+
_ = testDynamic.Resource(rebootNodeGVR).Delete(ctx, existingCRName, metav1.DeleteOptions{})
1177+
_ = testDynamic.Resource(gpuResetGVR).Delete(ctx, "maintenance-"+nodeName+"-event-uses-stored-gvk",
1178+
metav1.DeleteOptions{})
1179+
}()
1180+
1181+
driftedRemediationActions := map[string]config.MaintenanceResource{
1182+
protos.RecommendedAction_RESTART_BM.String(): {
1183+
ApiGroup: "janitor.dgxc.nvidia.com",
1184+
Version: "v1alpha1",
1185+
Kind: "GPUReset",
1186+
TemplateFileName: "gpureset-template.yaml",
1187+
CompleteConditionType: "NodeReady",
1188+
EquivalenceGroup: "restart",
1189+
},
1190+
}
1191+
remediationClient, err := createTestRemediationClient(false, driftedRemediationActions)
1192+
require.NoError(t, err)
1193+
1194+
cfg := ReconcilerConfig{
1195+
RemediationClient: remediationClient,
1196+
StateManager: statemanager.NewStateManager(testClient),
1197+
UpdateMaxRetries: 3,
1198+
UpdateRetryDelay: 100 * time.Millisecond,
1199+
}
1200+
r := &FaultRemediationReconciler{
1201+
Config: cfg,
1202+
annotationManager: cfg.RemediationClient.GetAnnotationManager(),
1203+
}
1204+
1205+
err = r.annotationManager.UpdateRemediationState(
1206+
ctx,
1207+
nodeName,
1208+
"restart",
1209+
existingCRName,
1210+
protos.RecommendedAction_RESTART_BM.String(),
1211+
annotation.MaintenanceResourceReference{
1212+
ApiGroup: "janitor.dgxc.nvidia.com",
1213+
Version: "v1alpha1",
1214+
Kind: "RebootNode",
1215+
},
1216+
)
1217+
require.NoError(t, err)
1218+
require.Eventually(t, func() bool {
1219+
state, _, err := r.annotationManager.GetRemediationState(ctx, nodeName)
1220+
if err != nil {
1221+
return false
1222+
}
1223+
1224+
groupState, ok := state.EquivalenceGroups["restart"]
1225+
return ok && groupState.MaintenanceCR == existingCRName && groupState.Kind == "RebootNode"
1226+
}, 5*time.Second, 100*time.Millisecond, "stored RebootNode reference should be visible before handling event")
1227+
1228+
healthEventDoc := &events.HealthEventDoc{
1229+
ID: "event-uses-stored-gvk",
1230+
HealthEventWithStatus: model.HealthEventWithStatus{
1231+
CreatedAt: time.Now(),
1232+
HealthEvent: &protos.HealthEvent{
1233+
NodeName: nodeName,
1234+
RecommendedAction: protos.RecommendedAction_RESTART_BM,
1235+
},
1236+
HealthEventStatus: &protos.HealthEventStatus{},
1237+
},
1238+
}
1239+
1240+
_, err = r.handleRemediationEvent(ctx, healthEventDoc, datastore.EventWithToken{}, nil, &MockHealthEventStore{})
1241+
require.NoError(t, err)
1242+
1243+
state, _, err := r.annotationManager.GetRemediationState(ctx, nodeName)
1244+
require.NoError(t, err)
1245+
require.Contains(t, state.EquivalenceGroups, "restart")
1246+
assert.Equal(t, existingCRName, state.EquivalenceGroups["restart"].MaintenanceCR)
1247+
assert.Equal(t, "RebootNode", state.EquivalenceGroups["restart"].Kind)
1248+
1249+
_, err = testDynamic.Resource(gpuResetGVR).Get(ctx, "maintenance-"+nodeName+"-event-uses-stored-gvk",
1250+
metav1.GetOptions{})
1251+
assert.True(t, apierrors.IsNotFound(err),
1252+
"current config points at GPUReset, but stored RebootNode reference should suppress creation")
1253+
}
1254+
11341255
// TestFullReconcilerWithMockedMongoDB tests the entire reconciler flow
11351256
func TestFullReconcilerWithMockedMongoDB_E2E(t *testing.T) {
11361257
ctx, cancel := context.WithTimeout(testContext, 30*time.Second)

0 commit comments

Comments
 (0)