@@ -29,7 +29,9 @@ import (
2929 "github.com/stretchr/testify/require"
3030 "google.golang.org/protobuf/types/known/timestamppb"
3131 corev1 "k8s.io/api/core/v1"
32+ apierrors "k8s.io/apimachinery/pkg/api/errors"
3233 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3335 "k8s.io/apimachinery/pkg/runtime/schema"
3436 "k8s.io/client-go/dynamic"
3537 "k8s.io/client-go/kubernetes"
@@ -440,6 +442,11 @@ func TestCRBasedDeduplication_Integration(t *testing.T) {
440442 require .NoError (t , err )
441443 assert .Contains (t , state .EquivalenceGroups , "restart" )
442444 assert .NotEmpty (t , state .EquivalenceGroups ["restart" ].MaintenanceCR )
445+ assert .Equal (t , protos .RecommendedAction_RESTART_BM .String (), state .EquivalenceGroups ["restart" ].ActionName )
446+ assert .Equal (t , "janitor.dgxc.nvidia.com" , state .EquivalenceGroups ["restart" ].ApiGroup )
447+ assert .Equal (t , "v1alpha1" , state .EquivalenceGroups ["restart" ].Version )
448+ assert .Equal (t , "RebootNode" , state .EquivalenceGroups ["restart" ].Kind )
449+ assert .Empty (t , state .EquivalenceGroups ["restart" ].Namespace )
443450 assert .WithinDuration (t , time .Now (), state .EquivalenceGroups ["restart" ].CreatedAt , 5 * time .Second )
444451
445452 // Verify CR was actually created
@@ -1131,6 +1138,120 @@ func TestEventSequenceWithSupersedingGroup(t *testing.T) {
11311138 _ = testDynamic .Resource (gpuResetGVR ).Delete (ctx , crName3 , metav1.DeleteOptions {})
11321139}
11331140
1141+ func TestExistingCRStatusUsesStoredGVKAnnotation_E2E (t * testing.T ) {
1142+ ctx := testContext
1143+
1144+ nodeName := "test-node-stored-gvk-e2e"
1145+ createTestNode (ctx , nodeName , nil , map [string ]string {"test" : "label" })
1146+ defer func () {
1147+ _ = testClient .CoreV1 ().Nodes ().Delete (ctx , nodeName , metav1.DeleteOptions {})
1148+ }()
1149+
1150+ rebootNodeGVR := schema.GroupVersionResource {
1151+ Group : "janitor.dgxc.nvidia.com" ,
1152+ Version : "v1alpha1" ,
1153+ Resource : "rebootnodes" ,
1154+ }
1155+ gpuResetGVR := schema.GroupVersionResource {
1156+ Group : "janitor.dgxc.nvidia.com" ,
1157+ Version : "v1alpha1" ,
1158+ Resource : "gpuresets" ,
1159+ }
1160+ existingCRName := "maintenance-" + nodeName + "-existing"
1161+
1162+ existingRebootNode := & unstructured.Unstructured {Object : map [string ]interface {}{
1163+ "apiVersion" : "janitor.dgxc.nvidia.com/v1alpha1" ,
1164+ "kind" : "RebootNode" ,
1165+ "metadata" : map [string ]interface {}{
1166+ "name" : existingCRName ,
1167+ },
1168+ "spec" : map [string ]interface {}{
1169+ "nodeName" : nodeName ,
1170+ "force" : false ,
1171+ },
1172+ }}
1173+ _ , err := testDynamic .Resource (rebootNodeGVR ).Create (ctx , existingRebootNode , metav1.CreateOptions {})
1174+ require .NoError (t , err )
1175+ defer func () {
1176+ _ = testDynamic .Resource (rebootNodeGVR ).Delete (ctx , existingCRName , metav1.DeleteOptions {})
1177+ _ = testDynamic .Resource (gpuResetGVR ).Delete (ctx , "maintenance-" + nodeName + "-event-uses-stored-gvk" ,
1178+ metav1.DeleteOptions {})
1179+ }()
1180+
1181+ driftedRemediationActions := map [string ]config.MaintenanceResource {
1182+ protos .RecommendedAction_RESTART_BM .String (): {
1183+ ApiGroup : "janitor.dgxc.nvidia.com" ,
1184+ Version : "v1alpha1" ,
1185+ Kind : "GPUReset" ,
1186+ TemplateFileName : "gpureset-template.yaml" ,
1187+ CompleteConditionType : "NodeReady" ,
1188+ EquivalenceGroup : "restart" ,
1189+ },
1190+ }
1191+ remediationClient , err := createTestRemediationClient (false , driftedRemediationActions )
1192+ require .NoError (t , err )
1193+
1194+ cfg := ReconcilerConfig {
1195+ RemediationClient : remediationClient ,
1196+ StateManager : statemanager .NewStateManager (testClient ),
1197+ UpdateMaxRetries : 3 ,
1198+ UpdateRetryDelay : 100 * time .Millisecond ,
1199+ }
1200+ r := & FaultRemediationReconciler {
1201+ Config : cfg ,
1202+ annotationManager : cfg .RemediationClient .GetAnnotationManager (),
1203+ }
1204+
1205+ err = r .annotationManager .UpdateRemediationState (
1206+ ctx ,
1207+ nodeName ,
1208+ "restart" ,
1209+ existingCRName ,
1210+ protos .RecommendedAction_RESTART_BM .String (),
1211+ annotation.MaintenanceResourceReference {
1212+ ApiGroup : "janitor.dgxc.nvidia.com" ,
1213+ Version : "v1alpha1" ,
1214+ Kind : "RebootNode" ,
1215+ },
1216+ )
1217+ require .NoError (t , err )
1218+ require .Eventually (t , func () bool {
1219+ state , _ , err := r .annotationManager .GetRemediationState (ctx , nodeName )
1220+ if err != nil {
1221+ return false
1222+ }
1223+
1224+ groupState , ok := state .EquivalenceGroups ["restart" ]
1225+ return ok && groupState .MaintenanceCR == existingCRName && groupState .Kind == "RebootNode"
1226+ }, 5 * time .Second , 100 * time .Millisecond , "stored RebootNode reference should be visible before handling event" )
1227+
1228+ healthEventDoc := & events.HealthEventDoc {
1229+ ID : "event-uses-stored-gvk" ,
1230+ HealthEventWithStatus : model.HealthEventWithStatus {
1231+ CreatedAt : time .Now (),
1232+ HealthEvent : & protos.HealthEvent {
1233+ NodeName : nodeName ,
1234+ RecommendedAction : protos .RecommendedAction_RESTART_BM ,
1235+ },
1236+ HealthEventStatus : & protos.HealthEventStatus {},
1237+ },
1238+ }
1239+
1240+ _ , err = r .handleRemediationEvent (ctx , healthEventDoc , datastore.EventWithToken {}, nil , & MockHealthEventStore {})
1241+ require .NoError (t , err )
1242+
1243+ state , _ , err := r .annotationManager .GetRemediationState (ctx , nodeName )
1244+ require .NoError (t , err )
1245+ require .Contains (t , state .EquivalenceGroups , "restart" )
1246+ assert .Equal (t , existingCRName , state .EquivalenceGroups ["restart" ].MaintenanceCR )
1247+ assert .Equal (t , "RebootNode" , state .EquivalenceGroups ["restart" ].Kind )
1248+
1249+ _ , err = testDynamic .Resource (gpuResetGVR ).Get (ctx , "maintenance-" + nodeName + "-event-uses-stored-gvk" ,
1250+ metav1.GetOptions {})
1251+ assert .True (t , apierrors .IsNotFound (err ),
1252+ "current config points at GPUReset, but stored RebootNode reference should suppress creation" )
1253+ }
1254+
11341255// TestFullReconcilerWithMockedMongoDB tests the entire reconciler flow
11351256func TestFullReconcilerWithMockedMongoDB_E2E (t * testing.T ) {
11361257 ctx , cancel := context .WithTimeout (testContext , 30 * time .Second )
0 commit comments