test(robot): Automate test case Upgrade Conflict Handling test

longhorn/longhorn-11160

Signed-off-by: Chris Chien <chris.chien@suse.com>

Author: chriscchien

e2e/keywords/longhorn.resource

@@ -125,3 +125,6 @@ Upgrade Longhorn to custom version should fail
 Check Longhorn node ${type} state on power off node is ${expect_state}
     ${status} =    get_longhorn_node_condition_status    ${last_volume_node}    ${type}
     Should Be Equal    ${status}    ${expect_state}
+
+Check longhorn manager pods not restarted after test start
+    check_pod_not_restart_after_specific_time    longhorn-system    app=longhorn-manager    ${test_start_time}

e2e/libs/keywords/workload_keywords.py

@@ -35,8 +35,10 @@
 from utility.constant import LABEL_LONGHORN_COMPONENT
 from utility.utility import convert_size_to_bytes
 from utility.utility import logging
+from utility.utility import list_namespaced_pod
 
 from volume import Volume
+from datetime import datetime, timezone
 
 
 class workload_keywords:
@@ -241,3 +243,14 @@ def check_workload_pod_did_not_restart(self, workload_name):
     def get_workload_pod_uids(self, workload_name):
         pod_list = get_workload_pods(workload_name)
         return {pod.metadata.name: pod.metadata.uid for pod in pod_list}
+
+    def check_pod_not_restart_after_specific_time(self, namespace, label, time):        
+        pods = list_namespaced_pod(namespace, label)
+
+        if time.tzinfo is None:
+            time = time.replace(tzinfo=timezone.utc)
+
+        for pod in pods:
+            creation_time = pod.metadata.creation_timestamp
+            logging(f"Comparing pod creation: {creation_time} < {time}")
+            assert creation_time < time, f"{pod.metadata.name} restarted after test started"

e2e/tests/negative/upgrade_conflict_handle.robot

@@ -0,0 +1,77 @@
+*** Settings ***
+Documentation    Manual Test Cases
+Test Tags    negative
+
+Resource    ../keywords/variables.resource
+Resource    ../keywords/common.resource
+Resource    ../keywords/longhorn.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/setting.resource
+Resource    ../keywords/storageclass.resource
+Resource    ../keywords/workload.resource
+Resource    ../keywords/statefulset.resource
+Resource    ../keywords/backup.resource
+Resource    ../keywords/recurringjob.resource
+Library     DateTime
+
+Test Setup    Set up test environment
+Test Teardown    Cleanup test resources
+
+*** Keywords ***
+Get test start time
+    ${test_start_time}=    Get Current Date    result_format=datetime
+    Log    ${test_start_time}
+    Set Test Variable    ${test_start_time}
+
+Perform recurring job workflow under load
+    ${NUM_VOLUMES} =    Set Variable    100
+    ${NUM_VOLUMES_DETACH} =    Set Variable    20
+    
+    When Get test start time
+    And Create storageclass longhorn-test with    dataEngine=${DATA_ENGINE}    
+    FOR    ${i}    IN RANGE    ${NUM_VOLUMES}
+        And Create statefulset ${i} using RWO volume with longhorn-test storageclass and size 10 Mi
+        And Write 1 MB data to file data.txt in statefulset ${i}
+    END
+
+    And Create snapshot recurringjob 0
+    ...    groups=["default"]
+    ...    cron=*/1 * * * *
+    ...    concurrency=5
+    ...    labels={"test":"recurringjob"}
+
+    FOR    ${i}    IN RANGE    ${NUM_VOLUMES_DETACH}
+        When Create backup ${i} for statefulset ${i} volume
+        Then Scale down statefulset ${i} to detach volume
+        And Scale up statefulset ${i} to attach volume
+    END
+            
+    And Check longhorn manager pods not restarted after test start
+
+*** Test Cases ***
+Test Stability of Longhorn with Large Workload
+    Perform recurring job workflow under load
+   
+Test Upgrade Stability with Large Workload
+    ${LONGHORN_STABLE_VERSION}=    Get Environment Variable    LONGHORN_STABLE_VERSION    default=''
+    IF    '${LONGHORN_STABLE_VERSION}' == ''
+        Fail    Environment variable LONGHORN_STABLE_VERSION is not set
+    END
+    
+    Given Set setting deleting-confirmation-flag to true
+    And Uninstall Longhorn
+    And Check Longhorn CRD removed
+
+    When Install Longhorn stable version    
+    And set_backupstore
+
+    When Perform recurring job workflow under load    
+    And Upgrade Longhorn to custom version
+    
+    When Get test start time
+    FOR    ${i}    IN RANGE    ${NUM_VOLUMES_DETACH}
+        And Scale down statefulset ${i} to detach volume
+        And Scale up statefulset ${i} to attach volume
+    END
+
+    And Check longhorn manager pods not restarted after test start