Skip to content

Commit ff34dd8

Browse files
yangchiuyangchiu
committed
test(robot): add test case Test RWX Failover And Auto Salvage When Volume Is Faulted
Signed-off-by: Yang Chiu <yang.chiu@suse.com>
1 parent 0ff68de commit ff34dd8

7 files changed

Lines changed: 153 additions & 101 deletions

File tree

‎e2e/keywords/sharemanager.resource‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ Check no sharemanager pod of ${workload_kind} ${workload_id} recreation
3737
${volume_name} = get_workload_volume_name ${workload_name}
3838
check_no_sharemanager_pod_recreation ${volume_name}
3939

40-
Wait For Sharemanager Pod Of ${workload_kind} ${workload_id} Running
40+
Wait for sharemanager pod of ${workload_kind} ${workload_id} running
4141
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
4242
${volume_name} = get_workload_volume_name ${workload_name}
4343
wait_for_share_manager_pod_running ${volume_name}
4444

45-
Wait For Sharemanager Pod Of ${workload_kind} ${workload_id} Deleted
45+
Wait for sharemanager pod of ${workload_kind} ${workload_id} deleted
4646
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
4747
${volume_name} = get_workload_volume_name ${workload_name}
4848
wait_for_sharemanager_pod_deleted ${volume_name}
@@ -63,31 +63,31 @@ Assert encrypted disk size in sharemanager pod for ${workload_kind} ${workload_i
6363
${share_manager_pod}= Set Variable share-manager-${volume_name}
6464
wait_for_encrypted_disk_size_in_sharemanager_pod ${share_manager_pod} ${volume_name} ${expected_size_byte}
6565

66-
Get Sharemanager Spec Image Of ${workload_kind} ${workload_id}
66+
Get sharemanager spec image of ${workload_kind} ${workload_id}
6767
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
6868
${volume_name} = get_workload_volume_name ${workload_name}
6969
${spec_image} = get_sharemanager_spec_image ${volume_name}
7070
[Return] ${spec_image}
7171

72-
Assert Sharemanager Current Image Of ${workload_kind} ${workload_id} Is ${expected_image}
72+
Assert sharemanager current image of ${workload_kind} ${workload_id} is ${expected_image}
7373
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
7474
${volume_name} = get_workload_volume_name ${workload_name}
7575
${current_image} = get_sharemanager_current_image ${volume_name}
7676
Should Be Equal ${current_image} ${expected_image}
7777

78-
Check Sharemanager Current Image
78+
Check sharemanager current image
7979
[Arguments] ${volume_name} ${expected_image}
8080
${current_image} = get_sharemanager_current_image ${volume_name}
8181
Should Be Equal ${current_image} ${expected_image}
8282

83-
Assert Sharemanager Current Image Of ${workload_kind} ${workload_id} Matches Spec Image
83+
Assert sharemanager current image of ${workload_kind} ${workload_id} matches spec image
8484
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
8585
${volume_name} = get_workload_volume_name ${workload_name}
8686
${spec_image} = get_sharemanager_spec_image ${volume_name}
8787
${current_image} = get_sharemanager_current_image ${volume_name}
8888
Should Be Equal ${current_image} ${spec_image}
8989

90-
Assert Sharemanager Pod Container Image Of ${workload_kind} ${workload_id} Matches Spec Image
90+
Assert sharemanager pod container image of ${workload_kind} ${workload_id} matches spec image
9191
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
9292
${volume_name} = get_workload_volume_name ${workload_name}
9393
${spec_image} = get_sharemanager_spec_image ${volume_name}

‎e2e/libs/metrics/metrics.py‎

Lines changed: 78 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -217,47 +217,53 @@ def check_longhorn_components_memory_cpu_usage():
217217
usage["backing-image-manager"]["memory"],
218218
cold_usage["backing-image-manager"]["memory"])
219219

220-
is_high_resource_consumption("csi-attacher", "cpu",
221-
usage["csi-attacher"]["cpu"],
222-
old_usage["csi-attacher"]["cpu"])
223-
is_high_resource_consumption("csi-attacher", "memory",
224-
usage["csi-attacher"]["memory"],
225-
old_usage["csi-attacher"]["memory"])
226-
227-
is_high_resource_consumption("csi-provisioner", "cpu",
228-
usage["csi-provisioner"]["cpu"],
229-
old_usage["csi-provisioner"]["cpu"])
230-
is_high_resource_consumption("csi-provisioner", "memory",
231-
usage["csi-provisioner"]["memory"],
232-
old_usage["csi-provisioner"]["memory"])
233-
234-
is_high_resource_consumption("csi-resizer", "cpu",
235-
usage["csi-resizer"]["cpu"],
236-
old_usage["csi-resizer"]["cpu"])
237-
is_high_resource_consumption("csi-resizer", "memory",
238-
usage["csi-resizer"]["memory"],
239-
old_usage["csi-resizer"]["memory"])
240-
241-
is_high_resource_consumption("csi-snapshotter", "cpu",
242-
usage["csi-snapshotter"]["cpu"],
243-
old_usage["csi-snapshotter"]["cpu"])
244-
is_high_resource_consumption("csi-snapshotter", "memory",
245-
usage["csi-snapshotter"]["memory"],
246-
old_usage["csi-snapshotter"]["memory"])
247-
248-
is_high_resource_consumption("engine-image", "cpu",
249-
usage["engine-image"]["cpu"],
250-
old_usage["engine-image"]["cpu"])
251-
is_high_resource_consumption("engine-image", "memory",
252-
usage["engine-image"]["memory"],
253-
old_usage["engine-image"]["memory"])
254-
255-
is_high_resource_consumption("instance-manager-v1", "cpu",
256-
usage["instance-manager-v1"]["cpu"],
257-
old_usage["instance-manager-v1"]["cpu"])
258-
is_high_resource_consumption("instance-manager-v1", "memory",
259-
usage["instance-manager-v1"]["memory"],
260-
old_usage["instance-manager-v1"]["memory"])
220+
if "csi-attacher" in usage and "csi-attacher" in old_usage:
221+
is_high_resource_consumption("csi-attacher", "cpu",
222+
usage["csi-attacher"]["cpu"],
223+
old_usage["csi-attacher"]["cpu"])
224+
is_high_resource_consumption("csi-attacher", "memory",
225+
usage["csi-attacher"]["memory"],
226+
old_usage["csi-attacher"]["memory"])
227+
228+
if "csi-provisioner" in usage and "csi-provisioner" in old_usage:
229+
is_high_resource_consumption("csi-provisioner", "cpu",
230+
usage["csi-provisioner"]["cpu"],
231+
old_usage["csi-provisioner"]["cpu"])
232+
is_high_resource_consumption("csi-provisioner", "memory",
233+
usage["csi-provisioner"]["memory"],
234+
old_usage["csi-provisioner"]["memory"])
235+
236+
if "csi-resizer" in usage and "csi-resizer" in old_usage:
237+
is_high_resource_consumption("csi-resizer", "cpu",
238+
usage["csi-resizer"]["cpu"],
239+
old_usage["csi-resizer"]["cpu"])
240+
is_high_resource_consumption("csi-resizer", "memory",
241+
usage["csi-resizer"]["memory"],
242+
old_usage["csi-resizer"]["memory"])
243+
244+
if "csi-snapshotter" in usage and "csi-snapshotter" in old_usage:
245+
is_high_resource_consumption("csi-snapshotter", "cpu",
246+
usage["csi-snapshotter"]["cpu"],
247+
old_usage["csi-snapshotter"]["cpu"])
248+
is_high_resource_consumption("csi-snapshotter", "memory",
249+
usage["csi-snapshotter"]["memory"],
250+
old_usage["csi-snapshotter"]["memory"])
251+
252+
if "engine-image" in usage and "engine-image" in old_usage:
253+
is_high_resource_consumption("engine-image", "cpu",
254+
usage["engine-image"]["cpu"],
255+
old_usage["engine-image"]["cpu"])
256+
is_high_resource_consumption("engine-image", "memory",
257+
usage["engine-image"]["memory"],
258+
old_usage["engine-image"]["memory"])
259+
260+
if "instance-manager-v1" in usage and "instance-manager-v1" in old_usage:
261+
is_high_resource_consumption("instance-manager-v1", "cpu",
262+
usage["instance-manager-v1"]["cpu"],
263+
old_usage["instance-manager-v1"]["cpu"])
264+
is_high_resource_consumption("instance-manager-v1", "memory",
265+
usage["instance-manager-v1"]["memory"],
266+
old_usage["instance-manager-v1"]["memory"])
261267

262268
if "instance-manager-v2" in usage and "instance-manager-v2" in old_usage:
263269
is_high_resource_consumption("instance-manager-v2", "cpu",
@@ -267,33 +273,37 @@ def check_longhorn_components_memory_cpu_usage():
267273
usage["instance-manager-v2"]["memory"],
268274
old_usage["instance-manager-v2"]["memory"])
269275

270-
is_high_resource_consumption("longhorn-csi-plugin", "cpu",
271-
usage["longhorn-csi-plugin"]["cpu"],
272-
old_usage["longhorn-csi-plugin"]["cpu"])
273-
is_high_resource_consumption("longhorn-csi-plugin", "memory",
274-
usage["longhorn-csi-plugin"]["memory"],
275-
old_usage["longhorn-csi-plugin"]["memory"])
276-
277-
is_high_resource_consumption("longhorn-driver-deployer", "cpu",
278-
usage["longhorn-driver-deployer"]["cpu"],
279-
old_usage["longhorn-driver-deployer"]["cpu"])
280-
is_high_resource_consumption("longhorn-driver-deployer", "memory",
281-
usage["longhorn-driver-deployer"]["memory"],
282-
old_usage["longhorn-driver-deployer"]["memory"])
283-
284-
is_high_resource_consumption("longhorn-manager", "cpu",
285-
usage["longhorn-manager"]["cpu"],
286-
old_usage["longhorn-manager"]["cpu"])
287-
is_high_resource_consumption("longhorn-manager", "memory",
288-
usage["longhorn-manager"]["memory"],
289-
old_usage["longhorn-manager"]["memory"])
290-
291-
is_high_resource_consumption("longhorn-ui", "cpu",
292-
usage["longhorn-ui"]["cpu"],
293-
old_usage["longhorn-ui"]["cpu"])
294-
is_high_resource_consumption("longhorn-ui", "memory",
295-
usage["longhorn-ui"]["memory"],
296-
old_usage["longhorn-ui"]["memory"])
276+
if "longhorn-csi-plugin" in usage and "longhorn-csi-plugin" in old_usage:
277+
is_high_resource_consumption("longhorn-csi-plugin", "cpu",
278+
usage["longhorn-csi-plugin"]["cpu"],
279+
old_usage["longhorn-csi-plugin"]["cpu"])
280+
is_high_resource_consumption("longhorn-csi-plugin", "memory",
281+
usage["longhorn-csi-plugin"]["memory"],
282+
old_usage["longhorn-csi-plugin"]["memory"])
283+
284+
if "longhorn-driver-deployer" in usage and "longhorn-driver-deployer" in old_usage:
285+
is_high_resource_consumption("longhorn-driver-deployer", "cpu",
286+
usage["longhorn-driver-deployer"]["cpu"],
287+
old_usage["longhorn-driver-deployer"]["cpu"])
288+
is_high_resource_consumption("longhorn-driver-deployer", "memory",
289+
usage["longhorn-driver-deployer"]["memory"],
290+
old_usage["longhorn-driver-deployer"]["memory"])
291+
292+
if "longhorn-manager" in usage and "longhorn-manager" in old_usage:
293+
is_high_resource_consumption("longhorn-manager", "cpu",
294+
usage["longhorn-manager"]["cpu"],
295+
old_usage["longhorn-manager"]["cpu"])
296+
is_high_resource_consumption("longhorn-manager", "memory",
297+
usage["longhorn-manager"]["memory"],
298+
old_usage["longhorn-manager"]["memory"])
299+
300+
if "longhorn-ui" in usage and "longhorn-ui" in old_usage:
301+
is_high_resource_consumption("longhorn-ui", "cpu",
302+
usage["longhorn-ui"]["cpu"],
303+
old_usage["longhorn-ui"]["cpu"])
304+
is_high_resource_consumption("longhorn-ui", "memory",
305+
usage["longhorn-ui"]["memory"],
306+
old_usage["longhorn-ui"]["memory"])
297307

298308
if "share-manager" in usage and "share-manager" in old_usage:
299309
is_high_resource_consumption("share-manager", "cpu",

‎e2e/tests/regression/test_rwx_volume.robot‎

Lines changed: 57 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,24 @@ Documentation RWX Volume Test Cases
33
44
Test Tags regression rwx
55

6+
Resource ../keywords/variables.resource
67
Resource ../keywords/common.resource
78
Resource ../keywords/deployment.resource
9+
Resource ../keywords/statefulset.resource
810
Resource ../keywords/storageclass.resource
911
Resource ../keywords/persistentvolumeclaim.resource
1012
Resource ../keywords/workload.resource
1113
Resource ../keywords/host.resource
1214
Resource ../keywords/k8s.resource
13-
Resource ../keywords/variables.resource
1415
Resource ../keywords/sharemanager.resource
1516
Resource ../keywords/longhorn.resource
1617
Resource ../keywords/setting.resource
18+
Resource ../keywords/metrics.resource
1719

1820
Test Setup Set up test environment
1921
Test Teardown Cleanup test resources
2022

2123
*** Variables ***
22-
${LOOP_COUNT} 1
23-
${RETRY_COUNT} 300
24-
${RETRY_INTERVAL} 1
2524
${RWX_UNINTERRUPTIBLE_SLEEP_CHECK_DURATION} 30
2625

2726
*** Test Cases ***
@@ -100,13 +99,13 @@ Test ShareManager Status Current Image After Fresh Install
10099
... - 3. Wait for the volume to become healthy and the share manager pod to be running
101100
... - 4. Assert that the ShareManager CR status.currentImage matches spec.image
102101
... - 5. Assert that the share manager pod container image matches spec.image
103-
Given Create Storageclass longhorn-test With dataEngine=${DATA_ENGINE}
104-
And Create Persistentvolumeclaim 0 volume_type=RWX sc_name=longhorn-test
105-
And Create Deployment 0 With Persistentvolumeclaim 0
106-
And Wait For Volume of deployment 0 Healthy
107-
When Wait For Sharemanager Pod Of deployment 0 Running
108-
Then Assert Sharemanager Current Image Of deployment 0 Matches Spec Image
109-
And Assert Sharemanager Pod Container Image Of deployment 0 Matches Spec Image
102+
Given Create storageclass longhorn-test with dataEngine=${DATA_ENGINE}
103+
And Create persistentvolumeclaim 0 volume_type=RWX sc_name=longhorn-test
104+
And Create deployment 0 with persistentvolumeclaim 0
105+
And Wait for volume of deployment 0 healthy
106+
When Wait for sharemanager pod of deployment 0 running
107+
Then Assert sharemanager current image of deployment 0 matches spec image
108+
And Assert sharemanager pod container image of deployment 0 matches spec image
110109

111110
Test ShareManager Status Current Image After Upgrade
112111
[Tags] upgrade sharemanager
@@ -137,35 +136,67 @@ Test ShareManager Status Current Image After Upgrade
137136
Skip LONGHORN_STABLE_VERSION not set - required for upgrade test
138137
END
139138

140-
Given Setting deleting-confirmation-flag Is Set To true
139+
Given setting deleting-confirmation-flag is set to true
141140
And Uninstall Longhorn
142141
And Check Longhorn CRD removed
143142

144143
When Install Longhorn stable version
145-
And Create Storageclass longhorn-test With dataEngine=v1
146-
And Create Persistentvolumeclaim 0 volume_type=RWX sc_name=longhorn-test
147-
And Create Deployment 0 With Persistentvolumeclaim 0
148-
And Wait For Volume of deployment 0 Healthy
149-
And Wait For Sharemanager Pod Of deployment 0 Running
144+
And Create storageclass longhorn-test with dataEngine=v1
145+
And Create persistentvolumeclaim 0 volume_type=RWX sc_name=longhorn-test
146+
And Create deployment 0 with persistentvolumeclaim 0
147+
And Wait for volume of deployment 0 healthy
148+
And Wait for sharemanager pod of deployment 0 running
150149

151150
# Record the pre-upgrade image — status.currentImage and spec.image are equal
152151
# on a freshly installed system; recording spec.image captures the stable-version image.
153-
${old_sharemanager_image} = Get Sharemanager Spec Image Of deployment 0
152+
${old_sharemanager_image} = get sharemanager spec image of deployment 0
154153

155-
When Upgrade Longhorn To Custom Version
154+
When Upgrade Longhorn to custom version
156155

157156
# The share manager pod has not been restarted yet, so status.currentImage must
158157
# still reflect the pre-upgrade (old) image.
159-
Then Assert Sharemanager Current Image Of deployment 0 Is ${old_sharemanager_image}
158+
Then Assert sharemanager current image of deployment 0 is ${old_sharemanager_image}
160159

161160
# Restart the share manager pod by cycling the workload.
162-
When Scale Down Deployment 0 To Detach Volume
163-
And Wait For Sharemanager Pod Of deployment 0 Deleted
164-
And Scale Up Deployment 0 To Attach Volume
165-
And Wait For Sharemanager Pod Of deployment 0 Running
161+
When Scale down deployment 0 to detach volume
162+
And Wait for sharemanager pod of deployment 0 deleted
163+
And Scale up deployment 0 to attach volume
164+
And Wait for sharemanager pod of deployment 0 running
166165

167166
# After the pod is recreated with the new image, status.currentImage must reflect
168167
# spec.image (the post-upgrade image).
169-
Then Assert Sharemanager Current Image Of deployment 0 Matches Spec Image
170-
And Assert Sharemanager Pod Container Image Of deployment 0 Matches Spec Image
168+
Then Assert sharemanager current image of deployment 0 matches spec image
169+
And Assert sharemanager pod container image of deployment 0 matches spec image
171170

171+
Test RWX Failover And Auto Salvage When Volume Is Faulted
172+
[Tags] rwx-fast-failover node-down
173+
[Documentation] If the volume is faulted, we don't need to have RWX fast failover
174+
... Issue: https://github.com/longhorn/longhorn/issues/9089
175+
...
176+
... 1. Enable RWX fast failover setting
177+
... 2. Deploy a workload which uses a RWX PVC
178+
... 3. Reduce the number of replicas to 1. Delete 2 replicas not on the same node as the share manager pod
179+
... 4. Record the CPU usage
180+
... 5. Turn off the node of the share manager pod
181+
... 6. Volume stay in faulted. Share manager pod is recreated but then error and deleted. the CPU usage is still small
182+
... 7. Turn on the node of the share-manager pod
183+
... 8. Volume is salvaged and should not be stuck in auto-salvage loop forever. Share manager pod is recreated and become running. CPU usage still is small
184+
Given Setting rwx-volume-fast-failover is set to true
185+
And Setting auto-salvage is set to true
186+
And Create storageclass longhorn-test with numberOfReplicas=3 dataEngine=${DATA_ENGINE}
187+
And Create statefulset 0 volume_type=RWX sc_name=longhorn-test
188+
And Wait for volume of statefulset 0 healthy
189+
And Update volume of statefulset 0 replica count to 1
190+
And Delete replica of statefulset 0 volume on all replica node
191+
And Write 1 MB data to file data.bin in statefulset 0
192+
And Get Longhorn components resource usage
193+
194+
When Power off volume node of statefulset 0
195+
Then Check volume of statefulset 0 kept in faulted
196+
And Check Longhorn components resource usage
197+
198+
When Power on off nodes
199+
Then Wait for volume of statefulset 0 healthy
200+
And Check statefulset 0 data in file data.bin is intact
201+
And Wait for sharemanager pod of statefulset 0 running
202+
And Check Longhorn components resource usage

‎pipelines/e2e/scripts/longhorn-setup.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ main(){
6262

6363
if [[ "${DISTRO}" == "talos" ]]; then
6464
install_metrics_server
65+
else
66+
scale_up_metrics_server
6567
fi
6668

6769
get_longhorn_repo

‎pipelines/pr_check/scripts/longhorn-setup.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ main(){
6565

6666
if [[ "${DISTRO}" == "talos" ]]; then
6767
install_metrics_server
68+
else
69+
scale_up_metrics_server
6870
fi
6971

7072
get_longhorn_repo

‎pipelines/utilities/install_metrics_server.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
install_metrics_server(){
22
kubectl apply -f "https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml"
33
kubectl patch deployment metrics-server -n kube-system --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]'
4+
scale_up_metrics_server
5+
}
6+
7+
scale_up_metrics_server(){
8+
kubectl scale deployment metrics-server -n kube-system --replicas=3
49
}

‎test_framework/scripts/longhorn-setup.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ main(){
9393

9494
if [[ "${DISTRO}" == "talos" ]]; then
9595
install_metrics_server
96+
else
97+
scale_up_metrics_server
9698
fi
9799

98100
get_longhorn_repo

0 commit comments

Comments
 (0)