Skip to content

Commit e5b5bdf

Browse files
chriscchienchriscchien
committed
test(robot): add case V2 Replica Migration Should Not Cause IO Stall
longhorn/longhorn#13311 Signed-off-by: Chris Chien <chris.chien@suse.com>
1 parent 4d4f97a commit e5b5bdf

5 files changed

Lines changed: 74 additions & 2 deletions

File tree

‎e2e/keywords/io.resource‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,32 @@ Wipe block device ${block_device_path} on node ${node_id}
4444
[Documentation] Wipe filesystem and partition signatures from a block device.
4545
${node_name} = get_node_by_index ${node_id}
4646
wipe_block_device_signatures ${block_device_path} ${node_name}
47+
48+
Start fsync writer on ${workload_kind} ${workload_id} and log per-write latency to ${file}
49+
[Documentation] Start a background fsync writer that logs per-write timing to /data/${file}
50+
${command}= Catenate SEPARATOR=${SPACE}
51+
... nohup sh -c 'while true; do
52+
... dd if=/dev/zero of=/data/io_scratch bs=64k count=1 conv=fsync 2>&1
53+
... | awk -F"copied, " "/copied/ {print \\$2+0}"
54+
... >> /data/${file}
55+
... ; done' > /dev/null 2>&1 & sleep 1
56+
57+
Run commands in ${workload_kind} ${workload_id}
58+
... commands=${command}
59+
60+
Get max latency from ${workload_kind} ${workload_id} latency log ${file}
61+
[Documentation] Get the maximum latency from the latency log file generated by the fsync writer.
62+
... The latency log file contains one plain float per line (elapsed seconds per write).
63+
${result} = Run commands in ${workload_kind} ${workload_id}
64+
... commands=awk 'BEGIN{m=0}{if($1+0>m)m=$1+0}END{print m}' /data/${file}
65+
${max_latency} = Convert To Number ${result}
66+
Set Test Variable ${max_latency}
67+
68+
Assert no IO stall from ${workload_kind} ${workload_id} latency log ${file}
69+
[Arguments] ${threshold_sec}=3.0
70+
Save file from ${workload_kind} ${workload_id} pod /data/${file} to host ${OUTPUT_DIR}/${file}
71+
Get max latency from ${workload_kind} ${workload_id} latency log ${file}
72+
Log Maximum latency: ${max_latency}s
73+
74+
Should Be True ${max_latency} < ${threshold_sec}
75+
... msg=IO stall detected: max latency ${max_latency}s exceeds ${threshold_sec}s threshold

‎e2e/keywords/workload.resource‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,12 @@ Write ${size} GB large data to file ${file_name} in pod ${pod_id}
6363
Run commands in ${workload_kind} ${workload_id}
6464
[Arguments] &{config}
6565
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
66-
run_commands_workload_pod ${workload_name} &{config}
66+
${result} = run_commands_workload_pod ${workload_name} &{config}
67+
RETURN ${result}
68+
69+
Save file from ${workload_kind} ${workload_id} pod ${src} to host ${dst}
70+
${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id}
71+
copy_file_from_workload_pod ${workload_name} ${src} ${dst}
6772

6873
Check pod ${pod_id} data in file ${file_name} is intact
6974
${pod_name} = generate_name_with_suffix pod ${pod_id}

‎e2e/libs/keywords/workload_keywords.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
from workload.workload import check_workload_pods_not_restarted
4444
from workload.workload import check_workload_pods_not_recreated
4545
from workload.workload import rollout_restart_workload
46+
from workload.workload import copy_file_from_workload_pod
4647

4748
from utility.constant import ANNOT_CHECKSUM
4849
from utility.constant import ANNOT_EXPANDED_SIZE
@@ -184,7 +185,10 @@ def stop_writing_workload_pod_data(self, workload_name):
184185
def run_commands_workload_pod(self, workload_name, commands):
185186
pod_name = get_workload_pod_names(workload_name)[0]
186187
logging(f'Running commands {commands} in pod {pod_name}')
187-
run_commands_in_pod(pod_name, commands)
188+
return run_commands_in_pod(pod_name, commands)
189+
190+
def copy_file_from_workload_pod(self, workload_name, src, dst):
191+
copy_file_from_workload_pod(workload_name, src, dst)
188192

189193
def wait_for_workload_pods_container_creating_or_running(self, workload_name, namespace="default"):
190194
logging(f'Waiting for {namespace} workload {workload_name} pods container creating or running')

‎e2e/libs/workload/workload.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -682,3 +682,8 @@ def check_workload_pods_not_recreated(workload_kind, workload_name, namespace="d
682682
def rollout_restart_workload(workload_kind, workload_name, namespace="default"):
683683
cmd = f"kubectl rollout restart {workload_kind}/{workload_name} -n {namespace}"
684684
subprocess_exec_cmd(cmd)
685+
686+
687+
def copy_file_from_workload_pod(workload_name, src, dst, namespace="default"):
688+
pod_name = get_workload_pod_names(workload_name, namespace)[0]
689+
subprocess_exec_cmd(f"kubectl cp {pod_name}:{src} {dst} -n {namespace}")

‎e2e/tests/regression/test_v2.robot‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Resource ../keywords/k8s.resource
2020
Resource ../keywords/orphan.resource
2121
Resource ../keywords/replica.resource
2222
Resource ../keywords/engine_frontend.resource
23+
Resource ../keywords/io.resource
2324

2425
Test Setup Set up v2 test environment
2526
Test Teardown Cleanup test resources
@@ -396,3 +397,31 @@ Test V2 Instance Manager Pod Recreate Loop When Engine Frontend Recovery Blocks
396397
And Wait for volume test-vol healthy
397398
And Write data to volume test-vol
398399
Then Check volume test-vol data is intact
400+
401+
V2 Replica Migration Should Not Cause IO Stall
402+
[Documentation] issue: https://github.com/longhorn/longhorn/issues/13309
403+
... Test steps:
404+
... 1. Create v2 SC with dataLocality: best-effort, numberOfReplicas: 2
405+
... 2. Create deployment using the volume, running an fsync writer logging per-write timing
406+
... 3. Cordon volume attached node
407+
... 4. Delete replica on volume attached node
408+
... 5. Wait for volume healthy
409+
... 6. Check no IO stall observed (max latency < 3 sec)
410+
IF '${DATA_ENGINE}' == 'v1'
411+
Skip Test only validate on v2 data engine
412+
END
413+
414+
Given Create storageclass longhorn-test with
415+
... dataEngine=v2
416+
... dataLocality=best-effort
417+
... numberOfReplicas=2
418+
And Create persistentvolumeclaim 0 volume_type=RWO sc_name=longhorn-test
419+
And Create deployment 0 with persistentvolumeclaim 0
420+
And Wait for volume of deployment 0 healthy
421+
422+
When Start fsync writer on deployment 0 and log per-write latency to lat.log
423+
And Cordon deployment 0 volume node
424+
Then Delete replica of deployment 0 volume on volume node
425+
And Wait for volume of deployment 0 attached and degraded
426+
And Wait for volume of deployment 0 healthy
427+
And Assert no IO stall from deployment 0 latency log lat.log

0 commit comments

Comments
 (0)