Skip to content

Commit 26d0eba

Browse files
yangchiuyangchiu
committed
test(robot): add test case Test CPU Manager Policy And Data Engine Number Of CPU Cores
Signed-off-by: Yang Chiu <yang.chiu@suse.com>
1 parent 8677762 commit 26d0eba

23 files changed

Lines changed: 375 additions & 41 deletions

‎e2e/keywords/common.resource‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,15 +64,23 @@ Run command until output is absent
6464
[Arguments] ${command} ${unexpected_output}
6565
execute_command_until_output_is_absent ${command} ${unexpected_output}
6666

67-
Run command in pod
68-
[Arguments] ${pod_name} ${namespace} ${command}
67+
Run command in pod ${namespace}/${pod_name}
68+
[Arguments] ${command}
6969
${result} = execute_command_in_pod ${pod_name} ${namespace} ${command}
7070
RETURN ${result}
7171

72-
Run command in pod and expect output
73-
[Arguments] ${pod_name} ${namespace} ${command} ${expected_output}
72+
Run command in pod ${namespace}/${pod_name} and expect output
73+
[Arguments] ${command} ${expected_output}
7474
execute_command_in_pod_and_expect_output ${pod_name} ${namespace} ${command} ${expected_output}
7575

76+
Run command in pod ${namespace}/${pod_name} and wait for output
77+
[Arguments] ${command} ${expected_output}
78+
execute_command_in_pod_and_wait_for_output ${pod_name} ${namespace} ${command} ${expected_output}
79+
80+
Run command in pod ${namespace}/${pod_name} and not expect output
81+
[Arguments] ${command} ${unexpected_output}
82+
execute_command_in_pod_and_not_expect_output ${pod_name} ${namespace} ${command} ${unexpected_output}
83+
7684
Test should pass
7785
No Operation
7886

‎e2e/keywords/host.resource‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ Documentation Physical Node Keywords
44
Library Collections
55
Library ../libs/keywords/common_keywords.py
66
Library ../libs/keywords/host_keywords.py
7+
Library ../libs/keywords/k8s_keywords.py
78
Library ../libs/keywords/network_keywords.py
89
Library ../libs/keywords/volume_keywords.py
910
Library ../libs/keywords/workload_keywords.py
@@ -128,6 +129,9 @@ SSH into node ${node_id} and run command
128129
${node_name} = get_node_by_index ${node_id}
129130
ssh_exec_on_node ${node_name} ${command}
130131

132+
Set cpu-manager-policy to ${policy} on all worker nodes
133+
set_cpu_manager_policy_on_all_worker_nodes ${policy}
134+
131135
Corrupt CSI block volume staging mount point on node ${node_id} for volume ${volume_name}
132136
[Documentation] Corrupt the CSI block volume staging mount point with explicit host operations.
133137
... 1. Create fake device symlink /dev/longhorn/${volume_name} -> /dev/zero.

‎e2e/keywords/k8s.resource‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ Wait for node ${node_id} ready
183183
${node_name} = get_node_by_index ${node_id}
184184
wait_for_node_ready ${node_name}
185185

186+
Wait for k8s cluster ready
187+
wait_for_cluster_ready
188+
186189
Check PDB not exist
187190
[Arguments] ${instance_manager}
188191
check_instance_manager_pdb_not_exist ${instance_manager}

‎e2e/keywords/longhorn.resource‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ Check ${engine_type} instance manager is not running on node ${node_id}
8787
Check v1 instance manager pods did not restart
8888
check_all_instance_managers_not_restart
8989

90+
Check v2 instance manager pods did not restart
91+
check_all_instance_managers_not_restart data_engine=v2
92+
9093
Check v1 instance manager pods recreated
9194
wait_all_instance_managers_recreated
9295

@@ -264,4 +267,12 @@ Verify ${engine_type} instance manager log on node ${node_id} not contain ${expe
264267
${node_name} = get_node_by_index ${node_id}
265268
${pod_name} = get_instance_manager_pod_on_node ${node_name} ${engine_type}
266269
Log ${pod_name}
267-
verify_pod_log_after_time_not_contains ${pod_name} ${expect_log} ${test_start_time}
270+
verify_pod_log_after_time_not_contains ${pod_name} ${expect_log} ${test_start_time}
271+
272+
Get ${engine_type} instance manager pod name on node ${node_id}
273+
${node_name} = get_node_by_index ${node_id}
274+
${pod_name} = get_instance_manager_pod_on_node ${node_name} ${engine_type}
275+
RETURN ${pod_name}
276+
277+
Wait for v2 instance manager pods restarted
278+
wait_all_instance_managers_recreated data_engine=v2

‎e2e/libs/instancemanager/base.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -58,18 +58,18 @@ def wait_for_all_instance_manager_removed(self):
5858

5959
assert False, f"Expected all instance managers to be removed, but still found {len(instance_managers)} instance managers: {instance_managers}"
6060

61-
def wait_all_instance_managers_recreated(self):
61+
def wait_all_instance_managers_recreated(self, data_engine="v1"):
6262
retry_count, retry_interval = get_retry_count_and_interval()
6363
core_api = client.CoreV1Api()
64-
baseline_time = datetime.now(timezone.utc)- timedelta(seconds=10)
64+
baseline_time = datetime.now(timezone.utc) - timedelta(seconds=10)
6565

6666
for i in range(retry_count):
6767
ims = get_longhorn_client().list_instance_manager()
68-
v1_im_names = [im.name for im in ims if im.dataEngine == "v1"]
68+
im_names = [im.name for im in ims if im.dataEngine == data_engine]
6969
recreated = []
70-
logging(f"Checking v1 instance managers {v1_im_names} have recreated")
70+
logging(f"Checking {data_engine} instance managers {im_names} have recreated ({i})")
7171

72-
for im_name in v1_im_names:
72+
for im_name in im_names:
7373
try:
7474
pod = core_api.read_namespaced_pod(name=im_name, namespace=constant.LONGHORN_NAMESPACE)
7575
creation_time = pod.metadata.creation_timestamp
@@ -81,21 +81,21 @@ def wait_all_instance_managers_recreated(self):
8181
logging(f"Checking instance manager existence error: {e}")
8282
continue
8383

84-
if len(recreated) == len(v1_im_names):
85-
logging(f"All instance-manager pods have restarted")
84+
if len(im_names) > 0 and len(recreated) == len(im_names):
85+
logging(f"All {data_engine} instance-manager pods have restarted: {recreated}")
8686
return
8787
time.sleep(retry_interval)
8888

89-
assert False, f"Instance managers never recreated after {retry_count} attempts"
89+
assert False, f"{data_engine} instance managers never recreated after {retry_count} attempts"
9090

91-
def check_all_instance_managers_not_restart(self):
91+
def check_all_instance_managers_not_restart(self, data_engine="v1"):
9292

9393
ims = get_longhorn_client().list_instance_manager()
94-
v1_im_names = [im.name for im in ims if im.dataEngine == "v1"]
95-
logging(f"Checking v1 instance managers {v1_im_names} didn't restart")
94+
im_names = [im.name for im in ims if im.dataEngine == data_engine]
95+
logging(f"Checking {data_engine} instance managers {im_names} didn't restart")
9696

9797
core_api = client.CoreV1Api()
98-
for im_name in v1_im_names:
98+
for im_name in im_names:
9999
pod = core_api.read_namespaced_pod(name=im_name, namespace=constant.LONGHORN_NAMESPACE)
100100
if pod.status.container_statuses[0].restart_count != 0:
101101
logging(f"Unexpected instance manager restart: {pod}")

‎e2e/libs/keywords/common_keywords.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,5 +128,41 @@ def execute_command_in_pod_and_expect_output(self, pod_name, namespace, cmd, exp
128128
time.sleep(retry_count)
129129
assert False, f"Failed to find '{expected_output}' in output of '{cmd}' in pod {pod_name}: {res}"
130130

131+
def execute_command_in_pod_and_wait_for_output(self, pod_name, namespace, cmd, expected_output):
132+
retry_count, retry_interval = get_retry_count_and_interval()
133+
try:
134+
pattern = re.compile(expected_output, re.DOTALL)
135+
except re.error:
136+
pattern = re.compile(re.escape(expected_output), re.DOTALL)
137+
for i in range(retry_count):
138+
logging(f"Waiting for command '{cmd}' in pod {pod_name} to return output matching '{expected_output}' ... ({i})")
139+
try:
140+
res = pod_exec(pod_name, namespace, cmd)
141+
if pattern.search(res):
142+
return
143+
except Exception as e:
144+
logging(f"Execute command '{cmd}' in pod {pod_name} and wait for output '{expected_output}' error: {e}")
145+
time.sleep(retry_interval)
146+
assert False, f"Failed to wait for command '{cmd}' in pod {pod_name} returning output matching '{expected_output}'"
147+
148+
def execute_command_in_pod_and_not_expect_output(self, pod_name, namespace, cmd, unexpected_output):
149+
retry_count, retry_interval = get_retry_count_and_interval()
150+
try:
151+
pattern = re.compile(unexpected_output, re.DOTALL)
152+
except re.error:
153+
pattern = re.compile(re.escape(unexpected_output), re.DOTALL)
154+
for i in range(retry_count):
155+
logging(f"Checking command '{cmd}' in pod {pod_name} does not return output matching '{unexpected_output}' ... ({i})")
156+
try:
157+
res = pod_exec(pod_name, namespace, cmd)
158+
if pattern.search(res):
159+
logging(f"Unexpected output matching '{unexpected_output}' found in command '{cmd}' result in pod {pod_name}: {res}")
160+
else:
161+
return
162+
except Exception as e:
163+
logging(f"Execute command '{cmd}' in pod {pod_name} and check for unexpected output '{unexpected_output}' error: {e}")
164+
time.sleep(retry_interval)
165+
assert False, f"Unexpected output matching '{unexpected_output}' found in command '{cmd}' result in pod {pod_name}: {res}"
166+
131167
def cleanup_events(self):
132168
cleanup_events()

‎e2e/libs/keywords/host_keywords.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from ssh.ssh import ssh_exec
1111

1212
from utility.utility import logging
13+
from utility.utility import get_retry_count_and_interval
1314

1415

1516
class host_keywords:
@@ -145,3 +146,49 @@ def get_host_log_files(self, node_name, log_path):
145146
def ssh_exec_on_node(self, node_name, cmd):
146147
logging(f"SSH into node {node_name} and run command: {cmd}")
147148
return ssh_exec(node_name, cmd)
149+
150+
def set_cpu_manager_policy_on_all_worker_nodes(self, policy):
151+
k8s_distro = os.environ.get("K8S_DISTRO", "k3s")
152+
if k8s_distro == "k3s":
153+
config_file = "/etc/rancher/k3s/config.yaml"
154+
service_name = "k3s-agent"
155+
elif k8s_distro == "rke2":
156+
config_file = "/etc/rancher/rke2/config.yaml"
157+
service_name = "rke2-agent"
158+
else:
159+
raise Exception(f"Unsupported K8S_DISTRO for cpu-manager-policy change: {k8s_distro}")
160+
161+
worker_nodes = self.node.list_node_names_by_role("worker")
162+
retry_count, retry_interval = get_retry_count_and_interval()
163+
for node_name in worker_nodes:
164+
set = True
165+
logging(f"Setting cpu-manager-policy to {policy} on node {node_name} via {config_file}")
166+
sed_cmd = f"sudo sed -i 's/cpu-manager-policy=[a-z]*/cpu-manager-policy={policy}/g' {config_file}"
167+
for i in range(retry_count):
168+
try:
169+
ssh_exec(node_name, sed_cmd)
170+
break
171+
except Exception as e:
172+
logging(f"Failed to set cpu-manager-policy on node {node_name}: {e} ... ({i})")
173+
set = False
174+
time.sleep(retry_interval)
175+
# Remove stale cpu-manager state file so kubelet accepts the new policy
176+
logging(f"Removing stale cpu_manager_state on node {node_name}")
177+
for i in range(retry_count):
178+
try:
179+
ssh_exec(node_name, "sudo rm -f /var/lib/kubelet/cpu_manager_state")
180+
break
181+
except Exception as e:
182+
logging(f"Failed to remove stale cpu_manager_state on node {node_name}: {e} ... ({i})")
183+
set = False
184+
time.sleep(retry_interval)
185+
logging(f"Restarting {service_name} on node {node_name}")
186+
for i in range(retry_count):
187+
try:
188+
ssh_exec(node_name, f"sudo systemctl restart {service_name}")
189+
break
190+
except Exception as e:
191+
logging(f"Failed to restart {service_name} on node {node_name}: {e} ... ({i})")
192+
set = False
193+
time.sleep(retry_interval)
194+
assert set, f"Failed to set cpu-manager-policy to {policy} on node {node_name}"

‎e2e/libs/keywords/instancemanager_keywords.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ def wait_for_all_instance_manager_removed(self):
1717
logging(f'Waiting for all instance manager removed')
1818
self.instancemanager.wait_for_all_instance_manager_removed()
1919

20-
def check_all_instance_managers_not_restart(self):
21-
self.instancemanager.check_all_instance_managers_not_restart()
20+
def check_all_instance_managers_not_restart(self, data_engine="v1"):
21+
self.instancemanager.check_all_instance_managers_not_restart(data_engine)
2222

23-
def wait_all_instance_managers_recreated(self):
24-
self.instancemanager.wait_all_instance_managers_recreated()
23+
def wait_all_instance_managers_recreated(self, data_engine="v1"):
24+
self.instancemanager.wait_all_instance_managers_recreated(data_engine)
2525

2626
def check_instance_manager_existence_on_node(self, node_name, engine_type, exist):
2727
logging(f"Checking {engine_type} instance manager exist = {exist} on node {node_name}")

‎e2e/libs/keywords/k8s_keywords.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
from k8s.k8s import wait_for_node_ready
3030
from k8s.k8s import get_csi_driver_storage_capacity
3131

32+
from utility.utility import wait_for_cluster_ready
33+
3234
from node import Node
3335

3436
from utility.utility import logging
@@ -207,3 +209,8 @@ def check_csi_driver_storage_capacity(self, expected_value, driver_name="driver.
207209
actual_value = get_csi_driver_storage_capacity(driver_name)
208210
assert actual_value == expected_value, f"Expected CSI driver {driver_name} storageCapacity to be {expected_value}, but got {actual_value}"
209211
return actual_value
212+
213+
def wait_for_cluster_ready(self):
214+
logging("Waiting for all Kubernetes nodes to be ready")
215+
wait_for_cluster_ready()
216+

‎e2e/tests/regression/test_environment.robot‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ Test mTLS Support
105105
And Check all Longhorn CRD removed
106106

107107
# Step 2: Create the longhorn-grpc-tls secret
108-
And And Run command
108+
And Run command
109109
... kubectl create ns ${LONGHORN_NAMESPACE}
110110
And Run command
111111
... kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn-tests/refs/heads/master/test_framework/templates/longhorn-grpc-tls.yml
@@ -119,33 +119,28 @@ Test mTLS Support
119119
${instance_manager_pod} = Get ${DATA_ENGINE} instance manager pod on node 0
120120

121121
# Step 5: Verify TCP port 8500 is reachable
122-
Then Run command in pod and expect output
123-
... ${instance_manager_pod} ${LONGHORN_NAMESPACE}
122+
Then Run command in pod ${LONGHORN_NAMESPACE}/${instance_manager_pod} and expect output
124123
... nc -zv 127.0.0.1 8500
125124
... succeeded
126125

127126
# Step 6: Install grpcurl inside the instance manager pod
128-
And Run command in pod
129-
... ${instance_manager_pod} ${LONGHORN_NAMESPACE}
127+
And Run command in pod ${LONGHORN_NAMESPACE}/${instance_manager_pod}
130128
... curl -fsSL https://github.com/fullstorydev/grpcurl/releases/download/v1.9.3/grpcurl_1.9.3_linux_x86_64.tar.gz | tar -xz -C /tmp && chmod +x /tmp/grpcurl
131129

132130
# Step 7: Plaintext gRPC must be rejected
133131
${random_suffix_1} = Evaluate __import__('uuid').uuid4().hex[:8]
134-
Run command in pod and expect output
135-
... ${instance_manager_pod} ${LONGHORN_NAMESPACE}
132+
Run command in pod ${LONGHORN_NAMESPACE}/${instance_manager_pod} and expect output
136133
... /tmp/grpcurl -plaintext -connect-timeout 5 -d '{"spec":{"name":"pwn-${random_suffix_1}","binary":"/bin/bash","args":["-c","id"],"portCount":0}}' '[127.0.0.1]:8500' ProcessManagerService/ProcessCreate
137134
... context deadline exceeded
138135

139136
# Step 8: Unauthenticated TLS (no client certificate) must be rejected
140137
${random_suffix_2} = Evaluate __import__('uuid').uuid4().hex[:8]
141-
Run command in pod and expect output
142-
... ${instance_manager_pod} ${LONGHORN_NAMESPACE}
138+
Run command in pod ${LONGHORN_NAMESPACE}/${instance_manager_pod} and expect output
143139
... /tmp/grpcurl -insecure -connect-timeout 5 -d '{"spec":{"name":"pwn-${random_suffix_2}","binary":"/bin/bash","args":["-c","id"],"portCount":0}}' '[127.0.0.1]:8500' ProcessManagerService/ProcessCreate
144140
... context deadline exceeded
145141

146142
# Step 9: gRPC server presents a valid TLS certificate
147-
Run command in pod and expect output
148-
... ${instance_manager_pod} ${LONGHORN_NAMESPACE}
143+
Run command in pod ${LONGHORN_NAMESPACE}/${instance_manager_pod} and expect output
149144
... timeout 5 openssl s_client -connect 127.0.0.1:8500
150145
... CONNECTED.*subject=.*issuer=
151146

0 commit comments

Comments
 (0)