From 82e2550bd1b0e5063e72dbe92ba147842b9bef7f Mon Sep 17 00:00:00 2001 From: Tiffany Nguyen Date: Wed, 26 Jul 2023 04:23:27 -0700 Subject: [PATCH] Scale test obc_start_time_respin_noobaa_pods (#7995) * Added new scale test obc_start_time_respin_noobaa_pods. * Fixed code base on reviewer comments. Signed-off-by: tiffanyn108 --- ocs_ci/ocs/scale_noobaa_lib.py | 28 +++- ...scale_obc_start_time_respin_noobaa_pods.py | 125 ++++++++++++++++++ 2 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/scale/noobaa/test_scale_obc_start_time_respin_noobaa_pods.py diff --git a/ocs_ci/ocs/scale_noobaa_lib.py b/ocs_ci/ocs/scale_noobaa_lib.py index 60b3ed36b72..ad4df41bf0c 100644 --- a/ocs_ci/ocs/scale_noobaa_lib.py +++ b/ocs_ci/ocs/scale_noobaa_lib.py @@ -331,23 +331,37 @@ def measure_obc_deletion_time(obc_name_list, timeout=60): return obc_dict -def noobaa_running_node_restart(pod_name): +def get_pod_obj(pod_name): """ - Function to restart node which has noobaa pod's running + Function to get pod object using pod name Args: - pod_name (str): Name of noobaa pod - + pod_name (str): Name of a pod + Return: + pod object (obj): Object of a pod """ - nb_pod_obj = pod.get_pod_obj( + pod_obj = pod.get_pod_obj( ( get_pod_name_by_pattern( - pattern=pod_name, namespace=config.ENV_DATA["cluster_namespace"] + pattern=pod_name, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE ) )[0], - namespace=config.ENV_DATA["cluster_namespace"], + namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, ) + return pod_obj + + +def noobaa_running_node_restart(pod_name): + """ + Function to restart node which has noobaa pod's running + + Args: + pod_name (str): Name of noobaa pod + + """ + + nb_pod_obj = get_pod_obj(pod_name) nb_node_name = pod.get_pod_node(nb_pod_obj).name factory = platform_nodes.PlatformNodesFactory() nodes = factory.get_nodes_platform() diff --git a/tests/e2e/scale/noobaa/test_scale_obc_start_time_respin_noobaa_pods.py b/tests/e2e/scale/noobaa/test_scale_obc_start_time_respin_noobaa_pods.py new file mode 100644 index 00000000000..631ad3254ee --- /dev/null +++ b/tests/e2e/scale/noobaa/test_scale_obc_start_time_respin_noobaa_pods.py @@ -0,0 +1,125 @@ +import logging +import time + +import pytest +import csv + +from ocs_ci.ocs import constants, scale_noobaa_lib +from ocs_ci.framework.testlib import scale, E2ETest +from ocs_ci.helpers import helpers +from ocs_ci.utility.utils import ocsci_log_path +from ocs_ci.ocs.resources.objectconfigfile import ObjectConfFile + +log = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def teardown(request): + def finalizer(): + scale_noobaa_lib.cleanup(constants.OPENSHIFT_STORAGE_NAMESPACE) + + request.addfinalizer(finalizer) + + +@scale +class TestScaleOBCStartTime(E2ETest): + + namespace = constants.OPENSHIFT_STORAGE_NAMESPACE + scale_obc_count = 10 + scale_obc_count_io = 2 + num_obc_batch = 5 + nb_pod_start_time = dict() + + @pytest.mark.parametrize( + argnames=["pod_name", "sc_name"], + argvalues=[ + pytest.param( + *["noobaa-core", constants.NOOBAA_SC], + marks=[ + pytest.mark.polarion_id("OCS-5127"), + ], + ), + pytest.param( + *["noobaa-db", constants.NOOBAA_SC], + marks=[ + pytest.mark.polarion_id("OCS-5128"), + ], + ), + ], + ) + def test_scale_obc_start_time_noobaa_pod_respin( + self, tmp_path, pod_name, sc_name, mcg_job_factory, timeout=5 + ): + """ + Created OBC without I/O running + Created OBC with I/O using mcg_job_factory() + Reset node which Noobaa pod is running on, then measure the startup time when + Noobaa pod reaches Running state + """ + + # Create OBCs with FIO running using mcg_job_factory() + for i in range(self.scale_obc_count_io): + exec(f"job{i} = mcg_job_factory()") + + log.info( + f"Start creating {self.scale_obc_count} " + f"OBC in a batch of {self.num_obc_batch}" + ) + for i in range(int(self.scale_obc_count / self.num_obc_batch)): + obc_dict_list = ( + scale_noobaa_lib.construct_obc_creation_yaml_bulk_for_kube_job( + no_of_obc=self.num_obc_batch, + sc_name=sc_name, + namespace=self.namespace, + ) + ) + # Create job profile + job_file = ObjectConfFile( + name="job_profile", + obj_dict_list=obc_dict_list, + project=self.namespace, + tmp_path=tmp_path, + ) + # Create kube_job + job_file.create(namespace=self.namespace) + time.sleep(timeout) + + # Check all the OBCs reached Bound state + obc_bound_list = scale_noobaa_lib.check_all_obc_reached_bound_state_in_kube_job( + kube_job_obj=job_file, + namespace=self.namespace, + no_of_obc=self.num_obc_batch, + ) + log.info(f"Number of OBCs in Bound state: {len(obc_bound_list)}") + + # Reset node which noobaa pods is running on + # And validate noobaa pods are re-spinned and in Running state + pod_obj = scale_noobaa_lib.get_pod_obj(pod_name) + scale_noobaa_lib.noobaa_running_node_restart(pod_name=pod_name) + + # Store noobaa pod start time on csv file + pod_start_time = helpers.pod_start_time(pod_obj=pod_obj) + log.info( + f"{pod_name} is taking {pod_start_time} seconds to reach Running state." + ) + + self.nb_pod_start_time.update(pod_start_time) + data_file = f"{ocsci_log_path()}/noobaa_pod_start_time.csv" + with open(f"{data_file}", "w") as fd: + csv_obj = csv.writer(fd) + for k, v in self.nb_pod_start_time.items(): + csv_obj.writerow([k, v]) + log.info(f"Noobaa pod(s) start time is saved in {data_file}") + + # Verify all OBCs are in Bound state after node restart + log.info("Verify all OBCs are in Bound state after node restart.....") + obc_status_list = scale_noobaa_lib.check_all_obcs_status( + namespace=self.namespace + ) + log.info( + "Number of OBCs in Bound state after node reset: " + f"{len(obc_status_list[0])}" + ) + assert ( + len(obc_status_list[0]) == self.scale_obc_count + ), "Not all OBCs in Bound state"