Scale test obc_start_time_respin_noobaa_pods (#7995)

* Added new scale test obc_start_time_respin_noobaa_pods.
* Fixed code base on reviewer comments.

Signed-off-by: tiffanyn108 <[email protected]>

ocs_ci/ocs/scale_noobaa_lib.py

@@ -331,23 +331,37 @@ def measure_obc_deletion_time(obc_name_list, timeout=60):
     return obc_dict
 
 
-def noobaa_running_node_restart(pod_name):
+def get_pod_obj(pod_name):
     """
-    Function to restart node which has noobaa pod's running
+    Function to get pod object using pod name
 
     Args:
-        pod_name (str): Name of noobaa pod
-
+        pod_name (str):  Name of a pod
+    Return:
+        pod object (obj): Object of a pod
     """
 
-    nb_pod_obj = pod.get_pod_obj(
+    pod_obj = pod.get_pod_obj(
         (
             get_pod_name_by_pattern(
-                pattern=pod_name, namespace=config.ENV_DATA["cluster_namespace"]
+                pattern=pod_name, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE
             )
         )[0],
-        namespace=config.ENV_DATA["cluster_namespace"],
+        namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
     )
+    return pod_obj
+
+
+def noobaa_running_node_restart(pod_name):
+    """
+    Function to restart node which has noobaa pod's running
+
+    Args:
+        pod_name (str): Name of noobaa pod
+
+    """
+
+    nb_pod_obj = get_pod_obj(pod_name)
     nb_node_name = pod.get_pod_node(nb_pod_obj).name
     factory = platform_nodes.PlatformNodesFactory()
     nodes = factory.get_nodes_platform()

tests/e2e/scale/noobaa/test_scale_obc_start_time_respin_noobaa_pods.py

@@ -0,0 +1,125 @@
+import logging
+import time
+
+import pytest
+import csv
+
+from ocs_ci.ocs import constants, scale_noobaa_lib
+from ocs_ci.framework.testlib import scale, E2ETest
+from ocs_ci.helpers import helpers
+from ocs_ci.utility.utils import ocsci_log_path
+from ocs_ci.ocs.resources.objectconfigfile import ObjectConfFile
+
+log = logging.getLogger(__name__)
+
+
+@pytest.fixture(autouse=True)
+def teardown(request):
+    def finalizer():
+        scale_noobaa_lib.cleanup(constants.OPENSHIFT_STORAGE_NAMESPACE)
+
+    request.addfinalizer(finalizer)
+
+
+@scale
+class TestScaleOBCStartTime(E2ETest):
+
+    namespace = constants.OPENSHIFT_STORAGE_NAMESPACE
+    scale_obc_count = 10
+    scale_obc_count_io = 2
+    num_obc_batch = 5
+    nb_pod_start_time = dict()
+
+    @pytest.mark.parametrize(
+        argnames=["pod_name", "sc_name"],
+        argvalues=[
+            pytest.param(
+                *["noobaa-core", constants.NOOBAA_SC],
+                marks=[
+                    pytest.mark.polarion_id("OCS-5127"),
+                ],
+            ),
+            pytest.param(
+                *["noobaa-db", constants.NOOBAA_SC],
+                marks=[
+                    pytest.mark.polarion_id("OCS-5128"),
+                ],
+            ),
+        ],
+    )
+    def test_scale_obc_start_time_noobaa_pod_respin(
+        self, tmp_path, pod_name, sc_name, mcg_job_factory, timeout=5
+    ):
+        """
+        Created OBC without I/O running
+        Created OBC with I/O using mcg_job_factory()
+        Reset node which Noobaa pod is running on, then measure the startup time when
+        Noobaa pod reaches Running state
+        """
+
+        # Create OBCs with FIO running using mcg_job_factory()
+        for i in range(self.scale_obc_count_io):
+            exec(f"job{i} = mcg_job_factory()")
+
+        log.info(
+            f"Start creating  {self.scale_obc_count} "
+            f"OBC in a batch of {self.num_obc_batch}"
+        )
+        for i in range(int(self.scale_obc_count / self.num_obc_batch)):
+            obc_dict_list = (
+                scale_noobaa_lib.construct_obc_creation_yaml_bulk_for_kube_job(
+                    no_of_obc=self.num_obc_batch,
+                    sc_name=sc_name,
+                    namespace=self.namespace,
+                )
+            )
+            # Create job profile
+            job_file = ObjectConfFile(
+                name="job_profile",
+                obj_dict_list=obc_dict_list,
+                project=self.namespace,
+                tmp_path=tmp_path,
+            )
+            # Create kube_job
+            job_file.create(namespace=self.namespace)
+            time.sleep(timeout)
+
+        # Check all the OBCs reached Bound state
+        obc_bound_list = scale_noobaa_lib.check_all_obc_reached_bound_state_in_kube_job(
+            kube_job_obj=job_file,
+            namespace=self.namespace,
+            no_of_obc=self.num_obc_batch,
+        )
+        log.info(f"Number of OBCs in Bound state: {len(obc_bound_list)}")
+
+        # Reset node which noobaa pods is running on
+        # And validate noobaa pods are re-spinned and in Running state
+        pod_obj = scale_noobaa_lib.get_pod_obj(pod_name)
+        scale_noobaa_lib.noobaa_running_node_restart(pod_name=pod_name)
+
+        # Store noobaa pod start time on csv file
+        pod_start_time = helpers.pod_start_time(pod_obj=pod_obj)
+        log.info(
+            f"{pod_name} is taking {pod_start_time} seconds to reach Running state."
+        )
+
+        self.nb_pod_start_time.update(pod_start_time)
+        data_file = f"{ocsci_log_path()}/noobaa_pod_start_time.csv"
+        with open(f"{data_file}", "w") as fd:
+            csv_obj = csv.writer(fd)
+            for k, v in self.nb_pod_start_time.items():
+                csv_obj.writerow([k, v])
+        log.info(f"Noobaa pod(s) start time is saved in {data_file}")
+
+        # Verify all OBCs are in Bound state after node restart
+        log.info("Verify all OBCs are in Bound state after node restart.....")
+        obc_status_list = scale_noobaa_lib.check_all_obcs_status(
+            namespace=self.namespace
+        )
+        log.info(
+            "Number of OBCs in Bound state after node reset: "
+            f"{len(obc_status_list[0])}"
+        )
+        assert (
+            len(obc_status_list[0]) == self.scale_obc_count
+        ), "Not all OBCs in Bound state"