diff --git a/tests/cross_functional/system_test/test_clone_deletion_after_max_cluster_space_utilization.py b/tests/cross_functional/system_test/test_clone_deletion_after_max_cluster_space_utilization.py index 6c2fc3c2adb..fad405267d3 100644 --- a/tests/cross_functional/system_test/test_clone_deletion_after_max_cluster_space_utilization.py +++ b/tests/cross_functional/system_test/test_clone_deletion_after_max_cluster_space_utilization.py @@ -6,8 +6,8 @@ from ocs_ci.utility.utils import TimeoutSampler from ocs_ci.helpers import helpers from ocs_ci.ocs.resources.pvc import flatten_image -from ocs_ci.ocs.exceptions import TimeoutExpiredError from ocs_ci.utility.prometheus import PrometheusAPI +from ocs_ci.ocs.exceptions import TimeoutExpiredError from ocs_ci.framework.pytest_customization.marks import ( skipif_external_mode, magenta_squad, @@ -61,6 +61,8 @@ def teardown(): logger.info("Starting the test setup") self.num_of_clones = 30 + self.clones_list = [] + self.interface_type = interface_type # Getting the total Storage capacity self.ceph_cluster = CephCluster() @@ -82,21 +84,20 @@ def teardown(): logger.info(f"capacity_to_write: {self.capacity_to_write}") # Calculating the file size - self.filesize = int(self.capacity_to_write / (self.num_of_clones + 1)) + self.filesize = float(self.capacity_to_write / (self.num_of_clones + 1)) logger.info(f"filesize to fill the cluster to full ratio: {self.filesize}") # Calculating the PVC size in GiB - self.pvc_size = int(self.filesize * 1.2) + self.pvc_size = float(self.filesize * 1.2) logger.info(f"pvc size: {self.pvc_size}") # Converting the filesize from GiB to MB - self.filesize = f"{int(self.filesize) * constants.GB2MB}M" + self.filesize = f"{round(self.filesize * constants.GB2MB)}M" logger.info( f"Total capacity size is : {self.ceph_capacity} GiB, " f"Free capacity size is : {self.ceph_free_capacity} GiB, " - f"Creating {self.num_of_clones} clones of {self.pvc_size} GiB PVC. " + f"Creating {self.num_of_clones} clones of {self.pvc_size} GiB PVC with file size of {self.filesize} " ) - self.pvc_obj = pvc_factory( interface=interface_type, size=self.pvc_size, status=constants.STATUS_BOUND ) @@ -111,6 +112,28 @@ def teardown(): self.pod_obj.get_fio_results() logger.info(f"IO finished on pod {self.pod_obj.name}") + # Function to create clones + def create_clones(self, num_of_clones, pvc_clone_factory, start_num=0): + for clone_num in range(start_num, start_num + num_of_clones): + logger.info(f"Start creation of clone number {clone_num}.") + cloned_pvc_obj = pvc_clone_factory( + self.pvc_obj, storageclass=self.pvc_obj.backed_sc, timeout=900 + ) + cloned_pvc_obj.reload() + + if self.interface_type == constants.CEPHBLOCKPOOL: + # flatten the image + flatten_image(cloned_pvc_obj) + logger.info( + f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created." + ) + cloned_pvc_obj.reload() + else: + logger.info( + f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created." + ) + self.clones_list.append(cloned_pvc_obj) + @skipif_external_mode @magenta_squad def test_clone_deletion_after_max_cluster_space_utilization( @@ -125,48 +148,48 @@ def test_clone_deletion_after_max_cluster_space_utilization( 5. After the cluster is out of full state and IOs started , Try to delete clones. 6. Clone deletion should be successful and should not give error messages. """ + # Creating the clones one by one and wait until they bound self.timeout = 1800 logger.info( f"Start creating {self.num_of_clones} clones on {interface_type} PVC of size {self.pvc_size} GB." ) - clones_list = [] - for clone_num in range(self.num_of_clones + 2): - logger.info(f"Start creation of clone number {clone_num}.") - cloned_pvc_obj = pvc_clone_factory( - self.pvc_obj, storageclass=self.pvc_obj.backed_sc, timeout=600 + # Create the initial set of clones + self.create_clones(self.num_of_clones, pvc_clone_factory) + + # Maximum number of attempts to avoid indefinite looping + max_attempts = 5 + attempt = 0 + + # Verify if expected alerts are seen; if not, continue creating extra clones + while attempt < max_attempts: + logger.info( + "Verify 'CephClusterCriticallyFull' ,CephOSDNearFull Alerts are seen " + ) + expected_alerts = ["CephOSDNearFull", "CephOSDCriticallyFull"] + prometheus = PrometheusAPI(threading_lock=threading_lock) + sample = TimeoutSampler( + timeout=180, + sleep=10, + func=prometheus.verify_alerts_via_prometheus, + expected_alerts=expected_alerts, + threading_lock=threading_lock, ) - cloned_pvc_obj.reload() - if interface_type == constants.CEPHBLOCKPOOL: - # flatten the image - flatten_image(cloned_pvc_obj) + if sample.wait_for_func_status(result=True): logger.info( - f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created." + "Expected alerts have been detected. Stopping clone creation." ) - cloned_pvc_obj.reload() + break else: - logger.info( - f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created." - ) - clones_list.append(cloned_pvc_obj) - - logger.info( - "Verify 'CephClusterCriticallyFull' ,CephOSDNearFull Alerts are seen " - ) - expected_alerts = ["CephOSDNearFull", "CephOSDCriticallyFull"] - prometheus = PrometheusAPI(threading_lock=threading_lock) - sample = TimeoutSampler( - timeout=600, - sleep=10, - func=prometheus.verify_alerts_via_prometheus, - expected_alerts=expected_alerts, - threading_lock=threading_lock, - ) + logger.info("Alerts not found yet. Creating extra clones...") + # Continue creating more clones (e.g., in batches of 2) + self.create_clones(2, start_num=len(self.clones_list)) + attempt += 1 - if not sample.wait_for_func_status(result=True): - logger.error(f"The alerts {expected_alerts} do not exist after 1200 sec") + if attempt == max_attempts: + logger.error("Maximum attempts reached. Expected alerts were not detected.") raise TimeoutExpiredError # Make the cluster out of full by increasing the full ratio. @@ -179,7 +202,7 @@ def test_clone_deletion_after_max_cluster_space_utilization( f"Start deleting {self.num_of_clones} clones on {interface_type} PVC of size {self.pvc_size} Gi." ) - for index, clone in enumerate(clones_list): + for index, clone in enumerate(self.clones_list): index += 1 pvc_reclaim_policy = clone.reclaim_policy clone.delete()