Fixed calculations of osd full ratio by restructuring code. (#10704)

Signed-off-by: Avdhoot <[email protected]>

tests/cross_functional/system_test/test_clone_deletion_after_max_cluster_space_utilization.py

@@ -6,8 +6,8 @@
 from ocs_ci.utility.utils import TimeoutSampler
 from ocs_ci.helpers import helpers
 from ocs_ci.ocs.resources.pvc import flatten_image
-from ocs_ci.ocs.exceptions import TimeoutExpiredError
 from ocs_ci.utility.prometheus import PrometheusAPI
+from ocs_ci.ocs.exceptions import TimeoutExpiredError
 from ocs_ci.framework.pytest_customization.marks import (
     skipif_external_mode,
     magenta_squad,
@@ -61,6 +61,8 @@ def teardown():
         logger.info("Starting the test setup")
 
         self.num_of_clones = 30
+        self.clones_list = []
+        self.interface_type = interface_type
 
         # Getting the total Storage capacity
         self.ceph_cluster = CephCluster()
@@ -82,21 +84,20 @@ def teardown():
         logger.info(f"capacity_to_write: {self.capacity_to_write}")
 
         # Calculating the file size
-        self.filesize = int(self.capacity_to_write / (self.num_of_clones + 1))
+        self.filesize = float(self.capacity_to_write / (self.num_of_clones + 1))
         logger.info(f"filesize to fill the cluster to full ratio: {self.filesize}")
 
         # Calculating the PVC size in GiB
-        self.pvc_size = int(self.filesize * 1.2)
+        self.pvc_size = float(self.filesize * 1.2)
         logger.info(f"pvc size: {self.pvc_size}")
 
         # Converting the filesize from GiB to MB
-        self.filesize = f"{int(self.filesize) * constants.GB2MB}M"
+        self.filesize = f"{round(self.filesize * constants.GB2MB)}M"
         logger.info(
             f"Total capacity size is : {self.ceph_capacity} GiB, "
             f"Free capacity size is : {self.ceph_free_capacity} GiB, "
-            f"Creating {self.num_of_clones} clones of {self.pvc_size} GiB PVC. "
+            f"Creating {self.num_of_clones} clones of {self.pvc_size} GiB PVC with file size of {self.filesize} "
         )
-
         self.pvc_obj = pvc_factory(
             interface=interface_type, size=self.pvc_size, status=constants.STATUS_BOUND
         )
@@ -111,6 +112,28 @@ def teardown():
         self.pod_obj.get_fio_results()
         logger.info(f"IO finished on pod {self.pod_obj.name}")
 
+    # Function to create clones
+    def create_clones(self, num_of_clones, pvc_clone_factory, start_num=0):
+        for clone_num in range(start_num, start_num + num_of_clones):
+            logger.info(f"Start creation of clone number {clone_num}.")
+            cloned_pvc_obj = pvc_clone_factory(
+                self.pvc_obj, storageclass=self.pvc_obj.backed_sc, timeout=900
+            )
+            cloned_pvc_obj.reload()
+
+            if self.interface_type == constants.CEPHBLOCKPOOL:
+                # flatten the image
+                flatten_image(cloned_pvc_obj)
+                logger.info(
+                    f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created."
+                )
+                cloned_pvc_obj.reload()
+            else:
+                logger.info(
+                    f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created."
+                )
+            self.clones_list.append(cloned_pvc_obj)
+
     @skipif_external_mode
     @magenta_squad
     def test_clone_deletion_after_max_cluster_space_utilization(
@@ -125,48 +148,48 @@ def test_clone_deletion_after_max_cluster_space_utilization(
             5. After the cluster is out of full state and IOs started , Try to delete clones.
             6. Clone deletion should be successful and should not give error messages.
         """
+
         # Creating the clones one by one and wait until they bound
         self.timeout = 1800
         logger.info(
             f"Start creating {self.num_of_clones} clones on {interface_type} PVC of size {self.pvc_size} GB."
         )
-        clones_list = []
 
-        for clone_num in range(self.num_of_clones + 2):
-            logger.info(f"Start creation of clone number {clone_num}.")
-            cloned_pvc_obj = pvc_clone_factory(
-                self.pvc_obj, storageclass=self.pvc_obj.backed_sc, timeout=600
+        # Create the initial set of clones
+        self.create_clones(self.num_of_clones, pvc_clone_factory)
+
+        # Maximum number of attempts to avoid indefinite looping
+        max_attempts = 5
+        attempt = 0
+
+        # Verify if expected alerts are seen; if not, continue creating extra clones
+        while attempt < max_attempts:
+            logger.info(
+                "Verify 'CephClusterCriticallyFull' ,CephOSDNearFull Alerts are seen "
+            )
+            expected_alerts = ["CephOSDNearFull", "CephOSDCriticallyFull"]
+            prometheus = PrometheusAPI(threading_lock=threading_lock)
+            sample = TimeoutSampler(
+                timeout=180,
+                sleep=10,
+                func=prometheus.verify_alerts_via_prometheus,
+                expected_alerts=expected_alerts,
+                threading_lock=threading_lock,
             )
-            cloned_pvc_obj.reload()
 
-            if interface_type == constants.CEPHBLOCKPOOL:
-                # flatten the image
-                flatten_image(cloned_pvc_obj)
+            if sample.wait_for_func_status(result=True):
                 logger.info(
-                    f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created."
+                    "Expected alerts have been detected. Stopping clone creation."
                 )
-                cloned_pvc_obj.reload()
+                break
             else:
-                logger.info(
-                    f"Clone with name {cloned_pvc_obj.name} of size {self.pvc_size}Gi was created."
-                )
-            clones_list.append(cloned_pvc_obj)
-
-        logger.info(
-            "Verify 'CephClusterCriticallyFull' ,CephOSDNearFull Alerts are seen "
-        )
-        expected_alerts = ["CephOSDNearFull", "CephOSDCriticallyFull"]
-        prometheus = PrometheusAPI(threading_lock=threading_lock)
-        sample = TimeoutSampler(
-            timeout=600,
-            sleep=10,
-            func=prometheus.verify_alerts_via_prometheus,
-            expected_alerts=expected_alerts,
-            threading_lock=threading_lock,
-        )
+                logger.info("Alerts not found yet. Creating extra clones...")
+                # Continue creating more clones (e.g., in batches of 2)
+                self.create_clones(2, start_num=len(self.clones_list))
+                attempt += 1
 
-        if not sample.wait_for_func_status(result=True):
-            logger.error(f"The alerts {expected_alerts} do not exist after 1200 sec")
+        if attempt == max_attempts:
+            logger.error("Maximum attempts reached. Expected alerts were not detected.")
             raise TimeoutExpiredError
 
         # Make the cluster out of full by increasing the full ratio.
@@ -179,7 +202,7 @@ def test_clone_deletion_after_max_cluster_space_utilization(
             f"Start deleting {self.num_of_clones} clones on {interface_type} PVC of size {self.pvc_size} Gi."
         )
 
-        for index, clone in enumerate(clones_list):
+        for index, clone in enumerate(self.clones_list):
             index += 1
             pvc_reclaim_policy = clone.reclaim_policy
             clone.delete()