Create two new test cases for resize osd and add verification for OSD…

…s encryption (#9813)

* Create two new test cases:
    Test resize osd when the cluster capacity is near full
    Test resize osd for large differences
    Add the MAX cluster size, and check that the expected size is less than the Max cluster size.
    Add verification steps for OSD encryption.

* Create a new fixture for cluster workload storage utilization, add new post check for MAX total cluster capacity, change the way we increase the osd size

Signed-off-by: Itzhak Kave <[email protected]>

ocs_ci/helpers/osd_resize.py

@@ -31,6 +31,8 @@
     MAX_RESIZE_OSD,
     AWS_MAX_RESIZE_OSD_COUNT,
     AWS_PLATFORM,
+    MAX_TOTAL_CLUSTER_CAPACITY,
+    MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY,
 )
 
 
@@ -325,24 +327,38 @@ def check_ceph_health_after_resize_osd(
     ), "Data re-balance failed to complete"
 
 
-def check_resize_osd_pre_conditions():
+def check_resize_osd_pre_conditions(expected_storage_size):
     """
     Check the resize osd pre-conditions:
     1. Check that the current storage size is less than the osd max size
     2. If we use AWS, check that the osd resize count is no more than the AWS max resize count
 
     If the conditions are not met, the test will be skipped.
 
+    Args:
+        expected_storage_size (str): The expected storage size for the storage cluster
+
     """
-    current_storage_size = get_storage_size()
-    current_storage_size_in_gb = convert_device_size(current_storage_size, "GB", 1024)
+    expected_storage_size_in_gb = convert_device_size(expected_storage_size, "GB", 1024)
     max_storage_size_in_gb = convert_device_size(MAX_RESIZE_OSD, "GB", 1024)
-    if current_storage_size_in_gb >= max_storage_size_in_gb:
+    if expected_storage_size_in_gb > max_storage_size_in_gb:
         pytest.skip(
-            f"The current storage size {current_storage_size} is greater or equal to the "
+            f"The expected storage size {expected_storage_size} is greater than the "
             f"max resize osd {MAX_RESIZE_OSD}"
         )
 
+    if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM:
+        max_cluster_capacity = MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY
+    else:
+        max_cluster_capacity = MAX_TOTAL_CLUSTER_CAPACITY
+    max_cluster_capacity_in_gb = convert_device_size(max_cluster_capacity, "GB", 1024)
+    expected_cluster_capacity_in_gb = expected_storage_size_in_gb * len(get_osd_pods())
+    if expected_cluster_capacity_in_gb > max_cluster_capacity_in_gb:
+        pytest.skip(
+            f"The expected cluster capacity {expected_cluster_capacity_in_gb}Gi is greater than the "
+            f"max cluster capacity {max_cluster_capacity}"
+        )
+
     config.RUN["resize_osd_count"] = config.RUN.get("resize_osd_count", 0)
     logger.info(f"resize osd count = {config.RUN['resize_osd_count']}")
     if (

ocs_ci/ocs/constants.py

@@ -2810,6 +2810,9 @@
 # Resize osd
 MAX_RESIZE_OSD = "8Ti"
 AWS_MAX_RESIZE_OSD_COUNT = 1
+# The max total cluster capacity, including all OSDs
+MAX_TOTAL_CLUSTER_CAPACITY = "12Ti"
+MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY = "24Ti"
 
 # CCOCTL
 CCOCTL_LOG_FILE = "ccoctl-service-id.log"

ocs_ci/utility/utils.py

@@ -4916,3 +4916,43 @@ def get_latest_release_version():
         return exec_cmd(cmd, shell=True).stdout.decode("utf-8").strip()
     except CommandFailed:
         return
+
+
+def sum_of_two_storage_sizes(storage_size1, storage_size2, convert_size=1024):
+    """
+    Calculate the sum of two storage sizes given as strings.
+    Valid units: "Mi", "Gi", "Ti", "MB", "GB", "TB".
+
+    Args:
+        storage_size1 (str): The first storage size, e.g., "800Mi", "100Gi", "2Ti".
+        storage_size2 (str): The second storage size, e.g., "700Mi", "500Gi", "300Gi".
+        convert_size (int): Set convert by 1000 or 1024. The default value is 1024.
+
+    Returns:
+        str: The sum of the two storage sizes as a string, e.g., "1500Mi", "600Gi", "2300Gi".
+
+    Raises:
+        ValueError: If the units of the storage sizes are not match the Valid units
+
+    """
+    valid_units = {"Mi", "Gi", "Ti", "MB", "GB", "TB"}
+    unit1 = storage_size1[-2:]
+    unit2 = storage_size2[-2:]
+    if unit1 not in valid_units or unit2 not in valid_units:
+        raise ValueError(f"Storage sizes must have valid units: {valid_units}")
+
+    storage_size1 = storage_size1.replace("B", "i")
+    storage_size2 = storage_size2.replace("B", "i")
+
+    if "Mi" in f"{storage_size1}{storage_size2}":
+        unit, units_to_convert = "Mi", "MB"
+    elif "Gi" in f"{storage_size1}{storage_size2}":
+        unit, units_to_convert = "Gi", "GB"
+    else:
+        unit, units_to_convert = "Ti", "TB"
+
+    size1 = convert_device_size(storage_size1, units_to_convert, convert_size)
+    size2 = convert_device_size(storage_size2, units_to_convert, convert_size)
+    size = size1 + size2
+    new_storage_size = f"{size}{unit}"
+    return new_storage_size

tests/conftest.py

@@ -34,6 +34,7 @@
 from ocs_ci.ocs import constants, defaults, fio_artefacts, node, ocp, platform_nodes
 from ocs_ci.ocs.acm.acm import login_to_acm
 from ocs_ci.ocs.awscli_pod import create_awscli_pod, awscli_pod_cleanup
+from ocs_ci.ocs.benchmark_operator_fio import get_file_size, BenchmarkOperatorFIO
 from ocs_ci.ocs.bucket_utils import (
     craft_s3_command,
     put_bucket_policy,
@@ -7660,3 +7661,53 @@ def update_current_active_test_marks_global(request):
     """
     marks = [mark.name for mark in request.node.iter_markers()]
     ocs_ci.framework.pytest_customization.marks.current_test_marks = marks
+
+
+@pytest.fixture(scope="function")
+def benchmark_workload_storageutilization(request):
+    """
+    This fixture is for cluster storage utilization using the benchmark operator.
+
+    """
+    benchmark_obj = None
+
+    def factory(
+        target_percentage,
+        jobs="read",
+        read_runtime=30,
+        bs="4096KiB",
+        storageclass=constants.DEFAULT_STORAGECLASS_RBD,
+        timeout_completed=2400,
+    ):
+        """
+        Setup of benchmark fio
+
+        Args:
+            target_percentage (int): The number of percentage to fill up the cluster
+            jobs (str): fio job types to run, for example the readwrite option
+            read_runtime (int): Amount of time in seconds to run read workloads
+            bs (str): the Block size that need to used for the prefill
+            storageclass (str): StorageClass to use for PVC per server pod
+            timeout_completed (int): timeout client pod move to completed state
+
+        """
+        nonlocal benchmark_obj
+
+        size = get_file_size(target_percentage)
+        benchmark_obj = BenchmarkOperatorFIO()
+        benchmark_obj.setup_benchmark_fio(
+            total_size=size,
+            jobs=jobs,
+            read_runtime=read_runtime,
+            bs=bs,
+            storageclass=storageclass,
+            timeout_completed=timeout_completed,
+        )
+        benchmark_obj.run_fio_benchmark_operator(is_completed=True)
+
+    def finalizer():
+        if benchmark_obj is not None:
+            benchmark_obj.cleanup()
+
+    request.addfinalizer(finalizer)
+    return factory

tests/functional/z_cluster/cluster_expansion/test_resize_osd.py

@@ -20,6 +20,7 @@
     tier1,
     tier4b,
     tier4c,
+    tier4a,
 )
 from ocs_ci.ocs.constants import VOLUME_MODE_BLOCK, OSD, ROOK_OPERATOR, MON_DAEMON
 from ocs_ci.helpers.osd_resize import (
@@ -35,12 +36,22 @@
     verify_md5sum_on_pod_files,
 )
 from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs, get_deviceset_pvs
-from ocs_ci.ocs.resources.storage_cluster import get_storage_size
+from ocs_ci.ocs.resources.storage_cluster import (
+    get_storage_size,
+    osd_encryption_verification,
+    resize_osd,
+)
 from ocs_ci.helpers.sanity_helpers import Sanity
 from ocs_ci.ocs.node import get_nodes, wait_for_nodes_status
 from ocs_ci.ocs.cluster import is_vsphere_ipi_cluster
 from ocs_ci.helpers.disruption_helpers import delete_resource_multiple_times
-
+from ocs_ci.framework import config
+from ocs_ci.utility.utils import (
+    convert_device_size,
+    get_pytest_fixture_value,
+    sum_of_two_storage_sizes,
+)
+from ocs_ci.ocs import defaults
 
 logger = logging.getLogger(__name__)
 
@@ -62,19 +73,31 @@ class TestResizeOSD(ManageTest):
     """
 
     @pytest.fixture(autouse=True)
-    def setup(self, create_pvcs_and_pods):
+    def setup(self, request, create_pvcs_and_pods):
         """
         Init all the data for the resize osd test
 
         """
-        check_resize_osd_pre_conditions()
+        self.old_storage_size = get_storage_size()
+        size_to_increase = (
+            get_pytest_fixture_value(request, "size_to_increase")
+            or self.old_storage_size
+        )
+        logger.info(
+            f"old storage size = {self.old_storage_size}, size to increase = {size_to_increase}"
+        )
+        self.new_storage_size = sum_of_two_storage_sizes(
+            self.old_storage_size, size_to_increase
+        )
+        logger.info(
+            f"The new expected storage size for the storage cluster is {self.new_storage_size}"
+        )
+        check_resize_osd_pre_conditions(self.new_storage_size)
         self.create_pvcs_and_pods = create_pvcs_and_pods
 
         self.old_osd_pods = get_osd_pods()
-        self.old_storage_size = get_storage_size()
         self.old_osd_pvcs = get_deviceset_pvcs()
         self.old_osd_pvs = get_deviceset_pvs()
-        self.new_storage_size = None
 
         self.pod_file_name = "fio_test"
         self.sanity_helpers = Sanity()
@@ -148,6 +171,10 @@ def verification_steps_post_resize_osd(self):
         )
         logger.info("Verify the md5sum of the pods for integrity check")
         verify_md5sum_on_pod_files(self.pods_for_integrity_check, self.pod_file_name)
+        # Verify OSDs are encrypted.
+        if config.ENV_DATA.get("encryption_at_rest"):
+            osd_encryption_verification()
+
         check_ceph_health_after_resize_osd()
 
         logger.info("Try to create more resources and run IO")
@@ -183,36 +210,89 @@ def test_resize_osd_with_node_restart(self, nodes):
         logger.info(f"Restart the worker node: {wnode.name}")
         if is_vsphere_ipi_cluster():
             nodes.restart_nodes(nodes=[wnode], wait=False)
-            wait_for_nodes_status(node_names=[wnode], timeout=300)
+            wait_for_nodes_status(node_names=[wnode.name], timeout=300)
         else:
             nodes.restart_nodes(nodes=[wnode], wait=True)
 
         self.verification_steps_post_resize_osd()
 
     @tier4c
     @pytest.mark.parametrize(
-        argnames=["resource_name", "num_of_iterations"],
-        argvalues=[
+        "resource_name, num_of_iterations, size_to_increase",
+        [
             pytest.param(
-                *[OSD, 3],
+                OSD,
+                3,
+                f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi",
                 marks=pytest.mark.polarion_id("OCS-5781"),
             ),
             pytest.param(
-                *[ROOK_OPERATOR, 3],
+                ROOK_OPERATOR,
+                3,
+                f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi",
                 marks=pytest.mark.polarion_id("OCS-5782"),
             ),
             pytest.param(
-                *[MON_DAEMON, 5],
+                MON_DAEMON,
+                5,
+                f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi",
                 marks=pytest.mark.polarion_id("OCS-5783"),
             ),
         ],
     )
-    def test_resize_osd_with_resource_delete(self, resource_name, num_of_iterations):
+    def test_resize_osd_with_resource_delete(
+        self, resource_name, num_of_iterations, size_to_increase
+    ):
         """
         Test resize OSD when one of the resources got deleted in the middle of the process
 
         """
         self.prepare_data_before_resize_osd()
-        self.new_storage_size = basic_resize_osd(self.old_storage_size)
+        resize_osd(self.new_storage_size)
         delete_resource_multiple_times(resource_name, num_of_iterations)
         self.verification_steps_post_resize_osd()
+
+    @tier4b
+    @polarion_id("OCS-5785")
+    def test_resize_osd_when_capacity_near_full(
+        self, benchmark_workload_storageutilization
+    ):
+        """
+        Test resize OSD when the cluster capacity is near full
+
+        """
+        target_percentage = 75
+        logger.info(
+            f"Fill up the cluster to {target_percentage}% of it's storage capacity"
+        )
+        benchmark_workload_storageutilization(target_percentage)
+        self.prepare_data_before_resize_osd()
+        resize_osd(self.new_storage_size)
+        self.verification_steps_post_resize_osd()
+
+    @tier4a
+    @pytest.mark.last
+    @pytest.mark.parametrize(
+        argnames=["size_to_increase"],
+        argvalues=[
+            pytest.param(*["2Ti"], marks=pytest.mark.polarion_id("OCS-5786")),
+        ],
+    )
+    def test_resize_osd_for_large_diff(self, size_to_increase):
+        """
+        Test resize osd for large differences. The test will increase the osd size to 4Ti.
+        If the current OSD size is less than 1024Gi, we will skip the test, as the purpose of the test
+        is to check resizing the osd for large differences.
+
+        """
+        logger.info(f"The current osd size is {self.old_storage_size}")
+        current_osd_size_in_gb = convert_device_size(self.old_storage_size, "GB", 1024)
+        max_osd_size_in_gb = 1024
+        if current_osd_size_in_gb > max_osd_size_in_gb:
+            pytest.skip(
+                f"The test will not run when the osd size is greater than {max_osd_size_in_gb}Gi"
+            )
+
+        self.prepare_data_before_resize_osd()
+        resize_osd(self.new_storage_size)
+        self.verification_steps_post_resize_osd()