Start working on the resize osd test

Signed-off-by: Itzhak Kave <[email protected]>

ocs_ci/ocs/cluster.py

@@ -3188,3 +3188,33 @@ def get_mon_quorum_ranks():
     for rank in list(out["quorum"]):
         mon_quorum_ranks[list(out["quorum_names"])[rank]] = rank
     return mon_quorum_ranks
+
+
+def ceph_verification_steps_post_resize_osd(
+    old_osd_pods, old_osd_pvcs, old_osd_pvs, new_osd_size
+):
+    old_osd_pod_names = [p.name for p in old_osd_pods]
+    pod.wait_for_pods_to_be_in_statuses(
+        expected_statuses=[constants.STATUS_TERMINATING],
+        pod_names=old_osd_pod_names,
+        timeout=300,
+        sleep=20,
+    )
+
+    ocp_pod = OCP(kind=constants.POD, namespace=config.ENV_DATA["cluster_namespace"])
+    ocp_pod.wait_for_resource(
+        condition=constants.STATUS_RUNNING,
+        selector=constants.OSD_APP_LABEL,
+        resource_count=len(old_osd_pods),
+        timeout=300,
+        sleep=20,
+    )
+
+    ocp_pvc = OCP(kind=constants.PVC, namespace=config.ENV_DATA["cluster_namespace"])
+    ocp_pvc.wait_for_resource(
+        timeout=180,
+        sleep=10,
+        condition=constants.STATUS_BOUND,
+        selector=constants.OSD_PVC_GENERIC_LABEL,
+        resource_count=len(old_osd_pvcs),
+    )

ocs_ci/ocs/resources/pod.py

@@ -3566,3 +3566,64 @@ def _check_if_pod_deleted(label, namespace):
         namespace=namespace,
     )
     sampler.wait_for_func_status(True)
+
+
+def calculate_md5sum_of_pod_files(pods_for_integrity_check, pod_file_name):
+    """
+    Calculate the md5sum of the pod files, and save it in the pod objects
+
+    Args:
+        pods_for_integrity_check (list): The list of the pod objects to calculate the md5sum
+        pod_file_name (str): The pod file name to save the md5sum
+
+    """
+    # Wait for IO to finish
+    logger.info("Wait for IO to finish on pods")
+    for pod_obj in pods_for_integrity_check:
+        pod_obj.get_fio_results()
+        logger.info(f"IO finished on pod {pod_obj.name}")
+        # Calculate md5sum
+        pod_file_name = (
+            pod_file_name
+            if (pod_obj.pvc.volume_mode == constants.VOLUME_MODE_FILESYSTEM)
+            else pod_obj.get_storage_path(storage_type="block")
+        )
+        logger.info(
+            f"Calculate the md5sum of the file {pod_file_name} in the pod {pod_obj.name}"
+        )
+        pod_obj.pvc.md5sum = cal_md5sum(
+            pod_obj,
+            pod_file_name,
+            pod_obj.pvc.volume_mode == constants.VOLUME_MODE_BLOCK,
+        )
+
+
+def verify_md5sum_on_pod_files(pods_for_integrity_check, pod_file_name):
+    """
+    Verify the md5sum of the pod files
+
+    Args:
+        pods_for_integrity_check (list): The list of the pod objects to verify the md5sum
+        pod_file_name (str): The pod file name to verify its md5sum
+
+    Raises:
+        AssertionError: If file doesn't exist or md5sum mismatch
+
+    """
+    for pod_obj in pods_for_integrity_check:
+        pod_file_name = (
+            pod_obj.get_storage_path(storage_type="block")
+            if (pod_obj.pvc.volume_mode == constants.VOLUME_MODE_BLOCK)
+            else pod_file_name
+        )
+        verify_data_integrity(
+            pod_obj,
+            pod_file_name,
+            pod_obj.pvc.md5sum,
+            pod_obj.pvc.volume_mode == constants.VOLUME_MODE_BLOCK,
+        )
+        logger.info(
+            f"Verified: md5sum of {pod_file_name} on pod {pod_obj.name} "
+            f"matches with the original md5sum"
+        )
+    logger.info("Data integrity check passed on all pods")

ocs_ci/ocs/resources/storage_cluster.py

@@ -2669,3 +2669,29 @@ def validate_serviceexport():
     assert mon_count == len(
         get_mon_pods()
     ), f"Mon serviceexport count mismatch {mon_count} != {len(get_mon_pods())}"
+
+
+def resize_osd(new_osd_size):
+    """
+    Resize the OSD(e.g., from 512 to 1024, 1024 to 2048, etc.)
+
+    Args:
+        new_osd_size (int): The new osd size in Gi
+
+    Returns:
+        bool: True in case if changes are applied. False otherwise
+
+    """
+    new_osd_size_in_gi = f"{new_osd_size}Gi"
+    sc = get_storage_cluster()
+    # Patch the OSD storage size
+    path = "/spec/storageDeviceSets/0/dataPVCTemplate/spec/resources/requests/storage"
+    params = (
+        f"""[{{ "op": "replace", "path": "{path}", "value": {new_osd_size_in_gi}}}]"""
+    )
+    res = sc.patch(
+        resource_name=sc.get()["items"][0]["metadata"]["name"],
+        params=params.strip("\n"),
+        format_type="json",
+    )
+    return res

tests/functional/z_cluster/cluster_expansion/test_resize_osd.py

@@ -0,0 +1,129 @@
+import random
+import pytest
+import logging
+
+from ocs_ci.framework.pytest_customization.marks import (
+    polarion_id,
+    skipif_aws_i3,
+    skipif_bm,
+    skipif_external_mode,
+    skipif_bmpsi,
+    skipif_ibm_power,
+    skipif_lso,
+    skipif_managed_service,
+    skipif_hci_provider_and_client,
+    brown_squad,
+)
+from ocs_ci.framework.testlib import (
+    ignore_leftovers,
+    ManageTest,
+    tier1,
+)
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.resources.pod import (
+    get_osd_pods,
+    calculate_md5sum_of_pod_files,
+    verify_md5sum_on_pod_files,
+)
+from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs, get_deviceset_pvs
+from ocs_ci.ocs.resources.storage_cluster import resize_osd, get_osd_size
+from ocs_ci.helpers.sanity_helpers import Sanity
+
+
+logger = logging.getLogger(__name__)
+
+
+@brown_squad
+@ignore_leftovers
+@polarion_id("OCS-1191")
+@skipif_managed_service
+@skipif_aws_i3
+@skipif_bm
+@skipif_bmpsi
+@skipif_lso
+@skipif_external_mode
+@skipif_ibm_power
+@skipif_managed_service
+@skipif_hci_provider_and_client
+class TestResizeOSD(ManageTest):
+    """
+    Automates the resize OSD procedure
+    """
+
+    @pytest.fixture(autouse=True)
+    def setup(
+        self,
+        create_pvcs_and_pods,
+    ):
+        self.old_osd_pods = get_osd_pods()
+        self.old_osd_size = get_osd_size()
+        self.old_osd_pvcs = get_deviceset_pvcs()
+        self.old_osd_pvs = get_deviceset_pvs()
+
+        self.pod_file_name = "fio_test"
+        self.sanity_helpers = Sanity()
+        pvc_size = random.randint(3, 7)
+        self.pvcs1, self.pods_for_integrity_check = create_pvcs_and_pods(
+            pvc_size=pvc_size, num_of_rbd_pvc=6, num_of_cephfs_pvc=6
+        )
+        pvc_size = random.randint(3, 8)
+        self.pvcs2, self.pods_for_run_io = create_pvcs_and_pods(
+            pvc_size=pvc_size, num_of_rbd_pvc=5, num_of_cephfs_pvc=5
+        )
+
+    def run_io_on_pods(self, pods, size="1G", runtime=30):
+        """
+        Run IO on the pods
+
+        Args:
+            pods (list): The list of pods for running the IO
+            size (str): Size in MB or Gi, e.g. '200M'. Default value is '1G'
+            runtime (int): The number of seconds IO should run for
+
+        """
+        logger.info("Starting IO on all pods")
+        for pod_obj in pods:
+            storage_type = (
+                "block"
+                if pod_obj.pvc.volume_mode == constants.VOLUME_MODE_BLOCK
+                else "fs"
+            )
+            rate = f"{random.randint(1, 5)}M"
+            pod_obj.run_io(
+                storage_type=storage_type,
+                size=size,
+                runtime=runtime,
+                rate=rate,
+                fio_filename=self.pod_file_name,
+                end_fsync=1,
+            )
+            logger.info(f"IO started on pod {pod_obj.name}")
+        logger.info("Started IO on all pods")
+
+    def prepare_data_before_resize_osd(self):
+        logger.info("Run IO on the pods for integrity check")
+        self.run_io_on_pods(self.pods_for_integrity_check)
+        logger.info("Calculate the md5sum of the pods for integrity check")
+        calculate_md5sum_of_pod_files(self.pods_for_integrity_check, self.pod_file_name)
+        runtime = 180
+        logger.info(f"Run IO on the pods in the test background for {runtime} seconds")
+        self.run_io_on_pods(self.pods_for_run_io, size="2G", runtime=runtime)
+
+    def verification_steps_post_resize_osd(self):
+        logger.info("Verify the md5sum of the pods for integrity check")
+        verify_md5sum_on_pod_files(self.pods_for_integrity_check, self.pod_file_name)
+        self.sanity_helpers.health_check()
+
+    @tier1
+    def test_resize_osd(self):
+        """
+        Test resize OSD
+        """
+        self.prepare_data_before_resize_osd()
+
+        logger.info(f"The current osd size is {self.old_osd_size}")
+        new_size = self.old_osd_size
+        logger.info(f"Increase the osd size to {new_size}")
+        resize_osd(self.old_osd_size)
+
+        self.verification_steps_post_resize_osd()