diff --git a/ocs_ci/helpers/osd_resize.py b/ocs_ci/helpers/osd_resize.py index 6de83dbe995..5c0316d3a45 100644 --- a/ocs_ci/helpers/osd_resize.py +++ b/ocs_ci/helpers/osd_resize.py @@ -31,6 +31,8 @@ MAX_RESIZE_OSD, AWS_MAX_RESIZE_OSD_COUNT, AWS_PLATFORM, + MAX_TOTAL_CLUSTER_CAPACITY, + MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY, ) @@ -325,7 +327,7 @@ def check_ceph_health_after_resize_osd( ), "Data re-balance failed to complete" -def check_resize_osd_pre_conditions(): +def check_resize_osd_pre_conditions(expected_storage_size): """ Check the resize osd pre-conditions: 1. Check that the current storage size is less than the osd max size @@ -333,16 +335,30 @@ def check_resize_osd_pre_conditions(): If the conditions are not met, the test will be skipped. + Args: + expected_storage_size (str): The expected storage size for the storage cluster + """ - current_storage_size = get_storage_size() - current_storage_size_in_gb = convert_device_size(current_storage_size, "GB", 1024) + expected_storage_size_in_gb = convert_device_size(expected_storage_size, "GB", 1024) max_storage_size_in_gb = convert_device_size(MAX_RESIZE_OSD, "GB", 1024) - if current_storage_size_in_gb >= max_storage_size_in_gb: + if expected_storage_size_in_gb > max_storage_size_in_gb: pytest.skip( - f"The current storage size {current_storage_size} is greater or equal to the " + f"The expected storage size {expected_storage_size} is greater than the " f"max resize osd {MAX_RESIZE_OSD}" ) + if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM: + max_cluster_capacity = MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY + else: + max_cluster_capacity = MAX_TOTAL_CLUSTER_CAPACITY + max_cluster_capacity_in_gb = convert_device_size(max_cluster_capacity, "GB", 1024) + expected_cluster_capacity_in_gb = expected_storage_size_in_gb * len(get_osd_pods()) + if expected_cluster_capacity_in_gb > max_cluster_capacity_in_gb: + pytest.skip( + f"The expected cluster capacity {expected_cluster_capacity_in_gb}Gi is greater than the " + f"max cluster capacity {max_cluster_capacity}" + ) + config.RUN["resize_osd_count"] = config.RUN.get("resize_osd_count", 0) logger.info(f"resize osd count = {config.RUN['resize_osd_count']}") if ( diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index e0e9c121691..ed110225f83 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -2810,6 +2810,9 @@ # Resize osd MAX_RESIZE_OSD = "8Ti" AWS_MAX_RESIZE_OSD_COUNT = 1 +# The max total cluster capacity, including all OSDs +MAX_TOTAL_CLUSTER_CAPACITY = "12Ti" +MAX_IBMCLOUD_TOTAL_CLUSTER_CAPACITY = "24Ti" # CCOCTL CCOCTL_LOG_FILE = "ccoctl-service-id.log" diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py index 9061159fcdd..ca5a2c8f96b 100644 --- a/ocs_ci/utility/utils.py +++ b/ocs_ci/utility/utils.py @@ -4916,3 +4916,43 @@ def get_latest_release_version(): return exec_cmd(cmd, shell=True).stdout.decode("utf-8").strip() except CommandFailed: return + + +def sum_of_two_storage_sizes(storage_size1, storage_size2, convert_size=1024): + """ + Calculate the sum of two storage sizes given as strings. + Valid units: "Mi", "Gi", "Ti", "MB", "GB", "TB". + + Args: + storage_size1 (str): The first storage size, e.g., "800Mi", "100Gi", "2Ti". + storage_size2 (str): The second storage size, e.g., "700Mi", "500Gi", "300Gi". + convert_size (int): Set convert by 1000 or 1024. The default value is 1024. + + Returns: + str: The sum of the two storage sizes as a string, e.g., "1500Mi", "600Gi", "2300Gi". + + Raises: + ValueError: If the units of the storage sizes are not match the Valid units + + """ + valid_units = {"Mi", "Gi", "Ti", "MB", "GB", "TB"} + unit1 = storage_size1[-2:] + unit2 = storage_size2[-2:] + if unit1 not in valid_units or unit2 not in valid_units: + raise ValueError(f"Storage sizes must have valid units: {valid_units}") + + storage_size1 = storage_size1.replace("B", "i") + storage_size2 = storage_size2.replace("B", "i") + + if "Mi" in f"{storage_size1}{storage_size2}": + unit, units_to_convert = "Mi", "MB" + elif "Gi" in f"{storage_size1}{storage_size2}": + unit, units_to_convert = "Gi", "GB" + else: + unit, units_to_convert = "Ti", "TB" + + size1 = convert_device_size(storage_size1, units_to_convert, convert_size) + size2 = convert_device_size(storage_size2, units_to_convert, convert_size) + size = size1 + size2 + new_storage_size = f"{size}{unit}" + return new_storage_size diff --git a/tests/conftest.py b/tests/conftest.py index 31ec0501ec1..ce8cbf64a7d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ from ocs_ci.ocs import constants, defaults, fio_artefacts, node, ocp, platform_nodes from ocs_ci.ocs.acm.acm import login_to_acm from ocs_ci.ocs.awscli_pod import create_awscli_pod, awscli_pod_cleanup +from ocs_ci.ocs.benchmark_operator_fio import get_file_size, BenchmarkOperatorFIO from ocs_ci.ocs.bucket_utils import ( craft_s3_command, put_bucket_policy, @@ -7660,3 +7661,53 @@ def update_current_active_test_marks_global(request): """ marks = [mark.name for mark in request.node.iter_markers()] ocs_ci.framework.pytest_customization.marks.current_test_marks = marks + + +@pytest.fixture(scope="function") +def benchmark_workload_storageutilization(request): + """ + This fixture is for cluster storage utilization using the benchmark operator. + + """ + benchmark_obj = None + + def factory( + target_percentage, + jobs="read", + read_runtime=30, + bs="4096KiB", + storageclass=constants.DEFAULT_STORAGECLASS_RBD, + timeout_completed=2400, + ): + """ + Setup of benchmark fio + + Args: + target_percentage (int): The number of percentage to fill up the cluster + jobs (str): fio job types to run, for example the readwrite option + read_runtime (int): Amount of time in seconds to run read workloads + bs (str): the Block size that need to used for the prefill + storageclass (str): StorageClass to use for PVC per server pod + timeout_completed (int): timeout client pod move to completed state + + """ + nonlocal benchmark_obj + + size = get_file_size(target_percentage) + benchmark_obj = BenchmarkOperatorFIO() + benchmark_obj.setup_benchmark_fio( + total_size=size, + jobs=jobs, + read_runtime=read_runtime, + bs=bs, + storageclass=storageclass, + timeout_completed=timeout_completed, + ) + benchmark_obj.run_fio_benchmark_operator(is_completed=True) + + def finalizer(): + if benchmark_obj is not None: + benchmark_obj.cleanup() + + request.addfinalizer(finalizer) + return factory diff --git a/tests/functional/z_cluster/cluster_expansion/test_resize_osd.py b/tests/functional/z_cluster/cluster_expansion/test_resize_osd.py index 77ad0ce57a1..70d34897797 100644 --- a/tests/functional/z_cluster/cluster_expansion/test_resize_osd.py +++ b/tests/functional/z_cluster/cluster_expansion/test_resize_osd.py @@ -20,6 +20,7 @@ tier1, tier4b, tier4c, + tier4a, ) from ocs_ci.ocs.constants import VOLUME_MODE_BLOCK, OSD, ROOK_OPERATOR, MON_DAEMON from ocs_ci.helpers.osd_resize import ( @@ -35,12 +36,22 @@ verify_md5sum_on_pod_files, ) from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs, get_deviceset_pvs -from ocs_ci.ocs.resources.storage_cluster import get_storage_size +from ocs_ci.ocs.resources.storage_cluster import ( + get_storage_size, + osd_encryption_verification, + resize_osd, +) from ocs_ci.helpers.sanity_helpers import Sanity from ocs_ci.ocs.node import get_nodes, wait_for_nodes_status from ocs_ci.ocs.cluster import is_vsphere_ipi_cluster from ocs_ci.helpers.disruption_helpers import delete_resource_multiple_times - +from ocs_ci.framework import config +from ocs_ci.utility.utils import ( + convert_device_size, + get_pytest_fixture_value, + sum_of_two_storage_sizes, +) +from ocs_ci.ocs import defaults logger = logging.getLogger(__name__) @@ -62,19 +73,31 @@ class TestResizeOSD(ManageTest): """ @pytest.fixture(autouse=True) - def setup(self, create_pvcs_and_pods): + def setup(self, request, create_pvcs_and_pods): """ Init all the data for the resize osd test """ - check_resize_osd_pre_conditions() + self.old_storage_size = get_storage_size() + size_to_increase = ( + get_pytest_fixture_value(request, "size_to_increase") + or self.old_storage_size + ) + logger.info( + f"old storage size = {self.old_storage_size}, size to increase = {size_to_increase}" + ) + self.new_storage_size = sum_of_two_storage_sizes( + self.old_storage_size, size_to_increase + ) + logger.info( + f"The new expected storage size for the storage cluster is {self.new_storage_size}" + ) + check_resize_osd_pre_conditions(self.new_storage_size) self.create_pvcs_and_pods = create_pvcs_and_pods self.old_osd_pods = get_osd_pods() - self.old_storage_size = get_storage_size() self.old_osd_pvcs = get_deviceset_pvcs() self.old_osd_pvs = get_deviceset_pvs() - self.new_storage_size = None self.pod_file_name = "fio_test" self.sanity_helpers = Sanity() @@ -148,6 +171,10 @@ def verification_steps_post_resize_osd(self): ) logger.info("Verify the md5sum of the pods for integrity check") verify_md5sum_on_pod_files(self.pods_for_integrity_check, self.pod_file_name) + # Verify OSDs are encrypted. + if config.ENV_DATA.get("encryption_at_rest"): + osd_encryption_verification() + check_ceph_health_after_resize_osd() logger.info("Try to create more resources and run IO") @@ -183,7 +210,7 @@ def test_resize_osd_with_node_restart(self, nodes): logger.info(f"Restart the worker node: {wnode.name}") if is_vsphere_ipi_cluster(): nodes.restart_nodes(nodes=[wnode], wait=False) - wait_for_nodes_status(node_names=[wnode], timeout=300) + wait_for_nodes_status(node_names=[wnode.name], timeout=300) else: nodes.restart_nodes(nodes=[wnode], wait=True) @@ -191,28 +218,81 @@ def test_resize_osd_with_node_restart(self, nodes): @tier4c @pytest.mark.parametrize( - argnames=["resource_name", "num_of_iterations"], - argvalues=[ + "resource_name, num_of_iterations, size_to_increase", + [ pytest.param( - *[OSD, 3], + OSD, + 3, + f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi", marks=pytest.mark.polarion_id("OCS-5781"), ), pytest.param( - *[ROOK_OPERATOR, 3], + ROOK_OPERATOR, + 3, + f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi", marks=pytest.mark.polarion_id("OCS-5782"), ), pytest.param( - *[MON_DAEMON, 5], + MON_DAEMON, + 5, + f"{config.ENV_DATA.get('device_size', defaults.DEVICE_SIZE)}Gi", marks=pytest.mark.polarion_id("OCS-5783"), ), ], ) - def test_resize_osd_with_resource_delete(self, resource_name, num_of_iterations): + def test_resize_osd_with_resource_delete( + self, resource_name, num_of_iterations, size_to_increase + ): """ Test resize OSD when one of the resources got deleted in the middle of the process """ self.prepare_data_before_resize_osd() - self.new_storage_size = basic_resize_osd(self.old_storage_size) + resize_osd(self.new_storage_size) delete_resource_multiple_times(resource_name, num_of_iterations) self.verification_steps_post_resize_osd() + + @tier4b + @polarion_id("OCS-5785") + def test_resize_osd_when_capacity_near_full( + self, benchmark_workload_storageutilization + ): + """ + Test resize OSD when the cluster capacity is near full + + """ + target_percentage = 75 + logger.info( + f"Fill up the cluster to {target_percentage}% of it's storage capacity" + ) + benchmark_workload_storageutilization(target_percentage) + self.prepare_data_before_resize_osd() + resize_osd(self.new_storage_size) + self.verification_steps_post_resize_osd() + + @tier4a + @pytest.mark.last + @pytest.mark.parametrize( + argnames=["size_to_increase"], + argvalues=[ + pytest.param(*["2Ti"], marks=pytest.mark.polarion_id("OCS-5786")), + ], + ) + def test_resize_osd_for_large_diff(self, size_to_increase): + """ + Test resize osd for large differences. The test will increase the osd size to 4Ti. + If the current OSD size is less than 1024Gi, we will skip the test, as the purpose of the test + is to check resizing the osd for large differences. + + """ + logger.info(f"The current osd size is {self.old_storage_size}") + current_osd_size_in_gb = convert_device_size(self.old_storage_size, "GB", 1024) + max_osd_size_in_gb = 1024 + if current_osd_size_in_gb > max_osd_size_in_gb: + pytest.skip( + f"The test will not run when the osd size is greater than {max_osd_size_in_gb}Gi" + ) + + self.prepare_data_before_resize_osd() + resize_osd(self.new_storage_size) + self.verification_steps_post_resize_osd()