From e253d83e66a0052d02ea9fb7f2667c17a20b1a7a Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Wed, 20 Sep 2023 11:36:14 +0530 Subject: [PATCH 1/5] System test for testing log based bucket replication deletion sync feature Signed-off-by: Mahesh Shetty --- ocs_ci/ocs/bucket_utils.py | 65 ++++++++- ocs_ci/ocs/constants.py | 1 + ocs_ci/ocs/resources/mockup_bucket_logger.py | 28 ++++ ocs_ci/ocs/resources/pod.py | 10 ++ tests/conftest.py | 79 ++++++++++ tests/cross_functional/conftest.py | 69 ++++++--- .../test_mcg_replication_with_disruptions.py | 138 ++++++++++++++++++ 7 files changed, 366 insertions(+), 24 deletions(-) diff --git a/ocs_ci/ocs/bucket_utils.py b/ocs_ci/ocs/bucket_utils.py index 7a3a2d6825a..f7ffe398fe5 100644 --- a/ocs_ci/ocs/bucket_utils.py +++ b/ocs_ci/ocs/bucket_utils.py @@ -2040,6 +2040,25 @@ def patch_replication_policy_to_bucketclass( ).patch(params=json.dumps(replication_policy_patch_dict), format_type="merge") +def remove_replication_policy(bucket_name): + """ + Remove replication policy for a bucket + + Args: + bucket_name(str): Name of the bucket + + """ + replication_policy_patch_dict = { + "spec": {"additionalConfig": {{"replicationPolicy": ""}}} + } + + OCP( + kind="obc", + namespace=config.ENV_DATA["cluster_namespace"], + resource_name=bucket_name, + ).patch(params=json.dumps(replication_policy_patch_dict), format_type="merge") + + def random_object_round_trip_verification( io_pod, bucket_name, @@ -2323,7 +2342,6 @@ def sample_if_objects_expired(mcg_obj, bucket_name, prefix="", timeout=600, slee assert sampler.wait_for_func_status(result=True), f"{message} are not expired" logger.info(f"{message} are expired") - def delete_all_noobaa_buckets(mcg_obj, request): """ Deletes all the buckets in noobaa and restores the first.bucket after the current test @@ -2604,3 +2622,48 @@ def bulk_s3_put_bucket_lifecycle_config(mcg_obj, buckets, lifecycle_config): Bucket=bucket.name, LifecycleConfiguration=lifecycle_config ) logger.info("Applied lifecyle rule on all the buckets") + + +def upload_test_objects_to_source_and_wait_for_replication( + mcg_obj, source_bucket, target_bucket, mockup_logger, timeout +): + """ + Upload a set of objects to the source bucket, logs the operations and wait for the replication to complete. + + """ + logger.info("Uploading test objects and waiting for replication to complete") + mockup_logger.upload_test_objs_and_log(source_bucket.name) + + logger.info( + "Resetting the noobaa-core pod to trigger the replication background worker" + ) + + assert compare_bucket_object_list( + mcg_obj, + source_bucket.name, + target_bucket.name, + timeout=timeout, + ), f"Standard replication failed to complete in {timeout} seconds" + + +def delete_objects_from_source_and_wait_for_deletion_sync( + mcg_obj, source_bucket, target_bucket, mockup_logger, timeout +): + """ + Delete all objects from the source bucket,logs the operations and wait for the deletion sync to complete. + + """ + logger.info("Deleting source objects and waiting for deletion sync with target") + mockup_logger.delete_all_objects_and_log(source_bucket.name) + + logger.info( + "Resetting the noobaa-core pod to trigger the replication background worker" + ) + + assert compare_bucket_object_list( + mcg_obj, + source_bucket.name, + target_bucket.name, + timeout=timeout, + ), f"Deletion sync failed to complete in {timeout} seconds" + diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 22b5ef74c02..9737f0d7277 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1109,6 +1109,7 @@ FUSIONAAS_PLATFORM, ] BAREMETAL_PLATFORMS = [BAREMETAL_PLATFORM, BAREMETALPSI_PLATFORM] +DEFAULT_AWS_REGION = "us-east-2" HCI_PROVIDER_CLIENT_PLATFORMS = [ HCI_BAREMETAL, diff --git a/ocs_ci/ocs/resources/mockup_bucket_logger.py b/ocs_ci/ocs/resources/mockup_bucket_logger.py index ddc532dc135..8c535493a23 100644 --- a/ocs_ci/ocs/resources/mockup_bucket_logger.py +++ b/ocs_ci/ocs/resources/mockup_bucket_logger.py @@ -56,6 +56,10 @@ def __init__(self, awscli_pod, mcg_obj, bucket_factory, platform, region): f"ls -A1 {constants.AWSCLI_TEST_OBJ_DIR}" ).split(" ") + @property + def standard_test_obj_list(self): + return self._standard_test_obj_list + def upload_test_objs_and_log(self, bucket_name): """ Uploads files from files_dir to the MCG bucket and write matching @@ -97,6 +101,30 @@ def upload_arbitrary_object_and_log(self, bucket_name): self._upload_mockup_logs(bucket_name, [obj_name], "PUT") + def delete_objs_and_log(self, bucket_name, objs): + """ + Delete list of objects from the MCG bucket and write + matching mockup logs + + Args: + bucket_name(str): Name of the MCG bucket + objs(list): List of the objects to delete + + """ + logger.info(f"Deleting the {objs} from the bucket") + obj_list = list_objects_from_bucket( + self.awscli_pod, + f"s3://{bucket_name}", + s3_obj=self.mcg_obj, + ) + if set(objs).issubset(set(obj_list)): + for i in range(len(objs)): + s3cmd = craft_s3_command( + f"rm s3://{bucket_name}/{objs[i]}", self.mcg_obj + ) + self.awscli_pod.exec_cmd_on_pod(s3cmd) + self._upload_mockup_logs(bucket_name, objs, "DELETE") + def delete_all_objects_and_log(self, bucket_name): """ Deletes all objects from the MCG bucket and write matching mockup logs diff --git a/ocs_ci/ocs/resources/pod.py b/ocs_ci/ocs/resources/pod.py index c2828de5856..4866d9364eb 100644 --- a/ocs_ci/ocs/resources/pod.py +++ b/ocs_ci/ocs/resources/pod.py @@ -990,6 +990,16 @@ def get_noobaa_endpoint_pods(): return noobaa_endpoint_pods +def get_noobaa_db_pod(): + + noobaa_db = get_pods_having_label( + label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE, + namespace=config.ENV_DATA["cluster_namespace"], + ) + nb_db_pods = [Pod(**pod) for pod in noobaa_db] + return nb_db_pods[0] + + def get_odf_operator_controller_manager( ocs_label=constants.ODF_OPERATOR_CONTROL_MANAGER_LABEL, namespace=None ): diff --git a/tests/conftest.py b/tests/conftest.py index fe45b516215..96545fdf711 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,8 @@ from ocs_ci.ocs.ocp import OCP from ocs_ci.ocs.resources import pvc from ocs_ci.ocs.resources.bucket_policy import gen_bucket_policy +from ocs_ci.ocs.resources.mcg_replication_policy import LogBasedReplicationPolicy +from ocs_ci.ocs.resources.mockup_bucket_logger import MockupBucketLogger from ocs_ci.ocs.scale_lib import FioPodScale from ocs_ci.ocs.utils import ( setup_ceph_toolbox, @@ -7439,3 +7441,80 @@ def finalizer(): request.addfinalizer(finalizer) return factory + +@pytest.fixture(scope="class") +def reduce_replication_delay_setup(add_env_vars_to_noobaa_core_class): + """ + A fixture to reduce the replication delay to one minute. + + Args: + new_delay_in_miliseconds (function): A function to add env vars to the noobaa-core pod + + """ + log.warning("Reducing replication delay") + + def factory(new_delay_in_miliseconds=60 * 1000): + new_env_var_tuples = [ + (constants.BUCKET_REPLICATOR_DELAY_PARAM, new_delay_in_miliseconds), + (constants.BUCKET_LOG_REPLICATOR_DELAY_PARAM, new_delay_in_miliseconds), + ] + add_env_vars_to_noobaa_core_class(new_env_var_tuples) + + return factory + + +@pytest.fixture() +def log_based_replication_setup( + awscli_pod_session, mcg_obj_session, bucket_factory, reduce_replication_delay_setup +): + """ + A fixture to set up standard log-based replication with deletion sync. + + Args: + awscli_pod_session(Pod): A pod running the AWS CLI + mcg_obj_session(MCG): An MCG object + bucket_factory: A bucket factory fixture + + Returns: + MockupBucketLogger: A MockupBucketLogger object + Bucket: The source bucket + Bucket: The target bucket + """ + + def factory(bucketclass_dict=None): + log.info("Starting log-based replication setup") + if bucketclass_dict is None: + bucketclass_dict = { + "interface": "OC", + "namespace_policy_dict": { + "type": "Single", + "namespacestore_dict": { + constants.AWS_PLATFORM: [(1, constants.DEFAULT_AWS_REGION)] + }, + }, + } + target_bucket = bucket_factory(bucketclass=bucketclass_dict)[0] + + mockup_logger = MockupBucketLogger( + awscli_pod=awscli_pod_session, + mcg_obj=mcg_obj_session, + bucket_factory=bucket_factory, + platform=constants.AWS_PLATFORM, + region=constants.DEFAULT_AWS_REGION, + ) + replication_policy = LogBasedReplicationPolicy( + destination_bucket=target_bucket.name, + sync_deletions=True, + logs_bucket=mockup_logger.logs_bucket_uls_name, + ) + + source_bucket = bucket_factory( + 1, bucketclass=bucketclass_dict, replication_policy=replication_policy + )[0] + + log.info("log-based replication setup complete") + + return mockup_logger, source_bucket, target_bucket + + return factory + diff --git a/tests/cross_functional/conftest.py b/tests/cross_functional/conftest.py index 441f1559d37..36c65dd6f8d 100644 --- a/tests/cross_functional/conftest.py +++ b/tests/cross_functional/conftest.py @@ -319,31 +319,10 @@ def finalizer(): @pytest.fixture() -def noobaa_db_backup_and_recovery(request, snapshot_factory): - """ - Verify noobaa backup and recovery - - 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC - 2. Scale down the statefulset noobaa-db - 3. Get the yaml of the current PVC, db-noobaa-db-0 and - change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC - 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. - The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ - 5. Edit again restore PV and remove the claimRef section. - The volume will transition to Available. - 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. - 7. Scale up the stateful set again and the pod should be running - - """ +def noobaa_db_backup(request, snapshot_factory): restore_pvc_objs = [] - def factory(snapshot_factory=snapshot_factory): - # Get noobaa pods before execution - noobaa_pods = pod.get_noobaa_pods() - - # Get noobaa PVC before execution - noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) - noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get("volumeName") + def factory(noobaa_pvc_obj): # Take snapshot db-noobaa-db-0 PVC logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") @@ -381,6 +360,15 @@ def factory(snapshot_factory=snapshot_factory): f"Succeesfuly created PVC {restore_pvc_obj.name} " f"from snapshot {snap_obj.name}" ) + return restore_pvc_objs, snap_obj + + return factory + + +@pytest.fixture() +def noobaa_db_recovery_from_backup(request): + def factory(snap_obj, noobaa_pvc_obj, noobaa_pods): + noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get("volumeName") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( @@ -474,6 +462,41 @@ def factory(snapshot_factory=snapshot_factory): "Changed the parameter persistentVolumeReclaimPolicy to Delete again" ) + return factory + + +@pytest.fixture() +def noobaa_db_backup_and_recovery( + request, snapshot_factory, noobaa_db_backup, noobaa_db_recovery_from_backup +): + """ + Verify noobaa backup and recovery + + 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC + 2. Scale down the statefulset noobaa-db + 3. Get the yaml of the current PVC, db-noobaa-db-0 and + change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC + 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. + The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ + 5. Edit again restore PV and remove the claimRef section. + The volume will transition to Available. + 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. + 7. Scale up the stateful set again and the pod should be running + + """ + restore_pvc_objs = [] + + def factory(snapshot_factory=snapshot_factory): + global restore_pvc_objs + # Get noobaa pods before execution + noobaa_pods = pod.get_noobaa_pods() + + # Get noobaa PVC before execution + noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) + + restore_pvc_objs, snap_obj = noobaa_db_backup(noobaa_pvc_obj) + noobaa_db_recovery_from_backup(snap_obj, noobaa_pvc_obj, noobaa_pods) + def finalizer(): # Get the statefulset replica count sst_obj = OCP( diff --git a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py index 3b97899cae7..094fe95e776 100644 --- a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py +++ b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py @@ -1,6 +1,9 @@ import logging import pytest +import random +import time +from concurrent.futures import ThreadPoolExecutor from ocs_ci.framework.testlib import ( E2ETest, @@ -21,13 +24,21 @@ compare_bucket_object_list, patch_replication_policy_to_bucket, write_random_test_objects_to_bucket, + upload_test_objects_to_source_and_wait_for_replication, + update_replication_policy, + remove_replication_policy, ) from ocs_ci.ocs import ocp +from ocs_ci.ocs.resources.pvc import get_pvc_objs from ocs_ci.ocs.utils import get_pod_name_by_pattern from ocs_ci.ocs.resources.pod import ( delete_pods, wait_for_pods_to_be_running, get_rgw_pods, + get_noobaa_db_pod, + get_noobaa_core_pod, + wait_for_storage_pods, + get_noobaa_pods, ) from ocs_ci.utility.retry import retry from ocs_ci.ocs.exceptions import CommandFailed, ResourceWrongStatusException @@ -228,3 +239,130 @@ def test_replication_with_disruptions( mcg_obj_session, source_bucket_name, target_bucket_name ) logger.info("Objects sync works even when the cluster is rebooted") + + +@system_test +@skipif_vsphere_ipi +class TestLogBasedReplicationWithDisruptions: + def test_log_based_replication_with_disruptions( + self, + mcg_obj_session, + log_based_replication_setup, + noobaa_db_backup, + noobaa_db_recovery_from_backup, + ): + """ + This is a system test flow to test log based bucket replication + deletion sync is not impacted due to some noobaa specific disruptions + like noobaa pod restarts, noobaa db backup & recovery etc + + 1. Setup log based bucket replication between the buckets + 2. Upload some objects and make sure replication works + 3. Keep deleting some objects from the source bucket and make sure + deletion sync works as expected through out. + 4. In another thread, restart the noobaa pods (db & core), make sure + deletion sync works for the step-3 deletion works as expected + 5. Now take backup of Noobaa db using PV backup method + 6. Remove the log based replication rules, perform some deletion in + source bucket. make sure deletion sync doesn't work + 7. Recover noobaa db from the backup taken in step-5 + 8. Now check if deletion sync works by deleting some objects from + source bucket. Note: Expectation is still unclear + 9. Now patch the bucket to remove complete replication policy and + make sure no replication - no deletion sync works + + """ + + mockup_logger, source_bucket, target_bucket = log_based_replication_setup() + + # upload test objects to the bucket and verify replication + upload_test_objects_to_source_and_wait_for_replication( + mcg_obj_session, + source_bucket, + target_bucket, + mockup_logger, + 600, + ) + + # Delete objects in the first set in a batch and perform noobaa pod + # restarts at the same time and make sure deletion sync works + + objs_in_bucket = mockup_logger.standard_test_obj_list + objs_to_delete = random.sample(objs_in_bucket, 3) + + from ocs_ci.utility.retry import retry + + @retry(Exception, tries=10, delay=5) + def delete_objs_in_batch(): + for obj in objs_to_delete: + mockup_logger.delete_objs_and_log(source_bucket.name, [obj]) + time.sleep(5) + logger.info(f"Successfully deleted these objects: {objs_to_delete}") + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(delete_objs_in_batch) + + # Restart noobaa pods + nb_core_pod = get_noobaa_core_pod() + nb_db_pod = get_noobaa_db_pod() + nb_core_pod.delete() + nb_db_pod.delete() + wait_for_pods_to_be_running(pod_names=[nb_core_pod.name, nb_db_pod.name]) + + # Wait for the object deletion worker in the BG to completion + future.result() + assert compare_bucket_object_list( + mcg_obj_session, + source_bucket.name, + target_bucket.name, + timeout=600, + ), f"Deletion sync failed to complete for the objects {objs_to_delete} deleted in the first bucket set" + + # Take noobaa db backup and remove deletion replication policy for the second bucket set + # Get noobaa pods before execution + noobaa_pods = get_noobaa_pods() + + # Get noobaa PVC before execution + noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) + + _, snap_obj = noobaa_db_backup(noobaa_pvc_obj) + + disable_deletion_sync = source_bucket.replication_policy + disable_deletion_sync["rules"][0]["sync_deletions"] = False + update_replication_policy(source_bucket.name, disable_deletion_sync) + logger.info("Deleting all the objects from the second bucket") + mockup_logger.delete_all_objects_and_log(source_bucket.name) + assert not compare_bucket_object_list( + mcg_obj_session, + source_bucket.name, + target_bucket.name, + timeout=600, + ), "Deletion sync was done but not expected" + + # Do noobaa db recovery and see if the deletion sync works now + noobaa_db_recovery_from_backup(snap_obj, noobaa_pvc_obj, noobaa_pods) + wait_for_storage_pods() + + assert compare_bucket_object_list( + mcg_obj_session, + source_bucket.name, + target_bucket.name, + timeout=600, + ), "Deletion sync was not done but expected" + + # Remove replication policy and upload some objects to the bucket + # make sure the replication itself doesn't take place + remove_replication_policy(source_bucket.name) + logger.info("Uploading test objects and waiting for replication to complete") + mockup_logger.upload_test_objs_and_log(source_bucket.name) + + logger.info( + "Resetting the noobaa-core pod to trigger the replication background worker" + ) + + assert not compare_bucket_object_list( + mcg_obj_session, + source_bucket.name, + target_bucket.name, + timeout=600, + ), f"Standard replication completed even though replication policy is removed" From ece09ce7451d6afd1be5fe33a616f79b95418952 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 26 Oct 2023 15:31:12 +0530 Subject: [PATCH 2/5] add newly added mcg background features entry critria setup Signed-off-by: Mahesh Shetty --- ocs_ci/ocs/resources/pod.py | 10 ---------- tests/conftest.py | 4 ++-- .../test_mcg_replication_with_disruptions.py | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ocs_ci/ocs/resources/pod.py b/ocs_ci/ocs/resources/pod.py index 4866d9364eb..c2828de5856 100644 --- a/ocs_ci/ocs/resources/pod.py +++ b/ocs_ci/ocs/resources/pod.py @@ -990,16 +990,6 @@ def get_noobaa_endpoint_pods(): return noobaa_endpoint_pods -def get_noobaa_db_pod(): - - noobaa_db = get_pods_having_label( - label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE, - namespace=config.ENV_DATA["cluster_namespace"], - ) - nb_db_pods = [Pod(**pod) for pod in noobaa_db] - return nb_db_pods[0] - - def get_odf_operator_controller_manager( ocs_label=constants.ODF_OPERATOR_CONTROL_MANAGER_LABEL, namespace=None ): diff --git a/tests/conftest.py b/tests/conftest.py index 96545fdf711..f2f1b18400a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,7 @@ from ocs_ci.ocs.ocp import OCP from ocs_ci.ocs.resources import pvc from ocs_ci.ocs.resources.bucket_policy import gen_bucket_policy -from ocs_ci.ocs.resources.mcg_replication_policy import LogBasedReplicationPolicy +from ocs_ci.ocs.resources.mcg_replication_policy import AwsLogBasedReplicationPolicy from ocs_ci.ocs.resources.mockup_bucket_logger import MockupBucketLogger from ocs_ci.ocs.scale_lib import FioPodScale from ocs_ci.ocs.utils import ( @@ -7502,7 +7502,7 @@ def factory(bucketclass_dict=None): platform=constants.AWS_PLATFORM, region=constants.DEFAULT_AWS_REGION, ) - replication_policy = LogBasedReplicationPolicy( + replication_policy = AwsLogBasedReplicationPolicy( destination_bucket=target_bucket.name, sync_deletions=True, logs_bucket=mockup_logger.logs_bucket_uls_name, diff --git a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py index 094fe95e776..3e79dc98d9a 100644 --- a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py +++ b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py @@ -242,6 +242,7 @@ def test_replication_with_disruptions( @system_test +@magenta_squad @skipif_vsphere_ipi class TestLogBasedReplicationWithDisruptions: def test_log_based_replication_with_disruptions( @@ -250,6 +251,8 @@ def test_log_based_replication_with_disruptions( log_based_replication_setup, noobaa_db_backup, noobaa_db_recovery_from_backup, + setup_mcg_bg_features, + validate_mcg_bg_features, ): """ This is a system test flow to test log based bucket replication @@ -273,6 +276,14 @@ def test_log_based_replication_with_disruptions( """ + # entry criteria setup + feature_setup_map = setup_mcg_bg_features( + num_of_buckets=5, + object_amount=5, + is_disruptive=True, + skip_any_features=["nsfs", "rgw kafka", "caching"], + ) + mockup_logger, source_bucket, target_bucket = log_based_replication_setup() # upload test objects to the bucket and verify replication @@ -366,3 +377,11 @@ def delete_objs_in_batch(): target_bucket.name, timeout=600, ), f"Standard replication completed even though replication policy is removed" + + validate_mcg_bg_features( + feature_setup_map, + run_in_bg=False, + skip_any_features=["nsfs", "rgw kafka", "caching"], + object_amount=5, + ) + logger.info("No issues seen with the MCG bg feature validation") From 5d969380b061fb0dec64409230e80840f0438802 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 30 Nov 2023 17:41:27 +0530 Subject: [PATCH 3/5] review changes Signed-off-by: Mahesh Shetty --- ocs_ci/ocs/bucket_utils.py | 29 +------------------ ocs_ci/ocs/constants.py | 1 + tests/conftest.py | 9 +++--- .../test_mcg_replication_with_disruptions.py | 29 +++++++++---------- 4 files changed, 21 insertions(+), 47 deletions(-) diff --git a/ocs_ci/ocs/bucket_utils.py b/ocs_ci/ocs/bucket_utils.py index f7ffe398fe5..6a992ebce0d 100644 --- a/ocs_ci/ocs/bucket_utils.py +++ b/ocs_ci/ocs/bucket_utils.py @@ -2040,25 +2040,6 @@ def patch_replication_policy_to_bucketclass( ).patch(params=json.dumps(replication_policy_patch_dict), format_type="merge") -def remove_replication_policy(bucket_name): - """ - Remove replication policy for a bucket - - Args: - bucket_name(str): Name of the bucket - - """ - replication_policy_patch_dict = { - "spec": {"additionalConfig": {{"replicationPolicy": ""}}} - } - - OCP( - kind="obc", - namespace=config.ENV_DATA["cluster_namespace"], - resource_name=bucket_name, - ).patch(params=json.dumps(replication_policy_patch_dict), format_type="merge") - - def random_object_round_trip_verification( io_pod, bucket_name, @@ -2342,6 +2323,7 @@ def sample_if_objects_expired(mcg_obj, bucket_name, prefix="", timeout=600, slee assert sampler.wait_for_func_status(result=True), f"{message} are not expired" logger.info(f"{message} are expired") + def delete_all_noobaa_buckets(mcg_obj, request): """ Deletes all the buckets in noobaa and restores the first.bucket after the current test @@ -2634,10 +2616,6 @@ def upload_test_objects_to_source_and_wait_for_replication( logger.info("Uploading test objects and waiting for replication to complete") mockup_logger.upload_test_objs_and_log(source_bucket.name) - logger.info( - "Resetting the noobaa-core pod to trigger the replication background worker" - ) - assert compare_bucket_object_list( mcg_obj, source_bucket.name, @@ -2656,14 +2634,9 @@ def delete_objects_from_source_and_wait_for_deletion_sync( logger.info("Deleting source objects and waiting for deletion sync with target") mockup_logger.delete_all_objects_and_log(source_bucket.name) - logger.info( - "Resetting the noobaa-core pod to trigger the replication background worker" - ) - assert compare_bucket_object_list( mcg_obj, source_bucket.name, target_bucket.name, timeout=timeout, ), f"Deletion sync failed to complete in {timeout} seconds" - diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 9737f0d7277..6b2e76237a6 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -288,6 +288,7 @@ DEFAULT_NOOBAA_BACKINGSTORE = "noobaa-default-backing-store" DEFAULT_NOOBAA_BUCKETCLASS = "noobaa-default-bucket-class" NOOBAA_RESOURCE_NAME = "noobaa" +NOOBAA_DB_PVC_NAME = "db-noobaa-db-pg-0" MIN_PV_BACKINGSTORE_SIZE_IN_GB = 17 JENKINS_BUILD = "jax-rs-build" JENKINS_BUILD_COMPLETE = "Complete" diff --git a/tests/conftest.py b/tests/conftest.py index f2f1b18400a..f67143b6494 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7352,7 +7352,6 @@ def scale_noobaa_resources(): log.info("Scaled noobaa pod resources") time.sleep(60) - @pytest.fixture(scope="function") def create_scale_pods_and_pvcs_using_kube_job_on_hci_clients(request): """ @@ -7442,7 +7441,8 @@ def finalizer(): request.addfinalizer(finalizer) return factory -@pytest.fixture(scope="class") + +@pytest.fixture() def reduce_replication_delay_setup(add_env_vars_to_noobaa_core_class): """ A fixture to reduce the replication delay to one minute. @@ -7464,7 +7464,7 @@ def factory(new_delay_in_miliseconds=60 * 1000): @pytest.fixture() -def log_based_replication_setup( +def aws_log_based_replication_setup( awscli_pod_session, mcg_obj_session, bucket_factory, reduce_replication_delay_setup ): """ @@ -7481,6 +7481,8 @@ def log_based_replication_setup( Bucket: The target bucket """ + reduce_replication_delay_setup() + def factory(bucketclass_dict=None): log.info("Starting log-based replication setup") if bucketclass_dict is None: @@ -7517,4 +7519,3 @@ def factory(bucketclass_dict=None): return mockup_logger, source_bucket, target_bucket return factory - diff --git a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py index 3e79dc98d9a..d9a55356460 100644 --- a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py +++ b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py @@ -5,6 +5,7 @@ import time from concurrent.futures import ThreadPoolExecutor +from ocs_ci.ocs import constants from ocs_ci.framework.testlib import ( E2ETest, skipif_ocs_version, @@ -26,7 +27,6 @@ write_random_test_objects_to_bucket, upload_test_objects_to_source_and_wait_for_replication, update_replication_policy, - remove_replication_policy, ) from ocs_ci.ocs import ocp from ocs_ci.ocs.resources.pvc import get_pvc_objs @@ -37,8 +37,8 @@ get_rgw_pods, get_noobaa_db_pod, get_noobaa_core_pod, - wait_for_storage_pods, get_noobaa_pods, + wait_for_noobaa_pods_running, ) from ocs_ci.utility.retry import retry from ocs_ci.ocs.exceptions import CommandFailed, ResourceWrongStatusException @@ -248,7 +248,7 @@ class TestLogBasedReplicationWithDisruptions: def test_log_based_replication_with_disruptions( self, mcg_obj_session, - log_based_replication_setup, + aws_log_based_replication_setup, noobaa_db_backup, noobaa_db_recovery_from_backup, setup_mcg_bg_features, @@ -281,10 +281,10 @@ def test_log_based_replication_with_disruptions( num_of_buckets=5, object_amount=5, is_disruptive=True, - skip_any_features=["nsfs", "rgw kafka", "caching"], + skip_any_features=["nsfs", "rgw kafka", "caching", "replication"], ) - mockup_logger, source_bucket, target_bucket = log_based_replication_setup() + mockup_logger, source_bucket, target_bucket = aws_log_based_replication_setup() # upload test objects to the bucket and verify replication upload_test_objects_to_source_and_wait_for_replication( @@ -334,7 +334,7 @@ def delete_objs_in_batch(): noobaa_pods = get_noobaa_pods() # Get noobaa PVC before execution - noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) + noobaa_pvc_obj = get_pvc_objs(pvc_names=[constants.NOOBAA_DB_PVC_NAME]) _, snap_obj = noobaa_db_backup(noobaa_pvc_obj) @@ -347,12 +347,12 @@ def delete_objs_in_batch(): mcg_obj_session, source_bucket.name, target_bucket.name, - timeout=600, + timeout=300, ), "Deletion sync was done but not expected" # Do noobaa db recovery and see if the deletion sync works now noobaa_db_recovery_from_backup(snap_obj, noobaa_pvc_obj, noobaa_pods) - wait_for_storage_pods() + wait_for_noobaa_pods_running(timeout=420) assert compare_bucket_object_list( mcg_obj_session, @@ -363,20 +363,19 @@ def delete_objs_in_batch(): # Remove replication policy and upload some objects to the bucket # make sure the replication itself doesn't take place - remove_replication_policy(source_bucket.name) + disable_replication = source_bucket.replication_policy + disable_replication["rules"] = [] + update_replication_policy(source_bucket.name, dict()) + logger.info("Uploading test objects and waiting for replication to complete") mockup_logger.upload_test_objs_and_log(source_bucket.name) - logger.info( - "Resetting the noobaa-core pod to trigger the replication background worker" - ) - assert not compare_bucket_object_list( mcg_obj_session, source_bucket.name, target_bucket.name, - timeout=600, - ), f"Standard replication completed even though replication policy is removed" + timeout=300, + ), "Standard replication completed even though replication policy is removed" validate_mcg_bg_features( feature_setup_map, From 16b7af214f7fadb8e25e120bea5e770f04dca379 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Mon, 4 Mar 2024 11:45:15 +0530 Subject: [PATCH 4/5] Add bugzilla marker & polarion id Signed-off-by: Mahesh Shetty --- tests/conftest.py | 1 + .../system_test/test_mcg_replication_with_disruptions.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index f67143b6494..1c8f1dbd30e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7352,6 +7352,7 @@ def scale_noobaa_resources(): log.info("Scaled noobaa pod resources") time.sleep(60) + @pytest.fixture(scope="function") def create_scale_pods_and_pvcs_using_kube_job_on_hci_clients(request): """ diff --git a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py index d9a55356460..20080162a4f 100644 --- a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py +++ b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py @@ -19,6 +19,8 @@ skipif_vsphere_ipi, magenta_squad, mcg, + bugzilla, + polarion_id, ) from ocs_ci.ocs.node import get_worker_nodes, get_node_objs from ocs_ci.ocs.bucket_utils import ( @@ -245,6 +247,8 @@ def test_replication_with_disruptions( @magenta_squad @skipif_vsphere_ipi class TestLogBasedReplicationWithDisruptions: + @polarion_id("OCS-5457") + @bugzilla("2266805") def test_log_based_replication_with_disruptions( self, mcg_obj_session, From f183987a924d2a25ef853142b3db3042c3e5b80d Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Wed, 13 Mar 2024 18:31:27 +0530 Subject: [PATCH 5/5] Review changes Signed-off-by: Mahesh Shetty --- tests/conftest.py | 1 + .../test_mcg_replication_with_disruptions.py | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1c8f1dbd30e..ddf0b954d0f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7480,6 +7480,7 @@ def aws_log_based_replication_setup( MockupBucketLogger: A MockupBucketLogger object Bucket: The source bucket Bucket: The target bucket + """ reduce_replication_delay_setup() diff --git a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py index 20080162a4f..10f244b5b99 100644 --- a/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py +++ b/tests/cross_functional/system_test/test_mcg_replication_with_disruptions.py @@ -247,6 +247,18 @@ def test_replication_with_disruptions( @magenta_squad @skipif_vsphere_ipi class TestLogBasedReplicationWithDisruptions: + @retry(Exception, tries=10, delay=5) + def delete_objs_in_batch(self, objs_to_delete, mockup_logger, source_bucket): + """ + This function deletes objects in a batch + """ + for obj in objs_to_delete: + mockup_logger.delete_objs_and_log(source_bucket.name, [obj]) + # adding momentary sleep just to slowdown the deletion + # process + time.sleep(5) + logger.info(f"Successfully deleted these objects: {objs_to_delete}") + @polarion_id("OCS-5457") @bugzilla("2266805") def test_log_based_replication_with_disruptions( @@ -279,7 +291,6 @@ def test_log_based_replication_with_disruptions( make sure no replication - no deletion sync works """ - # entry criteria setup feature_setup_map = setup_mcg_bg_features( num_of_buckets=5, @@ -305,17 +316,10 @@ def test_log_based_replication_with_disruptions( objs_in_bucket = mockup_logger.standard_test_obj_list objs_to_delete = random.sample(objs_in_bucket, 3) - from ocs_ci.utility.retry import retry - - @retry(Exception, tries=10, delay=5) - def delete_objs_in_batch(): - for obj in objs_to_delete: - mockup_logger.delete_objs_and_log(source_bucket.name, [obj]) - time.sleep(5) - logger.info(f"Successfully deleted these objects: {objs_to_delete}") - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(delete_objs_in_batch) + future = executor.submit( + self.delete_objs_in_batch, objs_to_delete, mockup_logger, source_bucket + ) # Restart noobaa pods nb_core_pod = get_noobaa_core_pod()