From c645cd14bc626f35c7587514f5df4f6b71a22410 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Wed, 4 Oct 2023 11:06:50 +0530 Subject: [PATCH] objects should be unique identifiers and data every time validation is called Signed-off-by: Mahesh Shetty --- ocs_ci/helpers/e2e_helpers.py | 168 ++++++++++++--------- ocs_ci/ocs/bucket_utils.py | 43 +----- ocs_ci/ocs/constants.py | 2 + ocs_ci/ocs/resources/pod.py | 14 +- tests/conftest.py | 44 ++---- tests/e2e/conftest.py | 9 +- tests/e2e/system_test/test_mcg_recovery.py | 16 +- 7 files changed, 133 insertions(+), 163 deletions(-) diff --git a/ocs_ci/helpers/e2e_helpers.py b/ocs_ci/helpers/e2e_helpers.py index 5342baa7d819..55723cbb04a4 100644 --- a/ocs_ci/helpers/e2e_helpers.py +++ b/ocs_ci/helpers/e2e_helpers.py @@ -5,7 +5,7 @@ import re import time -import botocore.exceptions as botoexceptions +from uuid import uuid4 from ocs_ci.utility.retry import retry from ocs_ci.ocs.bucket_utils import ( random_object_round_trip_verification, @@ -27,8 +27,27 @@ def create_muliple_types_provider_obcs( num_of_buckets, bucket_types, cloud_providers, bucket_factory ): + """ + This function creates valid OBCs of different cloud providers + and bucket types + + Args: + num_of_buckets (int): Number of buckets + bucket_types (dict): Dict representing mapping between + bucket type and relevant configuration + cloud_providers (dict): Dict representing mapping between + cloud providers and relevant configuration + bucket_factory (fixture): bucket_factory fixture method + + Returns: + List: list of created buckets + + """ + def get_all_combinations_map(providers, bucket_types): """ + Create valid combination of cloud-providers and bucket-types + Args: providers (dict): dictionary representing cloud providers and the respective config @@ -36,6 +55,7 @@ def get_all_combinations_map(providers, bucket_types): types of bucket and the respective config Returns: List: containing all the possible combination of buckets + """ all_combinations = dict() @@ -69,13 +89,13 @@ def get_all_combinations_map(providers, bucket_types): ) ) - for i in range(0, buckets_left): + for index in range(0, buckets_left): buckets.extend( bucket_factory( interface="OC", amount=1, bucketclass=all_combination_of_obcs[ - list(all_combination_of_obcs.keys())[i] + list(all_combination_of_obcs.keys())[index] ], ) ) @@ -121,11 +141,12 @@ def validate_mcg_bucket_replicaton( upload_dir=bidi_uploaded_objs_dir_1, download_dir=bidi_downloaded_objs_dir_1, amount=object_amount, - pattern="FirstBiDi", + pattern=f"FirstBiDi-{uuid4().hex}", wait_for_replication=True, second_bucket_name=second_bucket.name, mcg_obj=mcg_obj_session, cleanup=True, + timeout=1200, ) random_object_round_trip_verification( @@ -134,11 +155,12 @@ def validate_mcg_bucket_replicaton( upload_dir=bidi_uploaded_objs_dir_2, download_dir=bidi_downloaded_objs_dir_2, amount=object_amount, - pattern="SecondBiDi", + pattern=f"SecondBiDi-{uuid4().hex}", wait_for_replication=True, second_bucket_name=first_bucket.name, mcg_obj=mcg_obj_session, cleanup=True, + timeout=1200, ) if event.is_set(): run_in_bg = False @@ -176,73 +198,70 @@ def validate_mcg_caching( """ while True: - verified_buckets = list() for bucket in cache_buckets: - try: - cache_uploaded_objs_dir = uploaded_objects_dir + "/cache" - cache_uploaded_objs_dir_2 = uploaded_objects_dir + "/cache_2" - cache_downloaded_objs_dir = downloaded_obejcts_dir + "/cache" - underlying_bucket_name = bucket.bucketclass.namespacestores[0].uls_name - - # Upload a random object to the bucket - logger.info(f"Uploading to the cache bucket: {bucket.name}") - objs_written_to_cache_bucket = write_random_test_objects_to_bucket( - awscli_pod_session, - bucket.name, - cache_uploaded_objs_dir, - pattern="Cache-", - mcg_obj=mcg_obj_session, - ) - wait_for_cache( - mcg_obj_session, - bucket.name, - objs_written_to_cache_bucket, - timeout=300, - ) + cache_uploaded_objs_dir = uploaded_objects_dir + "/cache" + cache_uploaded_objs_dir_2 = uploaded_objects_dir + "/cache_2" + cache_downloaded_objs_dir = downloaded_obejcts_dir + "/cache" + underlying_bucket_name = bucket.bucketclass.namespacestores[0].uls_name + + # Upload a random object to the bucket + logger.info(f"Uploading to the cache bucket: {bucket.name}") + obj_name = f"Cache-{uuid4().hex}" + objs_written_to_cache_bucket = write_random_test_objects_to_bucket( + awscli_pod_session, + bucket.name, + cache_uploaded_objs_dir, + pattern=obj_name, + mcg_obj=mcg_obj_session, + ) + wait_for_cache( + mcg_obj_session, + bucket.name, + objs_written_to_cache_bucket, + timeout=300, + ) - # Write a random, larger object directly to the underlying storage of the bucket - logger.info( - f"Uploading to the underlying bucket {underlying_bucket_name} directly" - ) - write_random_test_objects_to_bucket( - awscli_pod_session, - underlying_bucket_name, - cache_uploaded_objs_dir_2, - pattern="Cache-", - s3_creds=cld_mgr.aws_client.nss_creds, - bs="2M", - ) + # Write a random, larger object directly to the underlying storage of the bucket + logger.info( + f"Uploading to the underlying bucket {underlying_bucket_name} directly" + ) + write_random_test_objects_to_bucket( + awscli_pod_session, + underlying_bucket_name, + cache_uploaded_objs_dir_2, + pattern=obj_name, + s3_creds=cld_mgr.aws_client.nss_creds, + bs="2M", + ) - # Download the object from the cache bucket - sync_object_directory( - awscli_pod_session, - f"s3://{bucket.name}", - cache_downloaded_objs_dir, - mcg_obj_session, - ) + # Download the object from the cache bucket + awscli_pod_session.exec_cmd_on_pod(f"mkdir -p {cache_downloaded_objs_dir}") + sync_object_directory( + awscli_pod_session, + f"s3://{bucket.name}", + cache_downloaded_objs_dir, + mcg_obj_session, + ) - assert verify_s3_object_integrity( - original_object_path=f"{cache_uploaded_objs_dir}/Cache-0", - result_object_path=f"{cache_downloaded_objs_dir}/Cache-0", + assert verify_s3_object_integrity( + original_object_path=f"{cache_uploaded_objs_dir}/{obj_name}0", + result_object_path=f"{cache_downloaded_objs_dir}/{obj_name}0", + awscli_pod=awscli_pod_session, + ), "The uploaded and downloaded cached objects have different checksums" + + assert ( + verify_s3_object_integrity( + original_object_path=f"{cache_uploaded_objs_dir_2}/{obj_name}0", + result_object_path=f"{cache_downloaded_objs_dir}/{obj_name}0", awscli_pod=awscli_pod_session, - ), "The uploaded and downloaded cached objects have different checksums" - - assert ( - verify_s3_object_integrity( - original_object_path=f"{cache_uploaded_objs_dir_2}/Cache-0", - result_object_path=f"{cache_downloaded_objs_dir}/Cache-0", - awscli_pod=awscli_pod_session, - ) - is False - ), "The cached object was replaced by the new one before the TTL has expired" - verified_buckets.append(bucket.name) - if event.is_set(): - run_in_bg = False - break - except Exception: - logger.warning(f"verified so far: {verified_buckets}") - - raise + ) + is False + ), "The cached object was replaced by the new one before the TTL has expired" + logger.info(f"Verified caching for bucket: {bucket.name}") + + if event.is_set(): + run_in_bg = False + break if not run_in_bg: logger.warning("Stopping noobaa caching verification") @@ -269,14 +288,10 @@ def validate_rgw_kafka_notification(kafka_rgw_dict, event, run_in_bg=False): kafka_topic = kafka_rgw_dict["kafka_topic"] while True: + data = data + f"{uuid4().hex}" - try: - - def put_object_to_bucket(bucket_name, key, body): - return s3_client.put_object(Bucket=bucket_name, Key=key, Body=body) - - except botoexceptions.ClientError: - logger.warning("s3 put object timedout but ignoring as of now") + def put_object_to_bucket(bucket_name, key, body): + return s3_client.put_object(Bucket=bucket_name, Key=key, Body=body) assert put_object_to_bucket( bucketname, "key-1", data @@ -345,7 +360,10 @@ def validate_mcg_object_expiration( for i in range(object_amount): s3_put_object( - mcg_obj, bucket.name, f"{prefix}/obj-key-{i}", "Some random data" + mcg_obj, + bucket.name, + f"{prefix}/obj-key-{uuid4().hex}", + "Some random data", ) expire_objects_in_bucket(bucket.name) sample_if_objects_expired(mcg_obj, bucket.name) diff --git a/ocs_ci/ocs/bucket_utils.py b/ocs_ci/ocs/bucket_utils.py index 83f8f08bc4e3..35af84bd9ab2 100644 --- a/ocs_ci/ocs/bucket_utils.py +++ b/ocs_ci/ocs/bucket_utils.py @@ -1333,6 +1333,7 @@ def check_cached_objects_by_name(mcg_obj, bucket_name, expected_objects_names=No Returns: bool: True if all the objects exist in the cache as expected, False otherwise + """ res = mcg_obj.send_rpc_query( "object_api", @@ -1344,7 +1345,7 @@ def check_cached_objects_by_name(mcg_obj, bucket_name, expected_objects_names=No list_objects_res = [name["key"] for name in res.get("reply").get("objects")] if not expected_objects_names: expected_objects_names = [] - # if set(expected_objects_names) == set(list_objects_res): + for obj in expected_objects_names: if obj not in list_objects_res: logger.warning( @@ -1774,6 +1775,7 @@ def random_object_round_trip_verification( cleanup=False, result_pod=None, result_pod_path=None, + **kwargs, ): """ Writes random objects in a pod, uploads them to a bucket, @@ -1817,7 +1819,7 @@ def random_object_round_trip_verification( ) written_objects = io_pod.exec_cmd_on_pod(f"ls -A1 {upload_dir}").split(" ") if wait_for_replication: - compare_bucket_object_list(mcg_obj, bucket_name, second_bucket_name) + compare_bucket_object_list(mcg_obj, bucket_name, second_bucket_name, **kwargs) bucket_name = second_bucket_name # Download the random objects that were uploaded to the bucket sync_object_directory( @@ -1917,50 +1919,19 @@ def create_aws_bs_using_cli( ) -def change_expiration_query_interval(new_interval): - """ - Change how often noobaa should check for object expiration - By default it will be 8 hours - Args: - new_interval (int): New interval in minutes - """ - - from ocs_ci.ocs.resources.pod import ( - get_noobaa_core_pod, - wait_for_pods_to_be_running, - ) - - nb_core_pod = get_noobaa_core_pod() - new_interval = new_interval * 60 * 1000 - params = ( - '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", ' - f'"value": {{ "name": "CONFIG_JS_LIFECYCLE_INTERVAL", "value": "{new_interval}" }}}}]' - ) - OCP(kind="statefulset", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).patch( - resource_name=constants.NOOBAA_CORE_STATEFULSET, - params=params, - format_type="json", - ) - logger.info(f"Updated the expiration query interval to {new_interval} ms") - nb_core_pod.delete() - wait_for_pods_to_be_running(pod_names=[nb_core_pod.name], timeout=300) - - -def expire_objects_in_bucket(bucket_name, new_expire_interval=None): +def expire_objects_in_bucket(bucket_name): """ Manually expire the objects in a bucket + Args: bucket_name (str): Name of the bucket - new_expire_interval (int): New expiration interval + """ from ocs_ci.ocs.resources.pod import ( get_noobaa_db_pod, ) - if new_expire_interval is not None and isinstance(new_expire_interval, int): - change_expiration_query_interval(new_expire_interval) - creation_time = f"{date.today().year-1}-06-25T14:18:28.712Z" nb_db_pod = get_noobaa_db_pod() query = ( diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 1d0240768881..fb0e87df186e 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -116,6 +116,7 @@ CONFIG_JS_PREFIX = "CONFIG_JS_" BUCKET_REPLICATOR_DELAY_PARAM = CONFIG_JS_PREFIX + "BUCKET_REPLICATOR_DELAY" BUCKET_LOG_REPLICATOR_DELAY_PARAM = CONFIG_JS_PREFIX + "BUCKET_LOG_REPLICATOR_DELAY" +LIFECYCLE_INTERVAL_PARAM = "CONFIG_JS_LIFECYCLE_INTERVAL" # Resources / Kinds CEPHFILESYSTEM = "CephFileSystem" @@ -1756,6 +1757,7 @@ AWS_S3_ENDPOINT = "https://s3.amazonaws.com" NAMESPACE_FILESYSTEM = "nsfs" + # Cosbench constants COSBENCH = "cosbench" COSBENCH_PROJECT = "cosbench-project" diff --git a/ocs_ci/ocs/resources/pod.py b/ocs_ci/ocs/resources/pod.py index af38c55f8f8c..be93f4fc0d09 100644 --- a/ocs_ci/ocs/resources/pod.py +++ b/ocs_ci/ocs/resources/pod.py @@ -899,16 +899,10 @@ def get_noobaa_db_pod(): Returns: Pod object: Noobaa db pod object """ - if version.get_semantic_ocs_version_from_config() > version.VERSION_4_6: - nb_db = get_pods_having_label( - label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE, - namespace=config.ENV_DATA["cluster_namespace"], - ) - else: - nb_db = get_pods_having_label( - label=constants.NOOBAA_DB_LABEL_46_AND_UNDER, - namespace=config.ENV_DATA["cluster_namespace"], - ) + nb_db = get_pods_having_label( + label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE, + namespace=config.ENV_DATA["cluster_namespace"], + ) nb_db_pod = Pod(**nb_db[0]) return nb_db_pod diff --git a/tests/conftest.py b/tests/conftest.py index 7b8cbb3d2a3f..a24ae3174d3f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,9 +35,7 @@ from ocs_ci.ocs.bucket_utils import ( craft_s3_command, put_bucket_policy, - change_expiration_query_interval, ) - from ocs_ci.ocs.dr.dr_workload import BusyBox, BusyBox_AppSet from ocs_ci.ocs.exceptions import ( CommandFailed, @@ -94,12 +92,9 @@ get_all_pods, verify_data_integrity_for_multi_pvc_objs, get_noobaa_pods, -<<<<<<< HEAD get_pod_count, wait_for_pods_by_label_count, -======= get_noobaa_core_pod, ->>>>>>> 3ef4de76 (Entry critieria for MCG system tests) ) from ocs_ci.ocs.resources.pvc import PVC, create_restore_pvc from ocs_ci.ocs.version import get_ocs_version, get_ocp_version_dict, report_ocs_version @@ -6894,33 +6889,22 @@ def setup_logwriter_rbd_workload_factory(request, project_factory, teardown_fact @pytest.fixture() -def change_noobaa_lifecycle_interval(request): - nb_core_pod = get_noobaa_core_pod() - interval_changed = False +def reduce_expiration_interval(add_env_vars_to_noobaa_core_class): + """ + Reduce the interval in which the lifecycle + background worker is running - def factory(interval): - nonlocal interval_changed - interval_changed = True - change_expiration_query_interval(new_interval=interval) + """ - def finalizer(): - if interval_changed: - params = ( - f'[{{"op": "test", "path": "/spec/template/spec/containers/0/env/20/name",' - f'"value": "CONFIG_JS_LIFECYCLE_INTERVAL"}},' - f'{{"op": "remove", "path": "/spec/template/spec/containers/0/env/20"}}]' - ) + def factory(interval): + """ + Args: + interval (int): new interval in minutes - OCP( - kind="statefulset", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE - ).patch( - resource_name=constants.NOOBAA_CORE_STATEFULSET, - params=params, - format_type="json", - ) - nb_core_pod.delete() - wait_for_pods_to_be_running(pod_names=[nb_core_pod.name], timeout=300) - log.info("Switched back to default lifecycle interval") + """ + new_intervals_in_miliseconds = 60 * interval * 1000 + add_env_vars_to_noobaa_core_class( + [(constants.LIFECYCLE_INTERVAL_PARAM, new_intervals_in_miliseconds)] + ) - request.addfinalizer(finalizer) return factory diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 63e0d35e137a..f91def5ad87b 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -827,6 +827,9 @@ def factory(number_of_buckets, bucket_types, cloud_providers): all_buckets = create_muliple_types_provider_obcs( number_of_buckets, cache_type, cloud_providers, bucket_factory ) + logger.info( + f"These are the cache buckets created: {[bucket.name for bucket in all_buckets]}" + ) return all_buckets return factory @@ -834,7 +837,7 @@ def factory(number_of_buckets, bucket_types, cloud_providers): @pytest.fixture() def setup_mcg_expiration_feature_buckets( - request, bucket_factory, mcg_obj, change_noobaa_lifecycle_interval + request, bucket_factory, mcg_obj, reduce_expiration_interval ): """ This fixture does the setup for validating MCG replication @@ -859,9 +862,9 @@ def factory(number_of_buckets, bucket_types, cloud_providers): """ type = dict() type["data"] = bucket_types["data"] - + reduce_expiration_interval(interval=2) logger.info("Changed noobaa lifecycle interval to 2 minutes") - change_noobaa_lifecycle_interval(interval=2) + expiration_rule = { "Rules": [ { diff --git a/tests/e2e/system_test/test_mcg_recovery.py b/tests/e2e/system_test/test_mcg_recovery.py index e38f52cb0b98..cbc622edec0a 100644 --- a/tests/e2e/system_test/test_mcg_recovery.py +++ b/tests/e2e/system_test/test_mcg_recovery.py @@ -1,5 +1,6 @@ import logging import pytest +import time from ocs_ci.framework.pytest_customization.marks import ( system_test, @@ -26,7 +27,7 @@ class TestMCGRecovery(E2ETest): @pytest.mark.parametrize( argnames=["bucket_amount", "object_amount"], - argvalues=[pytest.param(2, 15)], + argvalues=[pytest.param(5, 5)], ) def test_mcg_db_backup_recovery( self, @@ -42,21 +43,18 @@ def test_mcg_db_backup_recovery( num_of_buckets=bucket_amount, object_amount=object_amount, is_disruptive=True, - skip_any_features=["caching", "nsfs", "rgw kafka"], + skip_any_features=["nsfs", "rgw kafka", "caching"], ) noobaa_db_backup_and_recovery(snapshot_factory=snapshot_factory) - import time + # wait 1 min for complete stabilization time.sleep(60) - event, threads = validate_mcg_bg_features( + + validate_mcg_bg_features( feature_setup_map, run_in_bg=False, - skip_any_features=["caching", "nsfs", "rgw kafka"], + skip_any_features=["nsfs", "rgw kafka", "caching"], object_amount=object_amount, ) - - event.set() - for th in threads: - th.result() log.info("No issues seen with the MCG bg feature validation")