Setup buckets for stress testing, upload objects to the bucket parallel

Signed-off-by: Mahesh Shetty <[email protected]>

ocs_ci/helpers/mcg_stress_helper.py

@@ -1,12 +1,51 @@
 import logging
+import concurrent.futures
 
+from ocs_ci.ocs.resources.objectbucket import OBC
 from ocs_ci.ocs.resources.bucket_policy import NoobaaAccount
 from ocs_ci.ocs.resources.mcg_lifecycle_policies import LifecyclePolicy, ExpirationRule
-from ocs_ci.ocs.bucket_utils import s3_copy_object, list_objects_from_bucket
+from ocs_ci.ocs.bucket_utils import (
+    s3_copy_object,
+    list_objects_from_bucket,
+    sync_object_directory,
+)
 
 logger = logging.getLogger(__name__)
 
 
+def upload_objs_to_buckets(mcg_obj, pod_obj, buckets):
+    """
+    This will upload objects present in the stress-cli pod
+    to the buckets provided concurrently
+
+    Args:
+        mcg_obj (MCG): MCG object
+        pod_obj (Pod): Pod object
+        buckets (Dict): Map of bucket type and bucket object
+
+    """
+    src_path = "/complex_directory/dir_0_0/dir_1_0/dir_2_0/dir_3_0/dir_4_0/dir_5_0/dir_6_0/dir_7_0/dir_8_0/dir_9_0/"
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = list()
+        for type, bucket in buckets.items():
+            if type == "rgw":
+                s3_obj = OBC(bucket.name)
+            else:
+                s3_obj = mcg_obj
+            logger.info(f"OBJECT UPLOAD: Uploading objects to the bucket {bucket.name}")
+            future = executor.submit(
+                sync_object_directory, pod_obj, src_path, f"s3://{bucket.name}", s3_obj
+            )
+            futures.append(future)
+
+        logger.info(
+            "OBJECT UPLOAD: Waiting for the objects upload to complete for all the buckets"
+        )
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
+
+
 def run_noobaa_metadata_intense_ops(mcg_obj, pod_obj, bucket_factory, bucket_name):
 
     # Run metadata specific to bucket

tests/cross_functional/conftest.py

@@ -1351,3 +1351,49 @@ def factory(
         return feature_setup_map
 
     return factory
+
+
+@pytest.fixture(scope="session")
+def setup_stress_testing_bucket(bucket_factory_session, rgw_bucket_factory_session):
+    """
+    This session scoped fixture is for setting up the buckets for the stress testing
+    in MCG. This creates buckets of type AWS, AZURE, PV-POOL, RGW.
+
+    """
+
+    def factory():
+
+        bucket_configs = {
+            "aws": {
+                "interface": "CLI",
+                "backingstore_dict": {"aws": [(1, "eu-central-1")]},
+            },
+            "azure": {
+                "interface": "CLI",
+                "backingstore_dict": {"azure": [(1, None)]},
+            },
+            "pv-pool": {
+                "interface": "CLI",
+                "backingstore_dict": {
+                    "pv": [(1, 50, constants.DEFAULT_STORAGECLASS_RBD)]
+                },
+            },
+            "rgw": None,
+        }
+
+        bucket_objects = dict()
+
+        for type, bucketclass_dict in bucket_configs.items():
+            if type == "rgw":
+                bucket = rgw_bucket_factory_session(interface="rgw-oc")[0]
+            else:
+                bucket = bucket_factory_session(
+                    interface="CLI", bucketclass=bucketclass_dict
+                )[0]
+
+            logger.info(f"BUCKET CREATION: Created bucket {bucket.name} of type {type}")
+            bucket_objects[type] = bucket
+
+        return bucket_objects
+
+    return factory

tests/cross_functional/stress/test_noobaa_under_stress.py

@@ -0,0 +1,22 @@
+from ocs_ci.helpers.mcg_stress_helper import upload_objs_to_buckets
+
+
+class TestNoobaaUnderStress:
+
+    base_setup_buckets = None
+
+    def test_noobaa_under_stress(
+        self,
+        setup_stress_testing_bucket,
+        nb_stress_cli_pod,
+        mcg_obj_session,
+        rgw_obj_session,
+    ):
+
+        self.base_setup_buckets = setup_stress_testing_bucket()
+
+        upload_objs_to_buckets(
+            mcg_obj_session,
+            nb_stress_cli_pod,
+            self.base_setup_buckets,
+        )