diff --git a/ocs_ci/ocs/bucket_utils.py b/ocs_ci/ocs/bucket_utils.py index 35af84bd9ab..b0b32b88d66 100644 --- a/ocs_ci/ocs/bucket_utils.py +++ b/ocs_ci/ocs/bucket_utils.py @@ -163,6 +163,7 @@ def list_objects_from_bucket( pod_obj, target, prefix=None, + recursive=False, s3_obj=None, signed_request_creds=None, timeout=600, @@ -186,6 +187,10 @@ def list_objects_from_bucket( retrieve_cmd = f"ls {target}/{prefix}" else: retrieve_cmd = f"ls {target}" + + if recursive: + retrieve_cmd += " --recursive" + if s3_obj: secrets = [s3_obj.access_key_id, s3_obj.access_key, s3_obj.s3_internal_endpoint] elif signed_request_creds: @@ -210,7 +215,9 @@ def list_objects_from_bucket( try: obj_list = [row.split()[3] for row in cmd_output.splitlines()] except Exception: - logger.warn(f"Failed to parse output of {retrieve_cmd} command: {cmd_output}") + logger.warning( + f"Failed to parse output of {retrieve_cmd} command: {cmd_output}" + ) return obj_list diff --git a/tests/conftest.py b/tests/conftest.py index 4015cd33e92..07f38887510 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2533,6 +2533,66 @@ def scalecli_pod_cleanup(): return scalecli_pod_obj +@pytest.fixture(scope="session") +def scale_cli_v2_pod(request): + return scale_cli_v2_fixture(request, scope_name="session") + + +def scale_cli_v2_fixture(request, scope_name): + """ + Creates AWSCLI pod for relaying commands which consists of + thousands of directories consisting millions of objects under large_objects/ + Args: + scope_name (str): The name of the fixture's scope, + used for giving a descriptive name to the pod and configmap + + Returns: + pod: A pod running the AWS CLI and consisting of large number of objects + """ + # Create the service-ca configmap to be mounted upon pod creation + service_ca_data = templating.load_yaml(constants.SCALECLI_SERVICE_CA_YAML) + service_ca_configmap_name = create_unique_resource_name( + constants.SCALECLI_SERVICE_CA_CM_NAME, scope_name + ) + service_ca_data["metadata"]["name"] = service_ca_configmap_name + log.info("Trying to create the SCALE CLI service CA") + service_ca_configmap = helpers.create_resource(**service_ca_data) + + # create scale-cli pod + pod_dict_path = constants.SCALECLI_POD_YAML + scalecli_pod_dict = templating.load_yaml(pod_dict_path) + scalecli_pod_dict["spec"]["containers"][0][ + "image" + ] = "quay.io/ocsci/scale-cli-v2:latest" + scalecli_pod_dict["spec"]["volumes"][0]["configMap"][ + "name" + ] = service_ca_configmap_name + scalecli_pod_name = create_unique_resource_name("scale-cli", scope_name) + scalecli_pod_dict["metadata"]["name"] = scalecli_pod_name + + update_container_with_mirrored_image(scalecli_pod_dict) + update_container_with_proxy_env(scalecli_pod_dict) + + scalecli_pod_obj = Pod(**scalecli_pod_dict) + assert scalecli_pod_obj.create( + do_reload=True + ), f"Failed to create pod {scalecli_pod_name}" + OCP(namespace=defaults.ROOK_CLUSTER_NAMESPACE, kind="ConfigMap").wait_for_resource( + resource_name=service_ca_configmap.name, column="DATA", condition="1" + ) + helpers.wait_for_resource_state( + scalecli_pod_obj, constants.STATUS_RUNNING, timeout=600 + ) + + def scalecli_pod_cleanup(): + scalecli_pod_obj.delete() + service_ca_configmap.delete() + + request.addfinalizer(scalecli_pod_cleanup) + + return scalecli_pod_obj + + @pytest.fixture(scope="session") def javasdk_pod_session(request): return javasdk_pod_fixture(request, scope_name="session") diff --git a/tests/e2e/scale/noobaa/test_list_objects_extended.py b/tests/e2e/scale/noobaa/test_list_objects_extended.py new file mode 100644 index 00000000000..30839fc5d61 --- /dev/null +++ b/tests/e2e/scale/noobaa/test_list_objects_extended.py @@ -0,0 +1,167 @@ +import pytest +import random +import logging + +from ocs_ci.ocs.bucket_utils import ( + sync_object_directory, + list_objects_from_bucket, + s3_list_objects_v2, +) + +logger = logging.getLogger(__name__) + + +def generate_random_unicode_prefix(number_of_chars=3): + unicode_blocks = [ + (0x0020, 0x007E), # Basic Latin + (0x00A0, 0x00FF), # Latin-1 Supplement + (0x0100, 0x017F), # Latin Extended-A + (0x0180, 0x024F), # Latin Extended-B + (0x0250, 0x02AF), # IPA Extensions + (0x0391, 0x03A1), + (0x03A3, 0x03A9), + (0x03B1, 0x03C1), + (0x03C3, 0x03C9), # Greek and Coptic + (0x0410, 0x042F), + (0x0430, 0x044F), # Cyrillic + (0x0531, 0x0556), # Armenian + (0x05D0, 0x05EA), # Hebrew + (0x0621, 0x063A), + (0x0641, 0x064A), # Arabic + (0x0905, 0x0939), # Devanagari + (0x4E00, 0x9FFF), # CJK Unified Ideographs (Chinese) + (0x3040, 0x309F), # Hiragana (Japanese) + (0x30A0, 0x30FF), # Katakana (Japanese) + (0x3041, 0x3096), # Additional Japanese kana + (0x4E00, 0x9FFF), # CJK Unified Ideographs (Chinese) + (0x3400, 0x4DBF), # Extension A (Chinese) + (0x2000, 0x206F), # General Punctuation + (0x2190, 0x21FF), # Arrows + (0x2200, 0x22FF), # Mathematical Operators + (0x25A0, 0x25FF), # Geometric Shapes + (0x2600, 0x26FF), # Miscellaneous Symbols + ] + prefix = [] + for _ in range(number_of_chars): + start, end = random.choice(unicode_blocks) + prefix.append(chr(random.randint(start, end))) + return "".join(prefix) + + +def get_number_of_objs(io_pod, objects_path): + return int(io_pod.exec_sh_cmd_on_pod(command=f"ls -ltr {objects_path} | wc -l")) - 1 + + +def make_dirs(io_pod): + + uploaded_dir = "/data/uploaded_dir" + uploading_dir = "/data/uploading_dir" + + io_pod.exec_sh_cmd_on_pod( + command=f"mkdir -p {uploading_dir} && mkdir -p {uploaded_dir}" + ) + return uploading_dir, uploaded_dir + + +def setup_dir_struct_and_upload_objects( + io_pod, s3_obj, bucket_name, objects_path, h_level, v_level, pref_len=5 +): + if v_level == 0: + v_level = 1 + if h_level == 0: + h_level = 1 + + uploading_dir, uploaded_dir = make_dirs(io_pod) + + all_prefixes = [] + for _ in range(v_level * h_level): + all_prefixes.append(generate_random_unicode_prefix(pref_len)) + + num_of_objs = get_number_of_objs(io_pod, objects_path) + logger.info(f"Total number of objects: {num_of_objs}") + + objs_per_pref = num_of_objs // (v_level * h_level) + rem = num_of_objs % (v_level * h_level) + + for i in range(v_level): + pref_str = "" + for j in range(h_level): + pref = generate_random_unicode_prefix(pref_len) + pref_str += pref + "/" + logger.info( + f"uploading {objs_per_pref} onto {bucket_name} with prefix {pref_str}" + ) + if i == v_level - 1 and j == h_level - 1: + n_obj = objs_per_pref + rem + else: + n_obj = objs_per_pref + + objs = io_pod.exec_sh_cmd_on_pod( + command=f"ls -ltr {objects_path}| tail -n +2 | head -{n_obj} | awk '{{print $9}}'" + ).split() + logger.info(objs) + for obj in objs: + io_pod.exec_sh_cmd_on_pod( + command=f"mv {objects_path}/{obj} {uploading_dir}/" + ) + sync_object_directory( + io_pod, + f"{uploading_dir}/", + f"s3://{bucket_name}/{pref_str}", + s3_obj=s3_obj, + ) + io_pod.exec_sh_cmd_on_pod(command=f"mv {uploading_dir}/* {uploaded_dir}") + + +class TestListObjectsExtended: + @pytest.mark.parametrize( + argnames=["bucketclass", "h_level", "v_level"], + argvalues=[ + pytest.param( + { + "interface": "OC", + "backingstore_dict": {"aws": [(1, "eu-central-1")]}, + }, + 5, + 2, + ), + ], + ids=[ + "AWS-Data", + ], + ) + def test_list_small_small( + self, + scale_cli_v2_pod, + bucket_factory, + mcg_obj_session, + bucketclass, + h_level, + v_level, + ): + + bucket = bucket_factory(bucketclass=bucketclass, amount=1)[0] + logger.info(f"bucket created: {bucket.name}") + + setup_dir_struct_and_upload_objects( + scale_cli_v2_pod, + mcg_obj_session, + bucket.name, + "/data/small_small", + h_level, + v_level, + ) + + # List all the objects recursively + ls_objs = list_objects_from_bucket( + scale_cli_v2_pod, bucket.name, recursive=True, s3_obj=mcg_obj_session + ) + assert len(ls_objs) == 100, "Not all the objects are listed" + + # List objects specific to some random prefix,delimiter combination + + # List all the objects in a directory + sync_object_directory( + scale_cli_v2_pod, "/data/small_small/", bucket.name, s3_obj=mcg_obj_session + ) + s3_list_objects_v2(mcg_obj_session, bucket.name)