diff --git a/ocs_ci/ocs/resources/mcg.py b/ocs_ci/ocs/resources/mcg.py index 2af62316d90..70dd4b41acc 100644 --- a/ocs_ci/ocs/resources/mcg.py +++ b/ocs_ci/ocs/resources/mcg.py @@ -1014,9 +1014,12 @@ def reset_core_pod(self): Delete the noobaa-core pod and wait for it to come up again """ + from ocs_ci.ocs.resources.pod import wait_for_pods_by_label_count self.core_pod.delete(wait=True) - sleep(60) + wait_for_pods_by_label_count( + label=constants.NOOBAA_CORE_POD_LABEL, exptected_count=1 + ) self.core_pod = Pod( **get_pods_having_label(constants.NOOBAA_CORE_POD_LABEL, self.namespace)[0] ) diff --git a/ocs_ci/ocs/resources/pod.py b/ocs_ci/ocs/resources/pod.py index 2fde18e16df..b8a7f548ce8 100644 --- a/ocs_ci/ocs/resources/pod.py +++ b/ocs_ci/ocs/resources/pod.py @@ -1517,7 +1517,12 @@ def run_io_and_verify_mount_point(pod_obj, bs="10M", count="950"): return used_percentage -def get_pods_having_label(label, namespace, cluster_config=None, statuses=None): +def get_pods_having_label( + label, + namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, + cluster_config=None, + statuses=None, +): """ Fetches pod resources with given label in given namespace @@ -2081,7 +2086,7 @@ def wait_for_storage_pods(timeout=200): helpers.wait_for_resource_state(resource=pod_obj, state=state, timeout=timeout) -def wait_for_noobaa_pods(timeout=300): +def wait_for_noobaa_pods_running(timeout=300, sleep=10): """ Wait until all the noobaa pods have reached status RUNNING @@ -2090,12 +2095,23 @@ def wait_for_noobaa_pods(timeout=300): """ - all_pod_objs = get_noobaa_pods() + def _check_nb_pods_status(): + nb_pod_labels = [ + constants.NOOBAA_CORE_POD_LABEL, + constants.NOOBAA_ENDPOINT_POD_LABEL, + constants.NOOBAA_OPERATOR_POD_LABEL, + constants.NOOBAA_DB_LABEL_47_AND_ABOVE, + ] + nb_pods_running = list() + for pod_label in nb_pod_labels: + pods = get_pods_having_label(pod_label, statuses=[constants.STATUS_RUNNING]) + if len(pods) == 0: + return False + nb_pods_running.append(pod_label) + return set(nb_pod_labels) == set(nb_pods_running) - for pod in all_pod_objs: - helpers.wait_for_resource_state( - resource=pod, state=constants.STATUS_RUNNING, timeout=timeout - ) + sampler = TimeoutSampler(timeout=timeout, sleep=10, func=_check_nb_pods_status) + sampler.wait_for_func_status(True) def verify_pods_upgraded(old_images, selector, count=1, timeout=720): diff --git a/tests/conftest.py b/tests/conftest.py index 07ed1bbee0f..78821cb05fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -94,7 +94,7 @@ verify_data_integrity_for_multi_pvc_objs, get_noobaa_pods, get_pod_count, - wait_for_pods_by_label_count, get_noobaa_core_pod, + wait_for_pods_by_label_count, ) from ocs_ci.ocs.resources.pvc import PVC, create_restore_pvc from ocs_ci.ocs.version import get_ocs_version, get_ocp_version_dict, report_ocs_version diff --git a/tests/cross_functional/system_test/test_object_expiration.py b/tests/cross_functional/system_test/test_object_expiration.py index dac80d2e9b2..efacef175c7 100644 --- a/tests/cross_functional/system_test/test_object_expiration.py +++ b/tests/cross_functional/system_test/test_object_expiration.py @@ -25,7 +25,7 @@ from ocs_ci.ocs.resources.pod import ( get_noobaa_core_pod, get_noobaa_db_pod, - wait_for_noobaa_pods, + wait_for_noobaa_pods_running, ) from ocs_ci.ocs.node import ( drain_nodes, @@ -122,8 +122,6 @@ def test_object_expiration( expire_objects_in_bucket(bucket) - logger.info(f"Getting {object_key} from bucket: {bucket} after 1 day + 8 hours") - sampler = TimeoutSampler( timeout=600, sleep=10, @@ -401,7 +399,7 @@ def check_if_objects_expired(mcg_obj, bucket_name, prefix=""): status=constants.NODE_READY, timeout=300, ) - wait_for_noobaa_pods(timeout=1200) + wait_for_noobaa_pods_running(timeout=1200) # check if the objects are expired sample_if_objects_expired() @@ -411,7 +409,7 @@ def check_if_objects_expired(mcg_obj, bucket_name, prefix=""): # Perform noobaa db backup and recovery noobaa_db_backup_and_recovery(snapshot_factory=snapshot_factory) - wait_for_noobaa_pods(timeout=1200) + wait_for_noobaa_pods_running(timeout=1200) sample_if_objects_expired()