Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes mirroring timeout issue #8658

Merged
merged 1 commit into from
Dec 12, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 27 additions & 19 deletions ocs_ci/ocs/resources/mcg.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ def cli_create_bucketclass(
f"bucketclass create {placement_type}{name}{backingstores}{placement_policy}{replication_policy}"
)

def check_if_mirroring_is_done(self, bucket_name, timeout=140):
def check_if_mirroring_is_done(self, bucket_name, timeout=300):
"""
Check whether all object chunks in a bucket
are mirrored across all backing stores.
Expand All @@ -756,12 +756,12 @@ def check_if_mirroring_is_done(self, bucket_name, timeout=140):
bucket_name: The name of the bucket that should be checked
timeout: timeout in seconds to check if mirroring

Returns:
bool: Whether mirroring finished successfully
Raises:
AssertionError: In case mirroring is not done in defined time.

"""

def _check_mirroring():
def _get_mirroring_percentage():
results = []
obj_list = (
self.send_rpc_query(
Expand Down Expand Up @@ -798,21 +798,29 @@ def _check_mirroring():
results.append(True)
else:
results.append(False)

return all(results)

try:
for mirroring_is_complete in TimeoutSampler(timeout, 5, _check_mirroring):
if mirroring_is_complete:
logger.info("All objects mirrored successfully.")
return True
else:
logger.info("Waiting for the mirroring process to finish...")
except TimeoutExpiredError:
logger.error(
"The mirroring process did not complete within the time limit."
)
assert False
current_percentage = (results.count(True) / len(results)) * 100
return current_percentage

mirror_percentage = _get_mirroring_percentage()
logger.info(f"{mirror_percentage}% mirroring is done.")
previous_percentage = 0
while mirror_percentage < 100:
previous_percentage = mirror_percentage
try:
for mirror_percentage in TimeoutSampler(
timeout, 5, _get_mirroring_percentage
):
if previous_percentage == mirror_percentage:
logger.warning("The mirroring process is stuck.")
else:
break
except TimeoutExpiredError:
logger.error(
f"The mirroring process is stuck from last {timeout} seconds."
)
assert False
mirror_percentage = _get_mirroring_percentage()
logger.info("All objects mirrored successfully.")

def check_backingstore_state(self, backingstore_name, desired_state, timeout=600):
"""
Expand Down
Loading