diff --git a/sdk/python/test/e2e/utils.py b/sdk/python/test/e2e/utils.py index 202e6be28d..5572a39268 100644 --- a/sdk/python/test/e2e/utils.py +++ b/sdk/python/test/e2e/utils.py @@ -51,20 +51,35 @@ def verify_job_e2e( raise Exception(f"{client.job_kind} conditions are invalid: {conditions}") # Job should have correct conditions. - if not client.is_job_created(job=job): - raise Exception(f"{client.job_kind} should be in Created condition") + if ( + not client.is_job_created(job=job) + or not client.is_job_succeeded(job=job) + or client.is_job_running(job=job) + or client.is_job_restarting(job=job) + or client.is_job_failed(job=job) + ): + print_job_results(client, name, namespace) + raise Exception( + f"{client.job_kind} should be in Succeeded and Created conditions. " + f"{client.job_kind} should not be in Running, Restarting, or Failed conditions" + ) + + print_job_results(client, name, namespace) - if client.is_job_running(job=job): - raise Exception(f"{client.job_kind} should not be in Running condition") - if client.is_job_restarting(job=job): - raise Exception(f"{client.job_kind} should not be in Restarting condition") +def get_pod_spec_scheduler_name(gang_scheduler_name: str) -> str: + if gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_SCHEDULER_PLUGINS: + return DEFAULT_SCHEDULER_PLUGINS_NAME + elif gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_VOLCANO: + return TEST_GANG_SCHEDULER_NAME_VOLCANO + + return "" - if not client.is_job_succeeded(job=job): - raise Exception(f"{client.job_kind} should be in Succeeded condition") - if client.is_job_failed(job=job): - raise Exception(f"{client.job_kind} should not be in Failed condition") +def print_job_results(client: TrainingClient, name: str, namespace: str): + # Print Job. + logging.info(f"\n\n\n{client.job_kind} info") + logging.info(client.get_job(name, namespace)) # Print Job pod names. logging.info(f"\n\n\n{client.job_kind} pod names") @@ -73,12 +88,3 @@ def verify_job_e2e( # Print Job logs. logging.info(f"\n\n\n{client.job_kind} logs") client.get_job_logs(name, namespace) - - -def get_pod_spec_scheduler_name(gang_scheduler_name: str) -> str: - if gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_SCHEDULER_PLUGINS: - return DEFAULT_SCHEDULER_PLUGINS_NAME - elif gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_VOLCANO: - return TEST_GANG_SCHEDULER_NAME_VOLCANO - - return ""