From e5313d26c666983f73266e9cc6941a53f97b442c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Sep 2024 19:11:39 +0200 Subject: [PATCH] add support for specifying that build job script is located in another repository --- tasks/build.py | 69 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/tasks/build.py b/tasks/build.py index 4196154..069a250 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -25,6 +25,7 @@ import os import shutil import sys +import tempfile # Third party imports (anything installed into the local Python environment) from pyghee.utils import error, log @@ -83,7 +84,17 @@ def get_build_env_cfg(cfg): config_data[config.BUILDENV_SETTING_LOCAL_TMP] = local_tmp build_job_script = buildenv.get(config.BUILDENV_SETTING_BUILD_JOB_SCRIPT) - log(f"{fn}(): build_job_script '{build_job_script}'") + # figure out whether path to build job script is just a path in current directory (path/to/job_script), + # or a location in another Git repository (path/to/job_script@repo) + if '@' in build_job_script: + build_job_script_path, build_job_script_repo = build_job_script.split('@', 1) + log(f"{fn}(): build_job_script '{build_job_script_path}' in repo {build_job_script_repo}") + build_job_script = { + 'path': build_job_script_path, + 'repo': build_job_script_repo, + } + else: + log(f"{fn}(): build_job_script '{build_job_script}'") config_data[config.BUILDENV_SETTING_BUILD_JOB_SCRIPT] = build_job_script submit_command = buildenv.get(config.BUILDENV_SETTING_SUBMIT_COMMAND) @@ -302,6 +313,19 @@ def create_pr_dir(pr, cfg, event_info): return year_month, pr_id, run_dir +def clone_git_repo(repo, path): + """ + Clone specified Git repo to specified path + """ + git_clone_cmd = ' '.join(['git clone', repo, path]) + log(f'cloning with command {git_clone_cmd}') + clone_output, clone_error, clone_exit_code = run_cmd( + git_clone_cmd, "Clone repo", path, raise_on_error=False + ) + + return (clone_output, clone_error, clone_exit_code) + + def download_pr(repo_name, branch_name, pr, arch_job_dir): """ Download pull request to job working directory @@ -323,11 +347,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): # - 'git checkout' base branch of pull request # - 'curl' diff for pull request # - 'git apply' diff file - git_clone_cmd = ' '.join(['git clone', f'https://github.com/{repo_name}', arch_job_dir]) - log(f'cloning with command {git_clone_cmd}') - clone_output, clone_error, clone_exit_code = run_cmd( - git_clone_cmd, "Clone repo", arch_job_dir, raise_on_error=False - ) + clone_output, clone_error, clone_exit_code = clone_git_repo(f'https://github.com/{repo_name}', arch_job_dir) if clone_exit_code != 0: error_stage = _ERROR_GIT_CLONE return clone_output, clone_error, clone_exit_code, error_stage @@ -709,13 +729,48 @@ def submit_job(job, cfg): job = job._replace(slurm_opts=det_submit_opts(job)) log(f"{fn}(): updated job.slurm_opts: {job.slurm_opts}") + build_job_script = build_env_cfg[config.BUILDENV_SETTING_BUILD_JOB_SCRIPT] + if isinstance(build_job_script, str): + build_job_script_path = build_job_script + log(f"{fn}(): path to build job script: {build_job_script_path}") + elif isinstance(build_job_script, dict): + build_job_script_repo = build_job_script.get('repo') + if build_job_script_repo: + log(f"{fn}(): repository in which build job script is located: {build_job_script_repo}") + else: + error(f"Failed to determine repository in which build job script is located from: {build_job_script}") + + build_job_script_path = build_job_script.get('path') + if build_job_script_path: + log(f"{fn}(): path to build job script in repository: {build_job_script_path}") + else: + error(f"Failed to determine path of build job script in repository from: {build_job_script}") + + # clone repo to temporary directory, and correctly set path to build job script + tmpdir = tempfile.mkdtemp() + repo_subdir = '-'.join(build_job_script_repo.split('/')[-2:]) + target_dir = os.path.join(tmpdir, repo_subdir) + os.makedirs(target_dir, exist_ok=True) + + clone_output, clone_error, clone_exit_code = clone_git_repo(build_job_script_repo, target_dir) + if clone_exit_code != 0: + error(f"Failed to clone repository {build_job_script_repo}: {clone_error}") + + repo_subdir = '-'.join(build_job_script_repo.split('/')[-2:]) + build_job_script_path = os.path.join(tmpdir, repo_subdir, build_job_script_path) + else: + error(f"Incorrect build job script specification, unknown type: {build_job_script}") + + if not os.path.exists(build_job_script_path): + error(f"Build job script not found at {build_job_script_path}") + command_line = ' '.join([ build_env_cfg[config.BUILDENV_SETTING_SUBMIT_COMMAND], build_env_cfg[config.BUILDENV_SETTING_SLURM_PARAMS], time_limit, job.slurm_opts] + ([f"--job-name='{job_name}'"] if job_name else []) + - [build_env_cfg[config.BUILDENV_SETTING_BUILD_JOB_SCRIPT]]) + [build_job_script_path]) cmdline_output, cmdline_error, cmdline_exit_code = run_cmd(command_line, "submit job for target '%s'" % job.arch_target,