From 18f3fa0b136fd177aa8a252e8134123c259e6fb3 Mon Sep 17 00:00:00 2001 From: Jonas Qvigstad Date: Fri, 31 Mar 2023 17:09:03 +0200 Subject: [PATCH] Add cfg settings as constants and settings check --- app.cfg.example | 10 +++---- eessi_bot_event_handler.py | 8 ++++-- eessi_bot_job_manager.py | 57 ++++++++++++++++++++++++++++---------- tasks/build.py | 16 +++++++---- tools/config.py | 18 ++++++++++++ 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index 769b8182..9a4de158 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -141,14 +141,14 @@ scontrol_command = /usr/bin/scontrol # variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it [submitted_job_comments] -description = New job on instance `{app_name}` for architecture `{arch_name}` in job dir `{symlink}` -comment = job id `{job_id}` awaits release by job manager +initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` +awaits_release = job id `{job_id}` awaits release by job manager [new_job_comments] -comment = job awaits launch by Slurm scheduler +awaits_lauch = job awaits launch by Slurm scheduler [running_job_comments] -comment = job `{job_id}` is running +running_job = job `{job_id}` is running [finished_job_comments] success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir @@ -158,4 +158,4 @@ slurm_out = Found slurm output `{slurm_out}` in job dir missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. -multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. +multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. \ No newline at end of file diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index e2444e63..14e30179 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -15,6 +15,7 @@ # import waitress import sys +import tasks.build as build from connections import github from tools import config @@ -162,8 +163,11 @@ def main(): """Main function.""" opts = event_handler_parse() - # config is read to raise an exception early when the event_handler starts. - config.read_config() + required_config = { + build.SUBMITTED_JOB_COMMENTS: [build.INITIAL_COMMENT, build.AWAITS_RELEASE] + } + # config is read and checked for settings to raise an exception early when the event_handler starts. + config.check_required_cfg_settings(required_config) github.connect() if opts.file: diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 4d4e4794..9c8cfc8b 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -43,6 +43,26 @@ from pyghee.utils import log, error +AWAITS_LAUCH = "awaits_lauch" +FAILURE = "failure" +FINISHED_JOB_COMMENTS = "finished_job_comments" +NEW_JOB_COMMENTS = "new_job_comments" +MISSING_MODULES = "missing_modules" +MULTIPLE_TARBALLS = "multiple_tarballs" +NO_MATCHING_TARBALL = "no_matching_tarball" +NO_SLURM_OUT = "no_slurm_out" +NO_TARBALL_MESSAGE = "no_tarball_message" +RUNNING_JOB = "running_job" +RUNNING_JOB_COMMENTS = "running_job_comments" +SLURM_OUT = "slurm_out" +SUCCESS = "success" + +REQUIRED_CONFIG = { + NEW_JOB_COMMENTS: [AWAITS_LAUCH], + RUNNING_JOB_COMMENTS: [RUNNING_JOB], + FINISHED_JOB_COMMENTS: [SUCCESS, FAILURE, NO_SLURM_OUT, SLURM_OUT, MISSING_MODULES, + NO_TARBALL_MESSAGE, NO_MATCHING_TARBALL, MULTIPLE_TARBALLS] +} class EESSIBotSoftwareLayerJobManager: "main class for (Slurm) job manager of EESSI bot (separate process)" @@ -291,10 +311,10 @@ def process_new_job(self, new_job): # (c) add a row to the table # add row to status table if we found a comment if "comment_id" in new_job: - comments = config.read_config()["new_job_comments"] + new_job_comments_cfg = config.read_config()[NEW_JOB_COMMENTS] dt = datetime.now(timezone.utc) update = "\n|%s|released|" % dt.strftime("%b %d %X %Z %Y") - update += f"{comments['comment']}|" + update += f"{new_job_comments_cfg[AWAITS_LAUCH]}|" update_comment(new_job["comment_id"], pr, update) else: log( @@ -364,8 +384,8 @@ def process_running_jobs(self, running_job): if "comment_id" in running_job: dt = datetime.now(timezone.utc) - comments = config.read_config()["running_job_comments"] - running_msg = comments['comment'].format(job_id=running_job['jobid']) + running_job_comments_cfg = config.read_config()[RUNNING_JOB_COMMENTS] + running_msg = running_job_comments_cfg[RUNNING_JOB].format(job_id=running_job['jobid']) if "comment_body" in running_job and running_msg in running_job["comment_body"]: log("Not updating comment, '%s' already found" % running_msg) else: @@ -465,7 +485,7 @@ def process_finished_job(self, finished_job): dt = datetime.now(timezone.utc) - comments = config.read_config()["finished_job_comments"] + finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" if (no_missing_modules and targz_created and len(eessi_tarballs) == 1): @@ -474,7 +494,10 @@ def process_finished_job(self, finished_job): # (installation status, tarball name, tarball size) tarball_name = os.path.basename(eessi_tarballs[0]) tarball_size = os.path.getsize(eessi_tarballs[0]) / 2**30 - success_comment = comments["success"].format(tarball_name=tarball_name, tarball_size=tarball_size) + success_comment = finished_job_comments_cfg[SUCCESS].format( + tarball_name=tarball_name, + tarball_size=tarball_size + ) comment_update += f"{success_comment}|" # NOTE explicitly name repo in build job comment? # comment_update += '\nAwaiting approval to @@ -488,33 +511,37 @@ def process_finished_job(self, finished_job): # prepare a message with details about the above conditions and # update PR with a comment - comment_update += f"{comments['failure']}