Skip to content

Commit

Permalink
Merge pull request EESSI#266 from trz42/reorganise_constants_and_vali…
Browse files Browse the repository at this point in the history 
…date_config

reorganisation of constants and validation of config settings
  • Loading branch information
@Neves-P
Neves-P authored May 6, 2024
2 parents 9d924c5 + d3b211c commit 02fd520
Show file tree
Hide file tree
Showing 9 changed files with 455 additions and 290 deletions.
8 changes: 4 additions & 4 deletions connections/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ def get_token():

global _token
cfg = config.read_config()
github_cfg = cfg['github']
app_id = github_cfg.get('app_id')
installation_id = github_cfg.get('installation_id')
private_key_path = github_cfg.get('private_key')
github_cfg = cfg[config.SECTION_GITHUB]
app_id = github_cfg.get(config.GITHUB_SETTING_APP_ID)
installation_id = github_cfg.get(config.GITHUB_SETTING_INSTALLATION_ID)
private_key_path = github_cfg.get(config.GITHUB_SETTING_PRIVATE_KEY)
private_key = ''

with open(private_key_path, 'r') as private_key_file:
Expand Down
92 changes: 68 additions & 24 deletions eessi_bot_event_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,8 @@

# Local application imports (anything from EESSI/eessi-bot-software-layer)
from connections import github
import tasks.build as build
from tasks.build import check_build_permission, get_architecture_targets, get_repo_cfg, \
request_bot_build_issue_comments, submit_build_jobs
import tasks.deploy as deploy
from tasks.deploy import deploy_built_artefacts
from tools import config
from tools.args import event_handler_parse
Expand All @@ -39,11 +37,59 @@
from tools.pr_comments import create_comment


APP_NAME = "app_name"
BOT_CONTROL = "bot_control"
COMMAND_RESPONSE_FMT = "command_response_fmt"
GITHUB = "github"
REPO_TARGET_MAP = "repo_target_map"
REQUIRED_CONFIG = {
config.SECTION_ARCHITECTURETARGETS: [
config.ARCHITECTURETARGETS_SETTING_ARCH_TARGET_MAP], # required
config.SECTION_BOT_CONTROL: [
config.BOT_CONTROL_SETTING_COMMAND_PERMISSION, # required
config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT], # required
config.SECTION_BUILDENV: [
config.BUILDENV_SETTING_BUILD_JOB_SCRIPT, # required
config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended
config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended
config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended
# config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional
# config.BUILDENV_SETTING_HTTPS_PROXY, # optional
# config.BUILDENV_SETTING_HTTP_PROXY, # optional
config.BUILDENV_SETTING_JOBS_BASE_DIR, # required
# config.BUILDENV_SETTING_LOAD_MODULES, # optional
config.BUILDENV_SETTING_LOCAL_TMP, # required
config.BUILDENV_SETTING_NO_BUILD_PERMISSION_COMMENT, # required
config.BUILDENV_SETTING_SHARED_FS_PATH, # optional+recommended
# config.BUILDENV_SETTING_SLURM_PARAMS, # optional
config.BUILDENV_SETTING_SUBMIT_COMMAND], # required
config.SECTION_DEPLOYCFG: [
config.DEPLOYCFG_SETTING_ARTEFACT_PREFIX, # (required)
config.DEPLOYCFG_SETTING_ARTEFACT_UPLOAD_SCRIPT, # required
config.DEPLOYCFG_SETTING_BUCKET_NAME, # required
config.DEPLOYCFG_SETTING_DEPLOY_PERMISSION, # optional+recommended
# config.DEPLOYCFG_SETTING_ENDPOINT_URL, # optional
config.DEPLOYCFG_SETTING_METADATA_PREFIX, # (required)
config.DEPLOYCFG_SETTING_NO_DEPLOY_PERMISSION_COMMENT, # required
config.DEPLOYCFG_SETTING_UPLOAD_POLICY], # required
config.SECTION_DOWNLOAD_PR_COMMENTS: [
config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP], # required
config.SECTION_EVENT_HANDLER: [
config.EVENT_HANDLER_SETTING_LOG_PATH], # required
config.SECTION_GITHUB: [
config.GITHUB_SETTING_APP_ID, # required
config.GITHUB_SETTING_APP_NAME, # required
config.GITHUB_SETTING_INSTALLATION_ID, # required
config.GITHUB_SETTING_PRIVATE_KEY], # required
config.SECTION_REPO_TARGETS: [
config.REPO_TARGETS_SETTING_REPO_TARGET_MAP, # required
config.REPO_TARGETS_SETTING_REPOS_CFG_DIR], # required
config.SECTION_SUBMITTED_JOB_COMMENTS: [
config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT, # required
config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE] # required
}


class EESSIBotSoftwareLayer(PyGHee):
Expand All @@ -62,8 +108,8 @@ def __init__(self, *args, **kwargs):
super(EESSIBotSoftwareLayer, self).__init__(*args, **kwargs)

self.cfg = config.read_config()
event_handler_cfg = self.cfg['event_handler']
self.logfile = event_handler_cfg.get('log_path')
event_handler_cfg = self.cfg[config.SECTION_EVENT_HANDLER]
self.logfile = event_handler_cfg.get(config.EVENT_HANDLER_SETTING_LOG_PATH)

def log(self, msg, *args):
"""
Expand Down Expand Up @@ -110,8 +156,8 @@ def handle_issue_comment_event(self, event_info, log_file=None):
# log level is set to debug
self.log(f"Comment in {issue_url} (owned by @{owner}) {action} by @{sender}")

app_name = self.cfg[GITHUB][APP_NAME]
command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]

# currently, only commands in new comments are supported
# - commands have the syntax 'bot: COMMAND [ARGS*]'
Expand Down Expand Up @@ -301,8 +347,8 @@ def handle_pull_request_labeled_event(self, event_info, pr):
request_body = event_info['raw_request_body']
repo_name = request_body['repository']['full_name']
pr_number = request_body['pull_request']['number']
app_name = self.cfg[GITHUB][APP_NAME]
command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]
comment_body = command_response_fmt.format(
app_name=app_name,
comment_response=msg,
Expand Down Expand Up @@ -330,7 +376,7 @@ def handle_pull_request_opened_event(self, event_info, pr):
PyGithub, not the github from the internal connections module)
"""
self.log("PR opened: waiting for label bot:build")
app_name = self.cfg[GITHUB][APP_NAME]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
# TODO check if PR already has a comment with arch targets and
# repositories
arch_map = get_architecture_targets(self.cfg)
Expand All @@ -343,7 +389,8 @@ def handle_pull_request_opened_event(self, event_info, pr):
comment += f"{', '.join([f'`{arch}`' for arch in architectures])}"
else:
comment += "none"
repositories = list(set([repo_id for repo_ids in repo_cfg[REPO_TARGET_MAP].values() for repo_id in repo_ids]))
repositories = list(set([repo_id for repo_ids in repo_cfg[config.REPO_TARGETS_SETTING_REPO_TARGET_MAP].values()
for repo_id in repo_ids]))
comment += "\n- repositories: "
if len(repositories) > 0:
comment += f"{', '.join([f'`{repo_id}`' for repo_id in repositories])}"
Expand Down Expand Up @@ -547,9 +594,7 @@ def start(self, app, port=3000):
print(port_info)
self.log(port_info)

event_handler_cfg = self.cfg['event_handler']
my_logfile = event_handler_cfg.get('log_path')
log_file_info = "logging in to %s" % my_logfile
log_file_info = "logging in to %s" % self.logfile
print(log_file_info)
self.log(log_file_info)
waitress.serve(app, listen='*:%s' % port)
Expand All @@ -563,13 +608,12 @@ def main():
"""
opts = event_handler_parse()

required_config = {
build.SUBMITTED_JOB_COMMENTS: [build.INITIAL_COMMENT, build.AWAITS_RELEASE],
build.BUILDENV: [build.NO_BUILD_PERMISSION_COMMENT],
deploy.DEPLOYCFG: [deploy.NO_DEPLOY_PERMISSION_COMMENT]
}
# config is read and checked for settings to raise an exception early when the event_handler starts.
config.check_required_cfg_settings(required_config)
if config.check_required_cfg_settings(REQUIRED_CONFIG):
print("Configuration check: PASSED")
else:
print("Configuration check: FAILED")
sys.exit(1)
github.connect()

if opts.file:
Expand Down
73 changes: 42 additions & 31 deletions eessi_bot_job_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,27 @@
from tools.pr_comments import get_submitted_job_comment, update_comment


AWAITS_LAUNCH = "awaits_launch"
FINISHED_JOB_COMMENTS = "finished_job_comments"
JOB_RESULT_COMMENT_DESCRIPTION = "comment_description"
JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt"
JOB_TEST_COMMENT_DESCRIPTION = "comment_description"
JOB_TEST_UNKNOWN_FMT = "job_test_unknown_fmt"
NEW_JOB_COMMENTS = "new_job_comments"
RUNNING_JOB = "running_job"
RUNNING_JOB_COMMENTS = "running_job_comments"

# settings that are required in 'app.cfg'
REQUIRED_CONFIG = {
FINISHED_JOB_COMMENTS: [JOB_RESULT_UNKNOWN_FMT, JOB_TEST_UNKNOWN_FMT],
NEW_JOB_COMMENTS: [AWAITS_LAUNCH],
RUNNING_JOB_COMMENTS: [RUNNING_JOB]
}
config.SECTION_FINISHED_JOB_COMMENTS: [
config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT, # required
config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT], # required
config.SECTION_GITHUB: [
config.GITHUB_SETTING_APP_ID, # required
# config.GITHUB_SETTING_APP_NAME, # unused
config.GITHUB_SETTING_INSTALLATION_ID, # required
config.GITHUB_SETTING_PRIVATE_KEY], # required
config.SECTION_JOB_MANAGER: [
config.JOB_MANAGER_SETTING_LOG_PATH, # required
config.JOB_MANAGER_SETTING_JOB_IDS_DIR, # required
config.JOB_MANAGER_SETTING_POLL_COMMAND, # required
config.JOB_MANAGER_SETTING_POLL_INTERVAL, # optional+recommended
config.JOB_MANAGER_SETTING_SCONTROL_COMMAND], # required
config.SECTION_NEW_JOB_COMMENTS: [
config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH], # required
config.SECTION_RUNNING_JOB_COMMENTS: [
config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB] # required
}


class EESSIBotSoftwareLayerJobManager:
Expand All @@ -77,8 +83,8 @@ def __init__(self):
configuration to set the path to the logfile.
"""
cfg = config.read_config()
job_manager_cfg = cfg['job_manager']
self.logfile = job_manager_cfg.get('log_path')
job_manager_cfg = cfg[config.SECTION_JOB_MANAGER]
self.logfile = job_manager_cfg.get(config.JOB_MANAGER_SETTING_LOG_PATH)

def get_current_jobs(self):
"""
Expand Down Expand Up @@ -340,10 +346,10 @@ def process_new_job(self, new_job):

# update status table if we found a comment
if "comment_id" in new_job:
new_job_comments_cfg = config.read_config()[NEW_JOB_COMMENTS]
new_job_comments_cfg = config.read_config()[config.SECTION_NEW_JOB_COMMENTS]
dt = datetime.now(timezone.utc)
update = "\n|%s|released|" % dt.strftime("%b %d %X %Z %Y")
update += f"{new_job_comments_cfg[AWAITS_LAUNCH]}|"
update += f"{new_job_comments_cfg[config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH]}|"
update_comment(new_job["comment_id"], pr, update)
else:
log(
Expand Down Expand Up @@ -412,8 +418,9 @@ def process_running_jobs(self, running_job):

if "comment_id" in running_job:
dt = datetime.now(timezone.utc)
running_job_comments_cfg = config.read_config()[RUNNING_JOB_COMMENTS]
running_msg = running_job_comments_cfg[RUNNING_JOB].format(job_id=running_job['jobid'])
running_job_comments_cfg = config.read_config()[config.SECTION_RUNNING_JOB_COMMENTS]
running_msg_fmt = running_job_comments_cfg[config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB]
running_msg = running_msg_fmt.format(job_id=running_job['jobid'])
if "comment_body" in running_job and running_msg in running_job["comment_body"]:
log("Not updating comment, '%s' already found" % running_msg)
else:
Expand Down Expand Up @@ -476,7 +483,7 @@ def process_finished_job(self, finished_job):
# status = {SUCCESS,FAILURE,UNKNOWN}

# obtain format templates from app.cfg
finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS]
finished_job_comments_cfg = config.read_config()[config.SECTION_FINISHED_JOB_COMMENTS]

# check if _bot_jobJOBID.result exits
job_result_file = f"_bot_job{job_id}.result"
Expand All @@ -485,13 +492,13 @@ def process_finished_job(self, finished_job):
job_metadata.JOB_RESULT_SECTION,
self.logfile)

job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT]
job_result_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT]
# set fallback comment_description in case no result file was found
# (job_metadata.get_section_from_file returned None)
comment_description = job_result_unknown_fmt.format(filename=job_result_file)
if job_results:
# get preformatted comment_description or use previously set default for unknown
comment_description = job_results.get(JOB_RESULT_COMMENT_DESCRIPTION, comment_description)
comment_description = job_results.get(job_metadata.JOB_RESULT_COMMENT_DESCRIPTION, comment_description)

# report to log
log(f"{fn}(): finished job {job_id}\n"
Expand All @@ -514,13 +521,13 @@ def process_finished_job(self, finished_job):
job_metadata.JOB_TEST_SECTION,
self.logfile)

job_test_unknown_fmt = finished_job_comments_cfg[JOB_TEST_UNKNOWN_FMT]
job_test_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT]
# set fallback comment_description in case no test file was found
# (job_metadata.get_section_from_file returned None)
comment_description = job_test_unknown_fmt.format(filename=job_test_file)
if job_tests:
# get preformatted comment_description or use previously set default for unknown
comment_description = job_tests.get(JOB_TEST_COMMENT_DESCRIPTION, comment_description)
comment_description = job_tests.get(job_metadata.JOB_TEST_COMMENT_DESCRIPTION, comment_description)

# report to log
log(f"{fn}(): finished job {job_id}, test suite result\n"
Expand Down Expand Up @@ -570,7 +577,11 @@ def main():

# config is read and checked for settings to raise an exception early when
# the job_manager runs
config.check_required_cfg_settings(REQUIRED_CONFIG)
if config.check_required_cfg_settings(REQUIRED_CONFIG):
print("Configuration check: PASSED")
else:
print("Configuration check: FAILED")
sys.exit(1)
github.connect()

job_manager = EESSIBotSoftwareLayerJobManager()
Expand Down Expand Up @@ -611,16 +622,16 @@ def main():
job_manager.scontrol_command = ""
if max_iter != 0:
cfg = config.read_config()
job_mgr = cfg["job_manager"]
job_manager.job_ids_dir = job_mgr.get("job_ids_dir")
job_mgr = cfg[config.SECTION_JOB_MANAGER]
job_manager.job_ids_dir = job_mgr.get(config.JOB_MANAGER_SETTING_JOB_IDS_DIR)
job_manager.submitted_jobs_dir = os.path.join(
job_manager.job_ids_dir, "submitted"
)
job_manager.poll_command = job_mgr.get("poll_command") or False
poll_interval = int(job_mgr.get("poll_interval") or 0)
job_manager.poll_command = job_mgr.get(config.JOB_MANAGER_SETTING_POLL_COMMAND) or False
poll_interval = int(job_mgr.get(config.JOB_MANAGER_SETTING_POLL_INTERVAL) or 0)
if poll_interval <= 0:
poll_interval = 60
job_manager.scontrol_command = job_mgr.get("scontrol_command") or False
job_manager.scontrol_command = job_mgr.get(config.JOB_MANAGER_SETTING_SCONTROL_COMMAND) or False
os.makedirs(job_manager.submitted_jobs_dir, exist_ok=True)

# max_iter
Expand Down
Loading

0 comments on commit 02fd520

Please sign in to comment.