From 1ef2e2feb64cdf994501e02f39adfa1111383129 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 10 Feb 2024 00:31:27 +0100 Subject: [PATCH 1/5] add function that reads PR section from job metadata and use it in job manager --- eessi_bot_job_manager.py | 11 +++-------- tools/job_metadata.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 4830c1e3..ddf90722 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -42,7 +42,7 @@ from connections import github from tools import config, run_cmd from tools.args import job_manager_parse -from tools.job_metadata import read_metadata_file +from tools.job_metadata import read_job_metadata_from_file, read_metadata_file from tools.pr_comments import get_submitted_job_comment, update_comment @@ -253,7 +253,7 @@ def determine_finished_jobs(self, known_jobs, current_jobs): def read_job_pr_metadata(self, job_metadata_path): """ - Read job metadata file and return the contents of the 'PR' section. + Determine metadata od a job Args: job_metadata_path (string): path to job metadata file @@ -262,12 +262,7 @@ def read_job_pr_metadata(self, job_metadata_path): (ConfigParser): instance of ConfigParser corresponding to the 'PR' section or None """ - # reuse function from module tools.job_metadata to read metadata file - metadata = read_metadata_file(job_metadata_path, self.logfile) - if metadata and "PR" in metadata: - return metadata["PR"] - else: - return None + return read_job_metadata_from_file(job_metadata_path, self.logfile) def read_job_result(self, job_result_file_path): """ diff --git a/tools/job_metadata.py b/tools/job_metadata.py index a90cad90..56116b04 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -80,3 +80,28 @@ def read_metadata_file(metadata_path, log_file=None): else: log(f"No metadata file found at {metadata_path}.", log_file) return None + + +def read_job_metadata_from_file(filepath, log_file=None):$ + """ + Read job metadata from file + + Args: + filepath (string): path to job metadata file + log_file (string): path to log file + + Returns: + job_metadata (dict): dictionary containing job metadata or None + """ + + metadata = read_metadata_file(filepath, log_file=log_file) + if metadata: + # get PR section + if "PR" in metadata: + metadata_pr = metadata["PR"] + else: + metadata_pr = {} + return metadata_pr + else: + log(f"Metadata file '{filepath}' does not exist or could not be read") + return None From 4f743a2569811bf981d92193c2eae4b0c98daff1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 10 Feb 2024 00:37:11 +0100 Subject: [PATCH 2/5] removing trailing $ --- tools/job_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/job_metadata.py b/tools/job_metadata.py index 56116b04..b8cd4f0d 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -82,7 +82,7 @@ def read_metadata_file(metadata_path, log_file=None): return None -def read_job_metadata_from_file(filepath, log_file=None):$ +def read_job_metadata_from_file(filepath, log_file=None): """ Read job metadata from file From 7ed3d8d32b24da271bbd0ac0e7b513f1e6ee2b9e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 10 Feb 2024 18:58:50 +0100 Subject: [PATCH 3/5] removed method read_job_pr_metadata - call read_job_metadata_from_file from tools/job_metadata.py - moved one test from tests for EESSIBotSoftwareLayerJobManager to tests for tools/job_metadata.py --- eessi_bot_job_manager.py | 19 +++------------- tests/test_eessi_bot_job_manager.py | 24 -------------------- tests/test_tools_job_metadata.py | 34 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 40 deletions(-) create mode 100644 tests/test_tools_job_metadata.py diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index ddf90722..7d40b545 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -251,19 +251,6 @@ def determine_finished_jobs(self, known_jobs, current_jobs): return finished_jobs - def read_job_pr_metadata(self, job_metadata_path): - """ - Determine metadata od a job - - Args: - job_metadata_path (string): path to job metadata file - - Returns: - (ConfigParser): instance of ConfigParser corresponding to the 'PR' - section or None - """ - return read_job_metadata_from_file(job_metadata_path, self.logfile) - def read_job_result(self, job_result_file_path): """ Read job result file and return the contents of the 'RESULT' section. @@ -345,7 +332,7 @@ def process_new_job(self, new_job): # assuming that a bot job's working directory contains a metadata # file, its existence is used to check if the job belongs to the bot - metadata_pr = self.read_job_pr_metadata(job_metadata_path) + metadata_pr = read_job_metadata_from_file(job_metadata_path, self.logfile) if metadata_pr is None: log(f"No metadata file found at {job_metadata_path} for job {job_id}, so skipping it", @@ -441,7 +428,7 @@ def process_running_jobs(self, running_job): job_metadata_path = os.path.join(job_dir, metadata_file) # check if metadata file exist - metadata_pr = self.read_job_pr_metadata(job_metadata_path) + metadata_pr = read_job_metadata_from_file(job_metadata_path, self.logfile) if metadata_pr is None: raise Exception("Unable to find metadata file") @@ -586,7 +573,7 @@ def process_finished_job(self, finished_job): # obtain id of PR comment to be updated (from file '_bot_jobID.metadata') metadata_file = f"_bot_job{job_id}.metadata" job_metadata_path = os.path.join(new_symlink, metadata_file) - metadata_pr = self.read_job_pr_metadata(job_metadata_path) + metadata_pr = read_job_metadata_from_file(job_metadata_path, self.logfile) if metadata_pr is None: raise Exception("Unable to find metadata file ... skip updating PR comment") diff --git a/tests/test_eessi_bot_job_manager.py b/tests/test_eessi_bot_job_manager.py index c04033b5..2843069a 100644 --- a/tests/test_eessi_bot_job_manager.py +++ b/tests/test_eessi_bot_job_manager.py @@ -10,34 +10,10 @@ # # license: GPLv2 # -import os -import shutil from eessi_bot_job_manager import EESSIBotSoftwareLayerJobManager -def test_read_job_pr_metadata(tmpdir): - # copy needed app.cfg from tests directory - shutil.copyfile("tests/test_app.cfg", "app.cfg") - - # if metadata file does not exist, we should get None as return value - job_manager = EESSIBotSoftwareLayerJobManager() - path = os.path.join(tmpdir, 'test.metadata') - assert job_manager.read_job_pr_metadata(path) is None - - with open(path, 'w') as fp: - fp.write('''[PR] - repo=test - pr_number=12345''') - - metadata_pr = job_manager.read_job_pr_metadata(path) - expected = { - "repo": "test", - "pr_number": "12345", - } - assert metadata_pr == expected - - def test_determine_running_jobs(): job_manager = EESSIBotSoftwareLayerJobManager() diff --git a/tests/test_tools_job_metadata.py b/tests/test_tools_job_metadata.py new file mode 100644 index 00000000..d667cae0 --- /dev/null +++ b/tests/test_tools_job_metadata.py @@ -0,0 +1,34 @@ +# Tests for 'tools/job_metadata.py' of the EESSI build-and-deploy bot, +# see https://github.com/EESSI/eessi-bot-software-layer +# +# The bot helps with requests to add software installations to the +# EESSI software layer, see https://github.com/EESSI/software-layer +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +import os +import shutil + +from tools.job_metadata import read_job_metadata_from_file + + +def test_read_job_metadata_from_file(tmpdir): + logfile = os.path.join(tmpdir, 'test_read_job_metadata_from_file.log') + # if metadata file does not exist, we should get None as return value + path = os.path.join(tmpdir, 'test.metadata') + assert read_job_metadata_from_file(path, logfile) is None + + with open(path, 'w') as fp: + fp.write('''[PR] + repo=test + pr_number=12345''') + + metadata_pr = read_job_metadata_from_file(path, logfile) + expected = { + "repo": "test", + "pr_number": "12345", + } + assert metadata_pr == expected From 03891c9a2f1ab200a6b307ffdff50d9e19caa11c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 10 Feb 2024 19:02:57 +0100 Subject: [PATCH 4/5] remove unused import --- tests/test_tools_job_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tools_job_metadata.py b/tests/test_tools_job_metadata.py index d667cae0..f5542c6f 100644 --- a/tests/test_tools_job_metadata.py +++ b/tests/test_tools_job_metadata.py @@ -10,7 +10,6 @@ # import os -import shutil from tools.job_metadata import read_job_metadata_from_file From ec2a31719bc7087fae2f52243341a6c2ad899b45 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 10 Feb 2024 19:09:36 +0100 Subject: [PATCH 5/5] still need app.cfg in some tests --- tests/test_eessi_bot_job_manager.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_eessi_bot_job_manager.py b/tests/test_eessi_bot_job_manager.py index 2843069a..bd492919 100644 --- a/tests/test_eessi_bot_job_manager.py +++ b/tests/test_eessi_bot_job_manager.py @@ -11,10 +11,15 @@ # license: GPLv2 # +import shutil + from eessi_bot_job_manager import EESSIBotSoftwareLayerJobManager def test_determine_running_jobs(): + # copy needed app.cfg from tests directory + shutil.copyfile("tests/test_app.cfg", "app.cfg") + job_manager = EESSIBotSoftwareLayerJobManager() assert job_manager.determine_running_jobs({}) == []