From ec6bfdb5eff7891e966fa5cc913f1561fc263e23 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 10 Nov 2024 18:52:08 +0100 Subject: [PATCH 1/2] implement exported variables filter --- scripts/bot-build.slurm | 9 +++++++++ tasks/build.py | 31 +++++++++++++++++++++++++++++++ tools/filter.py | 4 +++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index bb0faa91..cec2774d 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -10,6 +10,7 @@ # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) +# author: Sam Moors (@smoors) # # license: GPLv2 # @@ -23,6 +24,14 @@ # for example, repos.cfg and configuration file bundles for repositories echo "Starting bot-build.slurm" +EXPORT_VARS_SCRIPT=bot/export_vars.sh +if [ -f ${EXPORT_VARS_SCRIPT} ]; then + echo "${EXPORT_VARS_SCRIPT} script found in '${PWD}', so sourcing it!" + source ${EXPORT_VARS_SCRIPT} +else + echo "could not find ${EXPORT_VARS_SCRIPT} script in '${PWD}', skipping" >&2 +fi +echo "$EXPORT_VARS_SCRIPT finished" BOT_BUILD_SCRIPT=bot/build.sh if [ -f ${BOT_BUILD_SCRIPT} ]; then echo "${BOT_BUILD_SCRIPT} script found in '${PWD}', so running it!" diff --git a/tasks/build.py b/tasks/build.py index 48cf3281..7e630b01 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -46,6 +46,8 @@ _ERROR_GIT_CLONE = "curl" _ERROR_NONE = "none" +# other constants +EXPORT_VARS_FILE = 'export_vars.sh' Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id', 'accelerator')) @@ -463,6 +465,29 @@ def apply_cvmfs_customizations(cvmfs_customizations, arch_job_dir): # for now, only existing mappings may be customized +def prepare_export_vars_file(job_dir, exportvars): + """ + Set up EXPORT_VARS_FILE in directory /bot. This file will be + sourced before running the bot/build.sh script. + + Args: + job_dir (string): working directory of the job + exportvars (list): strings of the form VAR=VALUE to be exported + + Returns: + None (implicitly) + """ + fn = sys._getframe().f_code.co_name + + content = '\n'.join(f'export {x}' for x in exportvars) + export_vars_path = os.path.join(job_dir, 'bot', EXPORT_VARS_FILE) + + with open(export_vars_path, 'w') as file: + file.write(content) + + log(f"{fn}(): created exported variables file {export_vars_path}") + + def prepare_jobs(pr, cfg, event_info, action_filter): """ Prepare all jobs whose context matches the given filter. Preparation includes @@ -510,6 +535,9 @@ def prepare_jobs(pr, cfg, event_info, action_filter): log(f"{fn}(): found no accelerator requirement") accelerator = None + # determine exportvars from action_filter argument + exportvars = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_EXPORT) + jobs = [] for arch, slurm_opt in arch_map.items(): arch_dir = arch.replace('/', '_') @@ -564,6 +592,9 @@ def prepare_jobs(pr, cfg, event_info, action_filter): prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type, accelerator) + if exportvars: + prepare_export_vars_file(job_dir, exportvars) + # enlist jobs to proceed job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id, accelerator) jobs.append(job) diff --git a/tools/filter.py b/tools/filter.py index 8b14f5eb..0caa2af8 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -20,16 +20,18 @@ # (none yet) -# NOTE because one can use any prefix of one of the four components below to +# NOTE because one can use any prefix of one of the components below to # define a filter, we need to make sure that no two filters share the same # prefix OR we have to change the handling of filters. FILTER_COMPONENT_ACCEL = 'accelerator' FILTER_COMPONENT_ARCH = 'architecture' +FILTER_COMPONENT_EXPORT = 'exportvariable' FILTER_COMPONENT_INST = 'instance' FILTER_COMPONENT_JOB = 'job' FILTER_COMPONENT_REPO = 'repository' FILTER_COMPONENTS = [FILTER_COMPONENT_ACCEL, FILTER_COMPONENT_ARCH, + FILTER_COMPONENT_EXPORT, FILTER_COMPONENT_INST, FILTER_COMPONENT_JOB, FILTER_COMPONENT_REPO From 4eef37f182fe8317ae46c100b24b8d69abb578a2 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 23 Nov 2024 17:34:34 +0100 Subject: [PATCH 2/2] add allowed_exportvars to app.cfg --- README.md | 9 +++++++++ app.cfg.example | 4 ++++ scripts/bot-build.slurm | 2 +- tasks/build.py | 42 ++++++++++++++++++++++++++++++++++++++--- tools/config.py | 3 ++- 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 836cc168..0c3be6df 100644 --- a/README.md +++ b/README.md @@ -433,6 +433,15 @@ allow_update_submit_opts = false options via custom module `det_submit_opts` provided by the pull request being processed. +``` +allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"] +``` +`allowed_exportvars` defines a list of name-value pairs (environment +variables) that are allowed to be specified in a PR command with the +`exportvariable` filter. To specify multiple environment variables, multiple +`exportvariable` filters must be used (one per variable). These variables will +be exported into the build environment before running the bot/build.sh script. + #### `[bot_control]` section diff --git a/app.cfg.example b/app.cfg.example index 152de2bc..19581219 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -132,6 +132,10 @@ no_build_permission_comment = Label `bot:build` has been set by user `{build_lab # whether or not to allow updating the submit options via custom module det_submit_opts allow_update_submit_opts = false +# defines which name-value pairs (environment variables) are allowed to be +# exported into the build environment via `exportvariable` filters +allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"] + [deploycfg] # script for uploading built software packages diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index cec2774d..ee6fac63 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -24,7 +24,7 @@ # for example, repos.cfg and configuration file bundles for repositories echo "Starting bot-build.slurm" -EXPORT_VARS_SCRIPT=bot/export_vars.sh +EXPORT_VARS_SCRIPT=cfg/export_vars.sh if [ -f ${EXPORT_VARS_SCRIPT} ]; then echo "${EXPORT_VARS_SCRIPT} script found in '${PWD}', so sourcing it!" source ${EXPORT_VARS_SCRIPT} diff --git a/tasks/build.py b/tasks/build.py index 7e630b01..0ddcf61f 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -123,7 +123,7 @@ def get_build_env_cfg(cfg): cvmfs_customizations = {} try: cvmfs_customizations_str = buildenv.get(config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS) - log("{fn}(): cvmfs_customizations '{cvmfs_customizations_str}'") + log(f"{fn}(): cvmfs_customizations '{cvmfs_customizations_str}'") if cvmfs_customizations_str is not None: cvmfs_customizations = json.loads(cvmfs_customizations_str) @@ -172,6 +172,34 @@ def get_architecture_targets(cfg): return arch_target_map +def get_allowed_exportvars(cfg): + """ + Obtain list of allowed export variables + + Args: + cfg (ConfigParser): ConfigParser instance holding full configuration + (typically read from 'app.cfg') + + Returns: + (list): list of allowed export variable-value pairs of the format VARIABLE=VALUE + """ + fn = sys._getframe().f_code.co_name + + buildenv = cfg[config.SECTION_BUILDENV] + allowed_str = buildenv.get(config.BUILDENV_SETTING_ALLOWED_EXPORTVARS) + allowed = [] + + if allowed_str: + try: + allowed = json.loads(allowed_str) + except json.JSONDecodeError as err: + print(err) + error(f"{fn}(): Value for allowed_exportvars ({allowed_str}) could not be decoded.") + + log(f"{fn}(): allowed_exportvars '{json.dumps(allowed)}'") + return allowed + + def get_repo_cfg(cfg): """ Obtain mappings of architecture targets to repository identifiers and @@ -467,7 +495,7 @@ def apply_cvmfs_customizations(cvmfs_customizations, arch_job_dir): def prepare_export_vars_file(job_dir, exportvars): """ - Set up EXPORT_VARS_FILE in directory /bot. This file will be + Set up EXPORT_VARS_FILE in directory /cfg. This file will be sourced before running the bot/build.sh script. Args: @@ -480,7 +508,7 @@ def prepare_export_vars_file(job_dir, exportvars): fn = sys._getframe().f_code.co_name content = '\n'.join(f'export {x}' for x in exportvars) - export_vars_path = os.path.join(job_dir, 'bot', EXPORT_VARS_FILE) + export_vars_path = os.path.join(job_dir, 'cfg', EXPORT_VARS_FILE) with open(export_vars_path, 'w') as file: file.write(content) @@ -509,6 +537,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): build_env_cfg = get_build_env_cfg(cfg) arch_map = get_architecture_targets(cfg) repocfg = get_repo_cfg(cfg) + allowed_exportvars = get_allowed_exportvars(cfg) base_repo_name = pr.base.repo.full_name log(f"{fn}(): pr.base.repo.full_name '{base_repo_name}'") @@ -538,6 +567,13 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # determine exportvars from action_filter argument exportvars = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_EXPORT) + # all exportvar filters must be allowed in order to run any jobs + if exportvars: + not_allowed = [x for x in exportvars if x not in allowed_exportvars] + if not_allowed: + log(f"{fn}(): exportvariable(s) {not_allowed} not allowed") + return [] + jobs = [] for arch, slurm_opt in arch_map.items(): arch_dir = arch.replace('/', '_') diff --git a/tools/config.py b/tools/config.py index ff641ebb..60554be0 100644 --- a/tools/config.py +++ b/tools/config.py @@ -25,7 +25,7 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from .logging import error -# define configration constants +# define configuration constants # SECTION_sectionname for any section name in app.cfg # sectionname_SETTING_settingname for any setting with name settingname in # section sectionname @@ -37,6 +37,7 @@ BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT = 'command_response_fmt' SECTION_BUILDENV = 'buildenv' +BUILDENV_SETTING_ALLOWED_EXPORTVARS = 'allowed_exportvars' BUILDENV_SETTING_ALLOW_UPDATE_SUBMIT_OPTS = 'allow_update_submit_opts' BUILDENV_SETTING_BUILD_JOB_SCRIPT = 'build_job_script' BUILDENV_SETTING_BUILD_LOGS_DIR = 'build_logs_dir'