add shared_fs_path configuration setting

EESSI · Sep 21, 2023 · 5bf10ba · 5bf10ba
1 parent 55e1ded
commit 5bf10ba
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 12 deletions.
diff --git a/app.cfg.example b/app.cfg.example
@@ -49,6 +49,10 @@ command_response_fmt =
 # name of the job script used for building an EESSI stack
 build_job_script = PATH_TO_EESSI_BOT/scripts/bot-build.slurm
 
+# path to directory on shared filesystem that can be used for sharing data across build jobs
+# (for example source tarballs used by EasyBuild)
+shared_fs_path = PATH_TO_SHARED_DIRECTORY
+
 # Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script
 build_logs_dir = PATH_TO_BUILD_LOGS_DIR
 

diff --git a/tasks/build.py b/tasks/build.py
@@ -72,6 +72,7 @@
 REPOS_CONTAINER = "container"
 REPO_TARGETS = "repo_targets"
 REPO_TARGET_MAP = "repo_target_map"
+SHARED_FS_PATH = "shared_fs_path"
 SLURM_PARAMS = "slurm_params"
 SUBMITTED_JOB_COMMENTS = "submitted_job_comments"
 SUBMIT_COMMAND = "submit_command"
@@ -120,6 +121,10 @@ def get_build_env_cfg(cfg):
     log(f"{fn}(): slurm_params '{slurm_params}'")
     config_data[SLURM_PARAMS] = slurm_params
 
+    shared_fs_path = buildenv.get(SHARED_FS_PATH)
+    log(f"{fn}(): shared_fs_path: '{shared_fs_path}'")
+    config_data[SHARED_FS_PATH] = shared_fs_path
+
     container_cachedir = buildenv.get(CONTAINER_CACHEDIR)
     log(f"{fn}(): container_cachedir '{container_cachedir}'")
     config_data[CONTAINER_CACHEDIR] = container_cachedir
@@ -488,6 +493,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     # create ini file job.cfg with entries:
     # [site_config]
     # local_tmp = LOCAL_TMP_VALUE
+    # shared_fs_path = SHARED_FS_PATH
     # build_logs_dir = BUILD_LOGS_DIR
     #
     # [repository]
@@ -502,18 +508,18 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     # os_type = OS_TYPE
     job_cfg = configparser.ConfigParser()
     job_cfg[JOB_SITECONFIG] = {}
-    if build_env_cfg[CONTAINER_CACHEDIR]:
-        job_cfg[JOB_SITECONFIG][CONTAINER_CACHEDIR] = build_env_cfg[CONTAINER_CACHEDIR]
-    if build_env_cfg[LOCAL_TMP]:
-        job_cfg[JOB_SITECONFIG][JOB_LOCAL_TMP] = build_env_cfg[LOCAL_TMP]
-    if build_env_cfg[HTTP_PROXY]:
-        job_cfg[JOB_SITECONFIG][JOB_HTTP_PROXY] = build_env_cfg[HTTP_PROXY]
-    if build_env_cfg[HTTPS_PROXY]:
-        job_cfg[JOB_SITECONFIG][JOB_HTTPS_PROXY] = build_env_cfg[HTTPS_PROXY]
-    if build_env_cfg[LOAD_MODULES]:
-        job_cfg[JOB_SITECONFIG][JOB_LOAD_MODULES] = build_env_cfg[LOAD_MODULES]
-    if build_env_cfg[BUILD_LOGS_DIR]:
-        job_cfg[JOB_SITECONFIG][BUILD_LOGS_DIR] = build_env_cfg[BUILD_LOGS_DIR]
+    build_env_to_job_cfg_keys = {
+        BUILD_LOGS_DIR: BUILD_LOGS_DIR,
+        CONTAINER_CACHEDIR: CONTAINER_CACHEDIR,
+        HTTP_PROXY: JOB_HTTP_PROXY,
+        HTTPS_PROXY: JOB_HTTPS_PROXY,
+        LOAD_MODULES: JOB_LOAD_MODULES,
+        LOCAL_TMP: JOB_LOCAL_TMP,
+        SHARED_FS_PATH: SHARED_FS_PATH,
+    }
+    for build_env_key, job_cfg_key in build_env_to_job_cfg_keys.items():
+        if build_env_cfg[build_env_key]:
+            job_cfg[JOB_SITECONFIG][job_cfg_key] = build_env_cfg[build_env_key]
 
     job_cfg[JOB_REPOSITORY] = {}
     # directory for repos.cfg