forked from EESSI/software-layer
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #60 from trz42/sync_nessi_main_with_eessi_main
Pull EESSI/main into NESSI/main
- Loading branch information
Showing
16 changed files
with
1,416 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions | ||
| name: Tests for eessi_container.sh script | ||
| on: [push, pull_request, workflow_dispatch] | ||
| permissions: | ||
| contents: read # to fetch code (actions/checkout) | ||
| jobs: | ||
| eessi_container_script: | ||
| runs-on: ubuntu-20.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| SCRIPT_TEST: | ||
| - help | ||
| - listrepos_default | ||
| - listrepos_custom | ||
| - run | ||
| - shell | ||
| - container | ||
| - resume | ||
| # FIXME disabled because '--access rw' is not working in CI environment | ||
| #- readwrite | ||
| #- save | ||
| steps: | ||
| - name: Check out software-layer repository | ||
| uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 | ||
|
|
||
| - name: install Apptainer | ||
| run: | | ||
| ./install_apptainer_ubuntu.sh | ||
| - name: Collect info on test environment | ||
| run: | | ||
| mount | ||
| df -h | ||
| - name: Test eessi_container.sh script | ||
| run: | | ||
| test_cmd="cat /etc/os-release" | ||
| out_pattern="Debian GNU/Linux 11" | ||
| if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then | ||
| ./eessi_container.sh --help | ||
| # test use of --list-repos without custom repos.cfg | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then | ||
| outfile=out_listrepos.txt | ||
| ./eessi_container.sh --verbose --list-repos | tee ${outfile} | ||
| grep "EESSI-pilot" ${outfile} | ||
| # test use of --list-repos with custom repos.cfg | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then | ||
| outfile=out_listrepos.txt | ||
| outfile2=out_listrepos_2.txt | ||
| mkdir -p ${PWD}/cfg | ||
| echo "[EESSI/20AB.CD]" > cfg/repos.cfg | ||
| echo "repo_version = 20AB.CD" >> cfg/repos.cfg | ||
| echo "[EESSI/20HT.TP]" >> cfg/repos.cfg | ||
| echo "repo_version = 20HT.TP" >> cfg/repos.cfg | ||
| ./eessi_container.sh --verbose --list-repos | tee ${outfile} | ||
| grep "EESSI-pilot" ${outfile} | ||
| export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg | ||
| ./eessi_container.sh --verbose --list-repos | tee ${outfile2} | ||
| grep "[EESSI/2023.02]" ${outfile2} | ||
| # test use of --mode run | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then | ||
| outfile=out_run.txt | ||
| echo "${test_cmd}" > test_script.sh | ||
| chmod u+x test_script.sh | ||
| export SINGULARITY_BIND="$PWD:/test" | ||
| ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile} | ||
| grep "${out_pattern}" ${outfile} | ||
| # test use of --mode shell | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then | ||
| outfile=out_shell.txt | ||
| ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} | ||
| grep "${out_pattern}" ${outfile} | ||
| # test use of --container option, using a totally different container; | ||
| # cfr. https://github.com/easybuilders/easybuild-containers | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then | ||
| outfile=out_container.txt | ||
| container="docker://ghcr.io/eessi/build-node:debian10" | ||
| ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} | ||
| grep "Debian GNU/Linux 10" ${outfile} | ||
| # test use of '--access rw' to get write access in container | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then | ||
| outfile=out_readwrite.txt | ||
| fn="test_${RANDOM}.txt" | ||
| echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh | ||
| chmod u+x test_script.sh | ||
| export SINGULARITY_BIND="$PWD:/test" | ||
| ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} | ||
| tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") | ||
| # note: must use '--access rw' again here, since touched file is in overlay upper dir | ||
| ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} | ||
| grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile | ||
| # test use of --resume | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then | ||
| outfile=out_resume.txt | ||
| ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile} | ||
| tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") | ||
| rm -f ${outfile} | ||
| # make sure that container image exists | ||
| test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1) | ||
| ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} | ||
| cat ${outfile} | ||
| grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} | ||
| grep "${out_pattern}" ${outfile} | ||
| # test use of --save (+ --resume) | ||
| elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then | ||
| outfile=out_save.txt | ||
| fn="test_${RANDOM}.txt" | ||
| test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" | ||
| ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} | ||
| rm -f ${outfile} | ||
| ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} | ||
| grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile | ||
| tar tfv test-save.tar | grep "overlay-upper/${fn}" | ||
| else | ||
| echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 | ||
| exit 1 | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| # Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 | ||
|
|
||
| [github] | ||
| # replace '123456' with the ID of your GitHub App | ||
| app_id = 281041 | ||
|
|
||
| # a short (!) name for your app instance that can be used for example | ||
| # when adding/updating a comment to a PR | ||
| # (!) a short yet descriptive name is preferred because it appears in | ||
| # comments to the PR | ||
| # for example, the name could include the name of the cluster the bot | ||
| # runs on and the username which runs the bot | ||
| # NOTE avoid putting an actual username here as it will be visible on | ||
| # potentially publicly accessible GitHub pages. | ||
| app_name = eessi-bot-citc-aws | ||
|
|
||
| # replace '12345678' with the ID of the installation of your GitHub App | ||
| # (can be derived by creating an event and then checking for the list | ||
| # of sent events and its payload either via the Smee channel's web page | ||
| # or via the Advanced section of your GitHub App on github.com) | ||
| installation_id = 33078935 | ||
|
|
||
| # path to the private key that was generated when the GitHub App was registered | ||
| private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem | ||
|
|
||
|
|
||
| [buildenv] | ||
| # name of the job script used for building an EESSI stack | ||
| build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm | ||
|
|
||
| # The container_cachedir may be used to reuse downloaded container image files | ||
| # across jobs. Thus, jobs can more quickly launch containers. | ||
| container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir | ||
|
|
||
| # it may happen that we need to customize some CVMFS configuration | ||
| # the value of cvmfs_customizations is a dictionary which maps a file | ||
| # name to an entry that needs to be added to that file | ||
| cvmfs_customizations = {} | ||
|
|
||
| # if compute nodes have no internet connection, we need to set http(s)_proxy | ||
| # or commands such as pip3 cannot download software from package repositories | ||
| # for example, the temporary EasyBuild is installed via pip3 first | ||
| # http_proxy = http://PROXY_DNS:3128/ | ||
| # https_proxy = http://PROXY_DNS:3128/ | ||
|
|
||
| # directory under which the bot prepares directories per job | ||
| # structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR | ||
| jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs | ||
|
|
||
| # configure environment | ||
| # list of comma-separated modules to be loaded by build_job_script | ||
| # useful/needed if some tool is not provided as system-wide package | ||
| # (read by bot and handed over to build_job_script via parameter | ||
| # --load-modules) | ||
| load_modules = | ||
|
|
||
| # PATH to temporary directory on build node ... ends up being used for | ||
| # for example, EESSI_TMPDIR --> /tmp/$USER/EESSI | ||
| # escaping variables with '\' delays expansion to the start of the | ||
| # build_job_script; this can be used for referencing environment | ||
| # variables that are only set inside a Slurm job | ||
| local_tmp = /tmp/$USER/EESSI | ||
|
|
||
| # parameters to be added to all job submissions | ||
| # NOTE do not quote parameter string. Quotes are retained when reading in config and | ||
| # then the whole 'string' is recognised as a single parameter. | ||
| # NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs | ||
| # to be initialised as if it is for a login shell. | ||
| # note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed | ||
| slurm_params = --hold --time=24:0:0 | ||
|
|
||
| # full path to the job submission command | ||
| submit_command = /usr/bin/sbatch | ||
|
|
||
| # which GH account has the permission to trigger the build (by setting | ||
| # the label 'bot:build' (apparently this cannot be restricted on GitHub) | ||
| # if value is left/empty everyone can trigger the build | ||
| # value can be a space delimited list of GH accounts | ||
| build_permission = boegel trz42 bedroge | ||
|
|
||
| [architecturetargets] | ||
| # defines both for which architectures the bot will build | ||
| # and what submission parameters shall be used | ||
| # medium instances (8 cores, 16GB RAM) | ||
| #arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} | ||
| # larger instances (16 cores, 32GB RAM) | ||
| arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} | ||
|
|
||
| [repo_targets] | ||
| # defines for which repository a arch_target should be build for | ||
| # | ||
| # only building for repository EESSI-pilot | ||
| repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} | ||
|
|
||
| # points to definition of repositories (default EESSI-pilot defined by build container) | ||
| repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles | ||
|
|
||
| # configuration for event handler which receives events from a GitHub repository. | ||
| [event_handler] | ||
| # path to the log file to log messages for event handler | ||
| log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log | ||
|
|
||
|
|
||
| [job_manager] | ||
| # path to the log file to log messages for job manager | ||
| log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log | ||
|
|
||
| # directory where job manager stores information about jobs to be tracked | ||
| # e.g. as symbolic link JOBID -> directory to job | ||
| job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs | ||
|
|
||
| # full path to the job status checking command | ||
| poll_command = /usr/bin/squeue | ||
|
|
||
| # polling interval in seconds | ||
| poll_interval = 60 | ||
|
|
||
| # full path to the command for manipulating existing jobs | ||
| scontrol_command = /usr/bin/scontrol | ||
|
|
||
| [deploycfg] | ||
| # script for uploading built software packages | ||
| tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging | ||
|
|
||
| # URL to S3/minio bucket | ||
| # if attribute is set, bucket_base will be constructed as follows | ||
| # bucket_base=${endpoint_url}/${bucket_name} | ||
| # otherwise, bucket_base will be constructed as follows | ||
| # bucket_base=https://${bucket_name}.s3.amazonaws.com | ||
| # - The former variant is used for non AWS S3 services, eg, minio, or when | ||
| # the bucket name is not provided in the hostname (see latter case). | ||
| # - The latter variant is used for AWS S3 services. | ||
| #endpoint_url = URL_TO_S3_SERVER | ||
|
|
||
| # bucket name | ||
| bucket_name = eessi-staging | ||
|
|
||
| # upload policy: defines what policy is used for uploading built artefacts | ||
| # to an S3 bucket | ||
| # 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) | ||
| # 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) | ||
| # only upload the latest built artefact | ||
| # 'once' : only once upload any built artefact for the build target | ||
| # 'none' : do not upload any built artefacts | ||
| upload_policy = once | ||
|
|
||
| # which GH account has the permission to trigger the deployment (by setting | ||
| # the label 'bot:deploy' (apparently this cannot be restricted on GitHub) | ||
| # if value is left/empty everyone can trigger the deployment | ||
| # value can be a space delimited list of GH accounts | ||
| deploy_permission = boegel trz42 bedroge |
Oops, something went wrong.