diff --git a/README.md b/README.md index 9832e271..317dc921 100644 --- a/README.md +++ b/README.md @@ -67,13 +67,13 @@ At the [app settings page](https://github.com/settings/apps) click "`New GitHub python3 -c 'import secrets; print(secrets.token_hex(64))' ``` - Permissions: assign the required permissions to the app (e.g., read access to commits, issues, pull requests); - - Make sure to assign read and write access to the Pull request in "Repository permissions" section; these permisions can be changed later on; + - Make sure to assign read and write access to the Pull requests and Issues in "Repository permissions" section; these permisions can be changed later on; - Make sure to accept the new permissions from the "Install App" section that you can reach via the menu on the left hand side. - Then select the wheel right next to your installed app, or use the link `https://github.com/settings/installations/INSTALLATION_ID` - Once the page is open you will be able to accept the new permissions there. - Some permissions (e.g., metadata) will be selected automatically because of others you have chosen. -- Events: subscribe the app to events it shall react on (e.g., related to pull requests) +- Events: subscribe the app to events it shall react on (e.g., related to pull requests and comments) - Select that the app can only be installed by this (your) GitHub account or organisation. Click on "`Create GitHub App`" to complete this step. diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 453a0cdc..d53b5a0d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,7 +1,21 @@ This file contains a description of the major changes to the EESSI build-and-deploy bot. For more detailed information, please see the git log. -v0.1.0 (29 september 2023) +v0.1.1 (14 November 2023) +-------------------------- + +This is a patch release of the EESSI build-and-deploy bot. + +Bug fixes: +* omit header in output of `squeue` command run by the job manager (#220) +* make bot compatible with more recent versions of PyGithub (#224) + +Improvements: +* added a script to help reducing disk usage by cleaning up tarballs stored by jobs (#217) +* clarified documentation about setting permissions and event subscriptions for the bot (#221) + + +v0.1.0 (29 September 2023) -------------------------- This is the first release of the EESSI build-and-deploy bot, which consists of diff --git a/connections/github.py b/connections/github.py index 20271c66..2204fe89 100644 --- a/connections/github.py +++ b/connections/github.py @@ -11,11 +11,11 @@ # # Standard library imports -import datetime +from datetime import datetime, timezone import time # Third party imports (anything installed into the local Python environment) -from github import Github, GithubIntegration +import github # Local application imports (anything from EESSI/eessi-bot-software-layer) from tools import config, logging @@ -57,7 +57,7 @@ def get_token(): # If the config keys are not set, get_access_token will raise a NotImplementedError # Returning NoneType token will stop the connection in get_instance try: - github_integration = GithubIntegration(app_id, private_key) + github_integration = github.GithubIntegration(app_id, private_key) _token = github_integration.get_access_token(installation_id) break except NotImplementedError as err: @@ -84,7 +84,7 @@ def connect(): Returns: Instance of Github """ - return Github(get_token().token) + return github.Github(get_token().token) def get_instance(): @@ -101,7 +101,16 @@ def get_instance(): global _gh, _token # TODO Possibly renew token already if expiry date is soon, not only # after it has expired. - if not _gh or (_token and datetime.datetime.utcnow() > _token.expires_at): + + # Check if PyGithub version is < 1.56 + if hasattr(github, 'GithubRetry'): + # Pygithub 2.x + time_now = datetime.now(timezone.utc) + else: + # Pygithub 1.x + time_now = datetime.utcnow() + + if not _gh or (_token and time_now > _token.expires_at): _gh = connect() return _gh diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index e1936c71..30be3626 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -106,7 +106,7 @@ def get_current_jobs(self): if username is None: raise Exception("Unable to find username") - squeue_cmd = "%s --long --user=%s" % (self.poll_command, username) + squeue_cmd = "%s --long --noheader --user=%s" % (self.poll_command, username) squeue_output, squeue_err, squeue_exitcode = run_cmd( squeue_cmd, "get_current_jobs(): squeue command", @@ -125,10 +125,10 @@ def get_current_jobs(self): } # get job info, logging any Slurm issues - # Note, the first two lines of the output are skipped ("range(2,...)") - # because they contain header information. - for i in range(2, len(lines)): - job = lines[i].rstrip().split() + # Note, all output lines of squeue are processed because we run it with + # --noheader. + for line in lines: + job = line.rstrip().split() if len(job) >= 9: job_id = job[0] state = job[4] diff --git a/scripts/cleanup_pr.sh b/scripts/cleanup_pr.sh new file mode 100755 index 00000000..41527675 --- /dev/null +++ b/scripts/cleanup_pr.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# +# GitHub App for the EESSI project +# +# A bot to help with requests to add software installations to the EESSI software layer, +# see https://github.com/EESSI/software-layer + +# This script cleans up (deletes) all build artefacts and temporary storage tarballs of a given PR +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +SCRIPT_DIR=$(dirname $(realpath $BASH_SOURCE)) + +function display_help +{ + echo "Usage: $0 [OPTIONS] " >&2 + echo " -b | --jobs-base-dir DIRECTORY - jobs base directory [default: reads" >&2 + echo " value from bot config file app.cfg or .]" >&2 + echo " -D | --dry-run - only show commands that would be run" >&2 + echo " [default: false]" >&2 + echo " -h | --help - display this usage information" >&2 +} + +function get_jobs_base_dir +{ + app_cfg_path=${1} + grep jobs_base_dir ${app_cfg_path} | grep -v '^[ ]*#' | sed -e 's/^[^=]*=[ ]*//' +} + +echo + +if [[ $# -lt 1 ]]; then + display_help + exit 1 +fi + +# process command line args +POSITIONAL_ARGS=() + +jobs_base_dir= +dry_run=false + +while [[ $# -gt 0 ]]; do + case $1 in + -b|--jobs-base-dir) + if [[ $# -gt 1 ]]; then + jobs_base_dir="$2" + shift 2 + else + echo "Error: missing argument (directory) for parameter '${1}'" + exit 2 + fi + ;; + -D|--dry-run) + dry_run=true + shift 1 + ;; + -h|--help) + display_help + exit 0 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +# restore potentially parsed filename(s) into $* +set -- "${POSITIONAL_ARGS[@]}" + +if [[ $# -ne 1 ]]; then + echo "Error: exactly one PR number should be provided as argument" + display_help + exit 3 +fi + +pull_request=${1} + +if ${dry_run} = true ; then + echo "DRY_RUN: not removing any files" +fi + +# determine jobs base dir if not given explicitly +# 1. check for file app.cfg in SCRIPT_DIR +# 2. check for file app.cfg in current dir +# if found try to obtain value of jobs_base_dir setting +# if not file not found or jobs_base_dir setting not found (or empty) --> error & exit +if [[ -z ${jobs_base_dir} ]]; then + echo "jobs base directory not given explicitly, trying to determine it" + if [[ -e ${SCRIPT_DIR}/app.cfg ]]; then + echo "check for app.cfg in '${SCRIPT_DIR}'" + jobs_base_dir=$(get_jobs_base_dir ${SCRIPT_DIR}/app.cfg) + else + if [[ -e ./app.cfg ]]; then + echo "check for app.cfg in '${PWD}' (current directory)" + jobs_base_dir=$(get_jobs_base_dir ./app.cfg) + fi + fi +fi +if [[ -z ${jobs_base_dir} ]]; then + echo "Error: jobs base directory is empty, please specify it as argument" + display_help + exit 4 +fi + +echo "processing all directories for PR ${pull_request}:" +find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request} | sed -e 's/^/ /' + +echo +echo "disk usage of directories for PR ${pull_request} BEFORE removing build artefacts and tmp storage" +for d in $(find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request}); do du -sh $d; done + +echo +echo "$([[ ${dry_run} = true ]] && echo "DRY_RUN: ")removing tmp storage tarballs for PR ${pull_request}" +for d in $(find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request}) +do + for f in $(find $d -type f -wholename "*[0-9].tgz") + do + if ${dry_run} = true ; then + echo "DRY_RUN: rm '$f' ($(ls -lh $f | awk '{print $5}'))" + else + echo "Removing file '$f'" + rm $f + fi + done +done + +echo +echo "disk usage of directories for PR ${pull_request} AFTER removing tmp storage tarballs" +for d in $(find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request}); do du -sh $d; done + +echo +echo "$([[ ${dry_run} = true ]] && echo "DRY_RUN: ")removing build artefacts for PR ${pull_request}" +for d in $(find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request}) +do + for f in $(find $d -type f -wholename "*tar.gz") + do + if ${dry_run} = true ; then + echo "DRY_RUN: rm '$f' ($(ls -lh $f | awk '{print $5}'))" + else + echo "Removing file '$f'" + rm $f + fi + done +done + +echo +echo "disk usage of directories for PR ${pull_request} AFTER removing build artefacts and tmp storage" +for d in $(find ${jobs_base_dir}/* -maxdepth 1 -type d -wholename */pr_${pull_request}); do du -sh $d; done +