From 697c3c993090f6a8f7c697531e05e9a328308719 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 15 Feb 2024 11:35:25 +0100 Subject: [PATCH 1/3] Clean up unused checkouts on dev-desktops User might checkout repositories on the dev-desktops, work on them for a while, and then move on to other projects. The unused checkouts, in particular their `build` or `target` directories, can use up a lot of space that we want to reclaim periodically. A script has been created that finds all cache directories, checks recursively if any file in their root directory has been modified within a given time frame, and if not deletes the cache directory. --- .../files/clean-unused-checkouts.sh | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 ansible/roles/dev-desktop/files/clean-unused-checkouts.sh diff --git a/ansible/roles/dev-desktop/files/clean-unused-checkouts.sh b/ansible/roles/dev-desktop/files/clean-unused-checkouts.sh new file mode 100644 index 000000000..22addb1d3 --- /dev/null +++ b/ansible/roles/dev-desktop/files/clean-unused-checkouts.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Clean up unused checkouts +# +# This script is used to find old checkouts that are no longer in use. Given the +# size of the build directory, regularly cleaning them up can save significant +# amounts of disk space. + +# Enable strict mode for Bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Print directories and their size instead of deleting them +dry_run=false + +# Default to search for checkouts older than 60 days +time="60" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + dry_run=true + shift # past argument + ;; + -t|--time) + time="${2}" + shift # past argument + shift # past value + ;; + -*) + echo "Unknown option $1" + exit 1 + ;; + esac +done + +# Find all build or target directories created by users +# +# This command combines (`-o`) two different conditions to find all build and +# target directories that users have created. Within each home directory, we +# recursively look for directories that either have a file named `x.py` and a +# directory named `build`, or a file named `Cargo.toml` and a directory named +# `target`. +all_cache_directories=$(find /home -type d \( -name build -execdir test -f "x.py" \; -o -name target -execdir test -f "Cargo.toml" \; \) -print | sort | uniq) + +# For each checkout, we want to determine if the user has been working on it +# within the `$time` number of days. +unused_cache_directories=$(for directory in $all_cache_directories; do + project=$(dirname "${directory}") + + # Find all directories with files that have been modified less than $time days ago + modified=$(find "${project}" -type f -mtime -"${time}" -printf '%h\n' | xargs -r dirname | sort | uniq) + + # If no files have been modified in the last 90 days, then the project is + # considered old. + if [[ -z "${modified}" ]]; then + echo "${directory}" + fi +done) + +# Delete the build directories in the unused checkouts +for directory in $unused_cache_directories; do + if [[ "${dry_run}" == true ]]; then + du -sh "${directory}" + else + echo "Deleting ${directory}" + rm -rf "${directory}" + fi +done From caccac3a4f5ed679043b4e2f1bbdee4d1559b028 Mon Sep 17 00:00:00 2001 From: Jan David Date: Fri, 16 Feb 2024 14:04:20 +0100 Subject: [PATCH 2/3] Create cronjob to clean up disk space periodically A new cronjob has been created that looks for unused build artifacts and deletes them. The job runs every day at midnight. --- ansible/roles/dev-desktop/tasks/cleanup.yml | 18 ++++++++++++++++++ ansible/roles/dev-desktop/tasks/team_login.yml | 2 +- .../clean-unused-checkouts.sh | 4 ++++ .../templates/cron_cleanup_disk_space.j2 | 2 ++ .../{crontab_append => cron_team_login.j2} | 0 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/dev-desktop/tasks/cleanup.yml rename ansible/roles/dev-desktop/{files => templates}/clean-unused-checkouts.sh (98%) create mode 100644 ansible/roles/dev-desktop/templates/cron_cleanup_disk_space.j2 rename ansible/roles/dev-desktop/templates/{crontab_append => cron_team_login.j2} (100%) diff --git a/ansible/roles/dev-desktop/tasks/cleanup.yml b/ansible/roles/dev-desktop/tasks/cleanup.yml new file mode 100644 index 000000000..5d77eff5f --- /dev/null +++ b/ansible/roles/dev-desktop/tasks/cleanup.yml @@ -0,0 +1,18 @@ +--- +- name: Copy cleanup script + template: + src: clean-unused-checkouts.sh + dest: /etc/cron.cleanup_disk_space + owner: root + group: root + mode: 0744 + +- name: Set up the cleanup cron job + template: + src: cron_cleanup_disk_space.j2 + dest: /etc/cron.d/cleanup_disk_space + # if the cron job is running right now, keep retrying until it finishes + register: cleanup_cron_result + until: cleanup_cron_result is not failed + retries: 10 + delay: 5 diff --git a/ansible/roles/dev-desktop/tasks/team_login.yml b/ansible/roles/dev-desktop/tasks/team_login.yml index 72d1e9b75..a6f3e3ea8 100644 --- a/ansible/roles/dev-desktop/tasks/team_login.yml +++ b/ansible/roles/dev-desktop/tasks/team_login.yml @@ -13,7 +13,7 @@ - name: Set up the team login cron job template: - src: crontab_append + src: cron_team_login.j2 dest: /etc/cron.d/team_login # if the cron job is running right now, keep retrying until it finishes register: task_result diff --git a/ansible/roles/dev-desktop/files/clean-unused-checkouts.sh b/ansible/roles/dev-desktop/templates/clean-unused-checkouts.sh similarity index 98% rename from ansible/roles/dev-desktop/files/clean-unused-checkouts.sh rename to ansible/roles/dev-desktop/templates/clean-unused-checkouts.sh index 22addb1d3..4e7927ca6 100644 --- a/ansible/roles/dev-desktop/files/clean-unused-checkouts.sh +++ b/ansible/roles/dev-desktop/templates/clean-unused-checkouts.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# +# {{ ansible_managed }} +# + # Clean up unused checkouts # # This script is used to find old checkouts that are no longer in use. Given the diff --git a/ansible/roles/dev-desktop/templates/cron_cleanup_disk_space.j2 b/ansible/roles/dev-desktop/templates/cron_cleanup_disk_space.j2 new file mode 100644 index 000000000..e17b44264 --- /dev/null +++ b/ansible/roles/dev-desktop/templates/cron_cleanup_disk_space.j2 @@ -0,0 +1,2 @@ +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +0 0 * * * root /etc/cron.cleanup_disk_space diff --git a/ansible/roles/dev-desktop/templates/crontab_append b/ansible/roles/dev-desktop/templates/cron_team_login.j2 similarity index 100% rename from ansible/roles/dev-desktop/templates/crontab_append rename to ansible/roles/dev-desktop/templates/cron_team_login.j2 From 0b71a06e97ca130efb1f948fa7729767c758c1fc Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 5 Mar 2024 15:53:24 +0100 Subject: [PATCH 3/3] Run cleanup tasks as part of the dev-desktop role --- ansible/roles/dev-desktop/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/dev-desktop/tasks/main.yml b/ansible/roles/dev-desktop/tasks/main.yml index 8e0d6d372..08fd2ac71 100644 --- a/ansible/roles/dev-desktop/tasks/main.yml +++ b/ansible/roles/dev-desktop/tasks/main.yml @@ -7,6 +7,7 @@ - include_tasks: usermod.yml with_items: "{{ vars_user_config }}" when: vars_user_config is not none and vars_user_config | length > 0 +- include_tasks: cleanup.yml - include_tasks: team_login.yml - include_tasks: github.yml - include_tasks: motd.yml