From 7e2892d3f5f51ea826e227e3f19714e0b3db5d79 Mon Sep 17 00:00:00 2001 From: Angelina Uno-Antonison Date: Fri, 19 Apr 2024 14:09:12 -0500 Subject: [PATCH] Backup and Restore a Rosalution DB to and from LTS (#169) * Updated the scripts to use the expected cgdslts remote source for rclone for the s3 bucket * Update etc/database/restore-database-lts.sh Co-authored-by: James Scherer Signed-off-by: Angelina Uno-Antonison --------- Signed-off-by: Angelina Uno-Antonison Co-authored-by: James Scherer --- .gitignore | 4 +- etc/database/backup-database-lts.sh | 62 ++++++++++++++++++++ etc/database/restore-database-lts.sh | 85 ++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100755 etc/database/backup-database-lts.sh create mode 100755 etc/database/restore-database-lts.sh diff --git a/.gitignore b/.gitignore index 4d42c8d2..dc1d1762 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,6 @@ media/ # SSL/TLS Certificates **/.certificates/** -*.pem \ No newline at end of file +*.pem + +etc/.rosalution-db-backups \ No newline at end of file diff --git a/etc/database/backup-database-lts.sh b/etc/database/backup-database-lts.sh new file mode 100755 index 00000000..6270473f --- /dev/null +++ b/etc/database/backup-database-lts.sh @@ -0,0 +1,62 @@ +#! /bin/bash +# ./backup-database-lts.sh +usage() { + echo "usage: $0 " + echo " Backups a local MongoDB dump archive into UAB Long Term Storage remote" + echo " source 'cgdslts'." + echo " " + echo " Visit https://docs.rc.uab.edu/data_management/transfer/rclone/#setting-up-an-s3-lts-remote" + echo " for setup instructions for interacting with UAB LTS using rclone. Use 'cgdslts'" + echo " instead of 'uablts' for the remote configured in the setup instructions." +} + +fail=false + +if ! rclone --version &> /dev/null +then + echo "Error: rclone could not be found." + fail=true +fi + +if [[ $# -ne 1 ]] +then + echo "Error: required input missing." + fail=true +fi + +if $fail +then + echo "Exiting script ..." + usage + exit 1 +fi + +TARGET_LOCAL_SOURCE_DATABASE_DUMP_FILEPATH=$1 +TARGET_S3_REMOTE="cgdslts" + +if [ ! -f "$TARGET_LOCAL_SOURCE_DATABASE_DUMP_FILEPATH" ] +then + echo "Error: $TARGET_LOCAL_SOURCE_DATABASE_DUMP_FILEPATH does not exist" + usage + exit 1 +fi + +if ! rclone listremotes | grep -q "$TARGET_S3_REMOTE" +then + echo "Missing expected '$TARGET_S3_REMOTE' rclone remote; aborting operation" + usage + exit 1 +fi + +if ! rclone lsf "$TARGET_S3_REMOTE:" | grep -q rosalution +then + echo "Missing expected root path 'rosalution/' in bucket" + usage + exit 1 +fi + + +destination_backup_path="$TARGET_S3_REMOTE:rosalution/db-backup/" +rclone copy -P "$TARGET_LOCAL_SOURCE_DATABASE_DUMP_FILEPATH" "$destination_backup_path" + +echo "Backup operation complete..." \ No newline at end of file diff --git a/etc/database/restore-database-lts.sh b/etc/database/restore-database-lts.sh new file mode 100755 index 00000000..e03db352 --- /dev/null +++ b/etc/database/restore-database-lts.sh @@ -0,0 +1,85 @@ +#! /bin/bash +# ./restore-database-lts.sh clean +usage() { + echo "usage: $0 clean(optional)" + echo " Restores an archived 'rosalution_db' MongoDB Database for Rosalution, into the specified" + echo " MongoDB docker container from the CGDS UAB Long Term Storage remote source 'cgdslts'." + echo " " + echo " Visit https://docs.rc.uab.edu/data_management/transfer/rclone/#setting-up-an-s3-lts-remote" + echo " for setup instructions for interacting with UAB LTS using rclone. Use 'cgdslts'" + echo " instead of 'uablts' for the remote configured in the setup instructions." +} + +OPTIONAL_CLEAN=false + +fail=false + +if ! rclone --version &> /dev/null +then + echo "Error: rclone could not be found." + fail=true +fi + +if [[ $# -lt 1 ]] +then + echo "Error: required input missing." + fail=true +fi + +if [[ $# -ge 2 && "$2" == "clean" ]] +then + OPTIONAL_CLEAN=true +fi + +if $fail +then + echo "Exiting script ..." + usage + exit 1 +fi + +DOCKER_CONTAINER=$1 +TARGET_S3_REMOTE="cgdslts" + +if ! rclone listremotes | grep -q "$TARGET_S3_REMOTE" +then + echo "Missing expected '$TARGET_S3_REMOTE' rclone remote; aborting operation" + usage + exit 1 +fi + +if ! rclone lsf "$TARGET_S3_REMOTE:" | grep -q rosalution +then + echo "Missing expected root path 'rosalution/' in bucket" + exit 1 +fi + +echo "Available Rosalution Backups" +echo "------------------------------------------------------" +rclone lsf cgdslts:rosalution/db-backup --files-only | sort +echo "------------------------------------------------------" + +echo "Which backup would you like to restore?" +read -r rosalution_db_backup + +backup_absolute_path="$TARGET_S3_REMOTE:rosalution/db-backup/$rosalution_db_backup" +local_destination_directory=".rosalution-db-backups" +local_relative_backup_path="$local_destination_directory/$rosalution_db_backup" +if ! rclone lsf "$backup_absolute_path" | grep -q "$rosalution_db_backup" +then + echo "Rosalution Backup '$backup_absolute_path' does not exist in CGDS' UAB LTS." +fi + +mkdir -p .rosalution-db-backups + +rclone copy -P "$backup_absolute_path" "$local_destination_directory" + +./restore-database.sh "$DOCKER_CONTAINER" "$local_relative_backup_path" + +if [ "$OPTIONAL_CLEAN" = true ] +then + echo "Removing local backup copy..." + rm "$local_relative_backup_path" +fi + +echo "Restore operation complete..." \ No newline at end of file