[EPIC] Add backup strategy

Merge pull request #46 from trjohnson19/trjohnson19/44-backup-strategy-development

.env.example

@@ -16,6 +16,7 @@ TZ=
 USERDIR=  # Full path (i.e. "/home/<user>", not "~")
 DOCKERDIR=  # e.g. "${USERDIR}/docker"
 APPDIR=  # e.g. "${DOCKERDIR}/appdata"
+BACKUPDIR=  # e.g. "${DOCKERDIR}/backup"
 SECRETSDIR=  # e.g. "${DOCKERDIR}/secrets"
 DATADIR=  # e.g. "${DOCKERDIR}/data" or "/mnt/nas/dockerdata"
 # dotenv-linter:on UnorderedKey
@@ -224,3 +225,7 @@ NRIA_LICENSE_KEY=
 ##### MISCELLANEOUS
 NTP_SERVERS=pool.ntp.org,time.nist.gov,time.cloudflare.com
 WIREGUARD_PEERS=misc1,misc2,misc3
+
+##### BORG
+BORG_BACKUP_DIR="${DOCKERDIR}"
+BORG_PATTERNFILE="${DOCKERDIR}/backup-scripts/borg-patternfile.lst"

.gitignore

@@ -179,6 +179,16 @@ appdata/unbound/config/*
 !appdata/unbound/config/root.hints.example.cron
 !appdata/unbound/config/unbound.conf
 
+!backup-scripts
+backup-scripts/*
+!backup-scripts/.passwords.example.env
+!backup-scripts/backup-*.sh
+!backup-scripts/borg-patternfile.example.lst
+!backup-scripts/cron.sh
+
+!backups
+backups/*
+
 # Include the .github directory
 !.github
 !.github/*

backup-scripts/.passwords.example.env

@@ -0,0 +1,8 @@
+# dotenv-linter:off UnorderedKey
+MARIADB_ROOT_USER=
+MARIADB_ROOT_PASSWORD=
+POSTGRES_ROOT_USER=
+BORG_REPO=
+BORG_SSH_ID_KEY=
+BORG_PASSPHRASE=
+# dotenv-linter:on UnorderedKey

backup-scripts/backup-borg.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#TODO(trjohnson19): rewrite in Python for maintainability
+
+function help_function() {
+	echo "Error parsing arguments"
+	echo ""
+	echo "Usage: $0 -r 'ssh://repo' -i '/path/to/id_key' -p 'borg_passphrase' -f '/path/to/patternfile' -d '/path/to/backupdir'"
+	echo -e "\t-r Borg backup repo"
+	echo -e "\t-i Borg backup ssh id_key. Default ''"
+	echo -e "\t-p Borg backup passphrase. Default ''"
+	echo -e "\t-f Borg backup patternfile"
+	echo -e "\t-d Borg backup dir"
+	exit 0 # Exit script after printing help
+}
+
+if [[ "$#" -eq 0 ]]; then
+	help_function
+fi
+
+while getopts d:f:h:i:p:r: flag; do
+	case "${flag}" in
+	d) borg_backup_dir="${OPTARG}" ;;
+	f) borg_patternfile="${OPTARG}" ;;
+	h) help_function ;;
+	i) borg_ssh_id_key="${OPTARG}" ;;
+	p) borg_passphrase="${OPTARG}" ;;
+	r) borg_repo="${OPTARG}" ;;
+	*) help_function ;;
+	esac
+done
+
+# Setting this, so the repo does not need to be given on the commandline:
+export BORG_REPO="${borg_repo}"
+
+# Setting this to ensure appropriate ssh key is used:
+if [[ -n $borg_ssh_id_key ]]; then
+	export BORG_RSH="ssh -i ${borg_ssh_id_key} -oBatchMode=yes"
+else
+	export BORG_RSH="ssh -oBatchMode=yes"
+fi
+
+# See the section "Passphrase notes" for more infos.
+export BORG_PASSPHRASE="${borg_passphrase}"
+
+# some helpers and error handling:
+info() {
+	printf "\n%s %s\n\n" "$(date)" "$*" >&2
+}
+trap 'echo $(date) Backup interrupted >&2; exit 2' INT TERM
+
+info "Starting backup"
+
+# Backup the most important directories into an archive named after
+# the machine this script is currently running on:
+
+borg create \
+	--verbose \
+	--filter AME \
+	--list \
+	--stats \
+	--show-rc \
+	--compression auto,zstd,9 \
+	--exclude-caches \
+	--patterns-from "${borg_patternfile}" \
+	\
+	::'{hostname}-{now}' \
+	"${borg_backup_dir}"
+
+backup_exit=$?
+
+info "Pruning repository"
+
+# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
+# archives of THIS machine. The '{hostname}-' prefix is very important to
+# limit prune's operation to this machine's archives and not apply to
+# other machines' archives also:
+
+borg prune \
+	--list \
+	--glob-archives '{hostname}-*' \
+	--show-rc \
+	--keep-daily 7 \
+	--keep-weekly 4 \
+	--keep-monthly 6
+
+prune_exit=$?
+
+# Actually free repo disk space by compacting segments
+
+info "Compacting repository"
+
+borg compact
+
+compact_exit=$?
+
+# use highest exit code as global exit code
+global_exit=$((backup_exit > prune_exit ? backup_exit : prune_exit))
+global_exit=$((compact_exit > global_exit ? compact_exit : global_exit))
+
+if [ ${global_exit} -eq 0 ]; then
+	info "Backup, Prune, and Compact finished successfully"
+elif [ ${global_exit} -eq 1 ]; then
+	info "Backup, Prune, and/or Compact finished with warnings"
+else
+	info "Backup, Prune, and/or Compact finished with errors"
+fi
+
+exit ${global_exit}

backup-scripts/backup-mariadb.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+#TODO(trjohnson19): rewrite in Python for maintainability
+
+function help_function() {
+	echo "Error parsing arguments"
+	echo ""
+	echo "Usage: $0 -u 'root' -p 'password' -d '.' -f 'dump.sql'"
+	echo -e "\t-u MariaDB root username. Default 'root'"
+	echo -e "\t-p MariaDB root password. Default ''"
+	echo -e "\t-d Backup directory. Default \"\${PWD}\""
+	echo -e "\t-f Backup filename. Default 'dump.sql'"
+	exit 0 # Exit script after printing help
+}
+
+if [[ "$#" -eq 0 ]]; then
+	help_function
+fi
+
+while getopts d:f:h:p:u: flag; do
+	case "${flag}" in
+	d) backup_dir="${OPTARG}" ;;
+	f) backup_filename="${OPTARG}" ;;
+	h) help_function ;;
+	p) mariadb_root_password="${OPTARG}" ;;
+	u) mariadb_root_user="${OPTARG}" ;;
+	*) help_function ;;
+	esac
+done
+
+mariadb_root_user="${mariadb_root_user:-root}"
+backup_dir="${backup_dir:-${PWD}}"
+backup_filename="${backup_filename:-dump.sql}"
+
+mariadb_dump_params=(
+	--user="${mariadb_root_user}"
+	--all-databases
+	--lock-all-tables
+	# --verbose
+)
+
+if [[ -v mariadb_root_password ]]; then
+	mariadb_dump_params+=(
+		--password="${mariadb_root_password}"
+	)
+fi
+
+/usr/bin/docker exec --user root mariadb /usr/bin/mariadb-dump \
+	"${mariadb_dump_params[@]}" >"${backup_dir}/${backup_filename}"
+backup_exit=$?
+
+if [[ $backup_exit -ne 0 ]]; then
+	echo "[ERROR] Unknown error occurred, backup cannot be relied upon"
+	echo "[ERROR] Erroneous MariaDB dump saved to: ${backup_dir}/${backup_filename}"
+	exit "${backup_exit}"
+else
+	echo "MariaDB dump saved to: ${backup_dir}/${backup_filename}"
+	exit 0
+fi

backup-scripts/backup-postgres.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+#TODO(trjohnson19): rewrite in Python for maintainability
+
+function help_function() {
+	echo "Error parsing arguments"
+	echo ""
+	echo "Usage: $0 -u 'postgres' -d '.' -f 'dump.sql'"
+	echo -e "\t-u Postgres root username. Default 'postgres'"
+	echo -e "\t-d Backup directory. Default \"\${PWD}\""
+	echo -e "\t-f Backup filename. Default 'dump.sql'"
+	exit 0 # Exit script after printing help
+}
+
+if [[ "$#" -eq 0 ]]; then
+	help_function
+fi
+
+while getopts d:f:h:u: flag; do
+	case "${flag}" in
+	d) backup_dir="${OPTARG}" ;;
+	f) backup_filename="${OPTARG}" ;;
+	h) help_function ;;
+	u) postgres_root_user="${OPTARG}" ;;
+	*) help_function ;;
+	esac
+done
+
+postgres_root_user="${postgres_root_user:-postgres}"
+backup_dir="${backup_dir:-${PWD}}"
+backup_filename="${backup_filename:-dump.sql}"
+
+postgres_dump_params=(
+	--username="${postgres_root_user}"
+	--no-password
+	# --verbose
+)
+
+/usr/bin/docker exec --user root postgres /usr/bin/pg_dumpall \
+	"${postgres_dump_params[@]}" >"${backup_dir}/${backup_filename}"
+backup_exit=$?
+
+if [[ $backup_exit -ne 0 ]]; then
+	echo "[ERROR] Unknown error occurred, backup cannot be relied upon"
+	echo "[ERROR] Erroneous Postgres dump saved to: ${backup_dir}/${backup_filename}"
+	exit "${backup_exit}"
+else
+	echo "Postgres dump saved to: ${backup_dir}/${backup_filename}"
+	exit 0
+fi