vps_backup

#!/bin/sh
#
# To put in Synology /etc/crontab:
#
#   0       */2     *       *       *       root    /volumeX/vpsbackups/scripts/vps_backup >> /volume3/VPS/logs/backup-log.txt 2>&1
#
#
#
# A push backup scheme that creates snapshot "rsync" backup of your remote
# server and stores it on your Synology NAS or other Linux based server.
# After a backup is complete other rolling hard linked snapshots are created
# generating cycles of month, week, day and hourly snapshots.
#
# The script will perform a backup of the SOURCE directory on the remote server
# using rsync and create the monthly, weekly, daily and hourly snapshots 
# afterwards. In case of a failure, a notification will be send to the 
# Synology dashboard and a log entry will be made in the Synology log.
# A log of all performed backups can be viewed in the Synology Diskstation 
# dashboard -> Log center -> Log Search.
# 
# A more detailed log called 'backup-log.txt' will be stored in the 'logs' directory.
#
# DIRECTORY STRUCTURE
#
# /volumeX/vpsbackups (or any other name)
#  ↳ backups
#  ↳ logs
#  ↳ scripts
#     ↳ linkdups
#     ↳ vps_backup
#     ↳ vps_backup_exclude
#     ↳ vps_backup_prep
#
#
# The script can also be used to backup a remote server to a non-Synology device.
# In that case, remove all references to Synology features (synolog and synodsmnotify)
#
# 
# TO PERFORM A MANUAL BACKUP
#
# > /volumeX/vpsbackups/scripts/vps_backup
#
# OPTIONS
#    -u   update only, do not 'roll' backup into snapshot backup cycles
#    -r   roll snapshots only (as needed), do not update the primary backup
#
# Extra script/data files used by this script...
#   home_backup_exclude  # exclude (and include) list of files (data caches)
#   home_backup_prep     # prepare backup on backup server (make linked copy)
#
####
#
# The actual backup method involves the way "rsync" updates files that change.
# When that happens any hardlink that is attached to that file in the backup
# directory is broken.  As such the older 'snapshot' of the file is preserved
# while the "current" backup gets a new version of the file, (the whole file).
#
# This also means any files which are unmodified from the last "snapshot" to
# the next remain hard linked together, so only one physical copy of each file
# version is kept on disk, resulting in enormous disk space savings.  The disk
# cost (not counting inodes) is the directory structure, and any files which
# are either new or had been modified.
#
# A detail discussion and examples of this technique is provided on the web
#  Easy Snapshot-Style Backups with Rsync
#     http://www.mikerubel.org/computers/rsync_snapshots/
#
# There is one caveat with this system.  When a file is moved, renamed, or
# worse a whole directory is renamed, rsync will also break the hardlink, even
# though the file itself has not changed.  As such a secondary method of
# 're-linking' moved or renamed files is needed. This is achieved using the
# "linkdups" script, and performed by the "home_backup_roll" script after the
# rsync is complete, and before it generates the various backup cycles from
# the new "current" backup, just made.
#
# As a side effect any copies of the same file may also become hard linked
# together in the backup.  To prevent lots of small (empty) but otherwise
# unrelated files being re-linking together, the "linkdups" limits its task
# to files larger than a few disk blocks.
#
####
#
# Anthony Thyssen -- February 2004 
# Original script called home_backup
#
# Edited by Laurens Maarschalkerweerd -- December 2014
# Modified to backup a remote server to a Synology NAS
#
####
#
# Pre-requirements and dependancies 
#
# SSH connection with the server should be made without password using SSH
# public key authentication
#
# Install using IPKG:
# - bc (terminal command /opt/bin/ipkg install bc)
#
# To install IPKG on a Synology DS414:
# http://www.noobunbox.net/synology/installer-ipkg-sur-un-nas-synology-ds414/
#
####


# -----------------------------------------------------------------------
# BACKUP SETTINGS
# -----------------------------------------------------------------------

SERVER="unitedstripes.cloud.tilaa.com"
USER="root"
SOURCE="/"
DEST="/volume3/VPS/backups"


# -----------------------------------------------------------------------
# SCRIPT SETTINGS
# -----------------------------------------------------------------------

PROGNAME=`type $0 | awk '{print $3}'`  	# search for executable on path
PROGDIR=`dirname $PROGNAME`            	# extract directory of program
PROGNAME=`basename $PROGNAME`          	# base name of program
cd $PROGDIR                            	# go to that directory
PROGDIR=`pwd`                          	# remove any symbolic link parts

DO_BACKUP=true							# do the actual backup
DO_SNAPSHOTS=true						# do the post-processing (make snapshots)
DO_NOTIFY=false							# send a notification to the Synology
										# dashboard for every successful backup

date_fmt_log='%d-%m-%Y %H:%M:%S'   		# for logging
date_fmt='%F_%H%M'		  	 			# for directory

# -----------------------------------------------------------------------
# -----------------------------------------------------------------------

Usage() {
  echo >&2 "$PROGNAME:" "$@"
  echo >&2 "Usage: $PROGNAME [-hur]"
  echo >&2 "For help use  $PROGNAME --help"
  exit 10
}
Help() {
  sed >&2 -n '/^###/q; /^#/!q; s/^#//; s/^ //; 3s/^/Usage: /; 2,$ p' \
          "$PROGDIR/$PROGNAME"
  exit 10;
}

# Path and environment setup (needed for direct execution from cron)
umask 077


# -----------------------------------------------------------------------
# FUNCTION - ADD TO SYNOLOGY LOG 
# -----------------------------------------------------------------------

synolog ( ) {
	type=$1
	date=$(date +"%Y/%m/%d %T")
	message=$2
	echo -e "$type\t$date\tSYSTEM:\t$message" >> /var/log/synolog/synosys.log
}


# -----------------------------------------------------------------------
# FUNCTION - ERROR
# -----------------------------------------------------------------------

error_exit ( ) {
	echo "Backup failed: $1"
	message="VPS Backup failed: $1"
	synolog "err" "$message"
	synodsmnotify @administrators "Backup VPS" "$message"
	exit 1
}


# -----------------------------------------------------------------------
# PREPARATION
# -----------------------------------------------------------------------

# Check if BC is installed. If not, throw an error and stop
if ! test -e /opt/bin/bc 2>/dev/null; then
	echo ""
	echo "bc is not installed. Trying to install it now."
	
	if test -e /opt/bin/ipkg; then
    	/opt/bin/ipkg install bc
    	wait
    	
    	if test -e /opt/bin/bc 2>/dev/null; then
    		echo "bc is successfully installed."
			synodsmnotify @administrators "Backup VPS" "bc was missing. Automatically installed."
			synolog "info" "VPS Backup info: bc was missing. Automatically installed."    	
    	fi
    	
	else 
        error_exit "bc is not installed. The script cannot install bc because IPKG is not installed. Go to http://www.noobunbox.net/synology/installer-ipkg-sur-un-nas-synology-ds414/ for instructions. When IPKG is installed, install bc via the Terminal, using the command /opt/bin/ipkg install bc."
	fi
fi



# -----------------------------------------------------------------------
# START BACKUP
# -----------------------------------------------------------------------


START=$(date +%s)

while [ $# -gt 0 ]; do
  case "$1" in
  --help|--doc) Help ;;

  -u) DO_SNAPSHOTS="" ;;   # update only, do not roll into snapshots
  -r) DO_BACKUP=""    ;;   # do the snapshot rolls only no actual backup

  --) shift; break ;;      		# end of user options
  esac
  shift   # next option
done


echo ""
echo ""
echo "---------------------------  NEW BACKUP ------------------------ "
echo ""
echo "Backing up $SERVER to $DEST"
echo ""

acct=''
acct_dir=$DEST
dir=`echo "$DEST" | cut -d: -f2`

# Select ssh program: 'r' is a special personal ssh which does personal
# account and host alias expansion, with appropriate DNS lookup regardless
# of the system configuration.
case "`type 'r' 2>&1`" in
  *'not found'*)   ssh=ssh ;;
  * )              ssh='r' ;;
esac

ssh_opts='-x'
rsync_opts='-aHvzxh'  # archive, hardlinks, verbose, compress, checksum (c, disabled), one-file-system, human readable
rsync_opts="$rsync_opts --rsh '$ssh $ssh_opts'"

# -----------------------------------------------------------------------

cd   # Go home.

cmd_found() {
  case "`type $1 2>&1`" in *'not found'*) return 1 ;; esac; return 0
}


# Prepare backup
echo "Prepare Backup"
date +"$date_fmt_log"
echo ""

  mkdir "$dir" 2>/dev/null
  ( cd "$dir"; "$PROGDIR/${PROGNAME}_prep"; )

# -----------------------------------------------------------------------
# DO THE BACKUP into 'current'

if [ "$DO_BACKUP" ]; then

# Verify connection
echo -ne "Connecting to the VPS"
result=$(ssh -o BatchMode=yes -o ConnectTimeout=5 $USER@$SERVER 'exit' 2>&1)

if [ $? -eq 255 ]
then 
	echo ""
	error_exit "Failed to connect to the VPS: $result"
else
	echo ": OK"
	date +"$date_fmt_log"
fi

echo ""

	echo "Creating/Updating Current Backup"
	date +"$date_fmt_log"
	echo ""
	
	trap "rm /tmp/$PROGNAME.$$; exit 10;" 1 2 3 15
	
	# RSYNC NOTES:
	#  --delete  recovers disk space at the start but has to scan the file
	#  system once to do this task. --delete-after has already has a scan and
	#  as such has better performance.  However any file 'changed' errors will
	#  cause it abort any and all deletions!
	#
	# The eval is needed to expand the quotes in $rsync_opts properly
	#
		
	eval "rsync $rsync_opts \
	--delete-after --ignore-errors --delete-excluded \
	--exclude-from '$PROGDIR/${PROGNAME}_exclude' \
	$USER@$SERVER:$SOURCE $dir/current"  | tee /tmp/$PROGNAME.$$
	echo ""
	
	echo "Recording Info and Files that were Backed Up"
	date +"$date_fmt_log"
	# transfer the list of changed to the backup in "BACKUP_LIST"
	eval rsync -aHc "/tmp/$PROGNAME.$$" "$acct_dir/current/BACKUP_LIST"
	
	# Record the times of this backup in "BACKUP_INFO"
	( date +"$date_fmt";  # formatted time of this backup
	date +%s;           # time in seconds since epoch
	) > /tmp/$PROGNAME.$$
	eval rsync -aHc "/tmp/$PROGNAME.$$" "$acct_dir/current/BACKUP_INFO"
	
	rm "/tmp/$PROGNAME.$$"
	trap "" 1 2 3 15
	echo ""
	
else
	
	echo "___ WARNING ___ Skipping Backup ___ WARNING ___"
	date +"$date_fmt_log"
	echo ""
	
fi


# -----------------------------------------------------------------------
# POST PROCESSING
# -----------------------------------------------------------------------

if [ "$DO_SNAPSHOTS" ]; then

	echo "Relink and Roll Backup Cycles"
	date +"$date_fmt_log"
	echo ""
	
	cd "$dir";

	# Sanity check
	if [ ! -d current ]; then
	  error_exit "No \"current\" backup directory found. Aborting post processing"
	fi
	
	# Debugging mode
	# DRYRUN=true  # Do not actually roll logs just report
	
	# Get the time now and the time of the last current backup
	formatted_time=`head -1 current/BACKUP_INFO`
	current_time=`awk 'NR == 2 {print 0+$1}' current/BACKUP_INFO`
	
	# --------------------------------------------------------
	# Relink Duplicate (moved) Files
	#

	# COMMENTED OUT -> SHOULD FIND ANOTHER SOLUTION FOR THIS
	# linkdups takes a loooong time when handling a lot of files
	#
	#	if [ -z "$DRYRUN" ]; then
	#	  # fix the old symbolic link to 'current' to give its 'date/time'
	#	  dirname="current_$formatted_time"
	#	  echo "Creating link from  $dirname"
	#	  rm -f current_*
	#	  ln -s current "$dirname"
	#	  echo ""
	#	
	#	  $PROGDIR/linkdups -q current hour_1_* day_01_*
	#	fi
		echo ""
		
	# --------------------------------------------------------
	# Constants for the various backup cycles
	
	# The following defines a shell database which details the number,
	# types, and timings of the various backup cycles to handle.
	
	cycle_names='hour day week mnth'
	
	# name of the cycle (for display)
	hour_name='hour'
	day_name='day'
	week_name='week'
	mnth_name='month'
	
	# Cycle length of rolled backups
	hour_length=12  # <- 2 digit!
	day_length=30  # <- 2 digit!
	week_length=9
	mnth_length=9
	
	# cycle directory name formatting
	hour_format='hour_%02d_%s'  # note 2 digit cycle count
	day_format='day_%02d_%s'  # note 2 digit cycle count
	week_format='week_%d_%s'
	mnth_format='mnth_%d_%s'
	
	# Preferences about what time/day a cycle should be cycled (from localtime())
	hour_pref=-  # No preference - hourly is not that exact
	day_pref=6   # Prefer making a Daily Snapshot around 6am
	week_pref=1  # Prefer a Weekly Snapshot to be made on a Monday
	mnth_pref=1  # Prefer Monthly on the first of the month
	
	# Force a roll if more than this time has passed since last cycle
	hour_force=40  # minutes for hour cycle  (less than an hour)
	day_force=23   # hours   for day cycle   (slightly less than a day)
	week_force=7   # days    for week cycle  (exactly a week)
	mnth_force=31  # days    for month cycle (slightly more than a month)
	
	# unit of time for reporting the cycle roll calculations
	hour_unit='min'
	day_unit='hour'
	week_unit='day'
	mnth_unit='day'
	
	# Number of seconds in for that unit of time
	hour_divisor=60    # mins - 60 secs
	day_divisor=3600   # hour -  '' x 60 minute
	week_divisor=86400 # days -  '' x 24 hours
	mnth_divisor=86400 # days again
	
	# The index from "localtime()" output for the time division
	# EG:  current_time=`perl -e "print ((localtime($current_time))[ INDEX ])"`
	hour_time_index=1  # mins of hour
	day_time_index=2   # hour of day
	week_time_index=6  # day of week
	mnth_time_index=3  # day of mnth
	
	# --------------------------------------------------------
	# Roll the backup cycles
	
	for cycle in $cycle_names
	do
	
	  # transfer constants for this cycle into simple variables
	  for var in name length pref force format divisor unit time_index
	  do
	    eval cycle_${var}=\$${cycle}_${var}
	    #eval echo "cycle_${var} = \$cycle_${var}"
	  done
	  cycle_delete=${cycle}_deleting   # directory for background deletion
	
	  echo "Snapshot Cycle ($cycle_name)..."
	  if [  "X$cycle_pref" = "X-" ]; then
	    echo "A new snapshot is made after $cycle_force ${cycle_unit}s"
	  else
	    echo "A new snapshot is made after $cycle_force ${cycle_unit}s" \
	         "or on the $cycle_pref $cycle_unit"
	  fi
	
	  # Compare Ages of Current backup Last Backup in this cycle
	  time=`perl -e "print ((localtime($current_time))[$cycle_time_index])"`
	  echo "Current backup was made on the $time $cycle_unit of the $cycle_name"
	
	  dirname=`printf "$cycle_format" 1 ''`
	  if [ "`ls -d $dirname* 2>/dev/null`" ]; then
	    # age of the last backup put into this cycle
	    last_time=`awk 'NR == 2 {print 0+$1}' $dirname*/BACKUP_INFO`
	    if [ -z "$last_time" ]; then
	      echo "ERROR: BACKUP_INFO parse failure - forcing cycle"
	      last_time=0
	    fi
	    	    	    
	    cycle_age=`echo "( $current_time - $last_time ) / $cycle_divisor" | /opt/bin/bc`
	    echo "and the last cycle roll is $cycle_age ${cycle_unit}s old"
	  else
	    echo "but the Snapshot Cycle does not exist yet - starting one"
	    cycle_age=999   # automatically start a new copy (big value)
	  fi
	
	  # Do we roll this backup cycle?
	  if [ X$time = X$cycle_pref  -o  $cycle_age -ge $cycle_force ]; then
	
	    echo "Rolling cycle..."
	    if [ -z "$DRYRUN" ]; then
	
	      # First remove the oldest possible backup in this cycle (if present)
	      n=$cycle_length      # maximum number of backups in cycle
	      dir_n=`printf "$cycle_format" $n ''`
	      if [ "`ls -d "${dir_n}"* 2>/dev/null`" ];  then
	        echo "delete '`ls -d "${dir_n}"*`'"
	        mkdir          $cycle_delete # create deletion directory
	        mv "${dir_n}"* $cycle_delete # move ALL old cycle dirs into it
	        chmod -R u+w   $cycle_delete # ensure we can remove it
	        nohup rm -rf   $cycle_delete >/dev/null 2>&1 & # delete (in background)
	      fi
	
	      # Roll the other backups in cycle to make space for new one
	      p=$n; n=`expr $n - 1`         # decrement cycle count
	      for n in `seq $n -1 1`; do
	        dir_n=`printf "$cycle_format" $n ''`
	        dir_p=`printf "$cycle_format" $p ''`
	        for dir in `ls -d "${dir_n}"* 2>/dev/null`; do
	          dir_new=`echo "$dir" | sed "s/^$dir_n/$dir_p/"`
	          echo "mv $dir $dir_new"   # log the directory move
	          mv "$dir" "$dir_new"      # rename directory
	        done
	        p=$n;
	      done
	    fi
	
	    dirname=`printf "$cycle_format" $p "$formatted_time"`
	    echo "linking current backup to... '$dirname'"
	    [ -z "$DRYRUN" ] &&
	      cp -rpl current "$dirname"  # PROBEREN: cp -al
	
	  elif [ $cycle == 'day' ]; then
	    echo ""
	    echo "No 'day' roll, so no need to test for 'week' or 'month' cycles"
	    break  # skip other backup cycles
	  fi
	  echo ""
	
	done
	echo "Snapshop cycle rolls complete."
	echo ""
	
fi

# -----------------------------------------------------------------

echo "Backup finished"
date +"$date_fmt_log"

FINISH=$(date +%s)
TOTALTIME="Total time: $(( ($FINISH-$START) / 60 ))m, $(( ($FINISH-$START) % 60 ))s"

echo ""
echo "Total time: $(( ($FINISH-$START) / 60 )) minutes, $(( ($FINISH-$START) % 60 )) seconds"
echo ""
echo "------------------------- BACKUP COMPLETED --------------------- "

# SEND NOTIFICATION TO SYNOLOGY DASHBOARD
if [ "$DO_NOTIFY" == true ]; then
	synodsmnotify @administrators "Backup VPS" "Backup completed. Total time: $TOTALTIME"
fi

# LOG in SYNOLOGY SYSTEM LOG
synolog "info" "VPS Backup completed. $TOTALTIME"