partialscrub.sh

#!/bin/bash
#set -x

#use argbash to parse arguments

#
# This is a rather minimal example Argbash potential
# Example taken from http://argbash.readthedocs.io/en/stable/example.html
#
# ARG_OPTIONAL_BOOLEAN([multiple_devices],[d],[specify to scrub one device at a time, useful for raid5/6])
# ARG_OPTIONAL_SINGLE([percentage],[p],[how much data to scrub, default is to scan all],[0])
# ARG_OPTIONAL_SINGLE([frequency],[f],[how frequently scrub should be run],[2629744])
# ARG_POSITIONAL_SINGLE([device],[],[specify the btrfs filesystem to scrub])
# ARG_HELP([Scrub btrfs data periodically, each time maximum p percentage is scrubbed. When scrub finishes it will not restart, unless f seconds have passed])
# ARGBASH_GO()
# needed because of Argbash --> m4_ignore([
### START OF CODE GENERATED BY Argbash v2.9.0 one line above ###
# Argbash is a bash code generator used to get arguments parsing right.
# Argbash is FREE SOFTWARE, see https://argbash.io for more info
# Generated online by https://argbash.io/generate


# # When called, the process ends.
# Args:
#       $1: The exit message (print to stderr)
#       $2: The exit code (default is 1)
# if env var _PRINT_HELP is set to 'yes', the usage is print to stderr (prior to $1)
# Example:
#       test -f "$_arg_infile" || _PRINT_HELP=yes die "Can't continue, have to supply file as an argument, got '$_arg_infile'" 4
die()
{
        local _ret="${2:-1}"
        test "${_PRINT_HELP:-no}" = yes && print_help >&2
        echo "$1" >&2
        exit "${_ret}"
}


# Function that evaluates whether a value passed to it begins by a character
# that is a short option of an argument the script knows about.
# This is required in order to support getopts-like short options grouping.
begins_with_short_option()
{
        local first_option all_short_options='dpfh'
        first_option="${1:0:1}"
        test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0
}

# THE DEFAULTS INITIALIZATION - POSITIONALS
# The positional args array has to be reset before the parsing, because it may already be defined
# - for example if this script is sourced by an argbash-powered script.
_positionals=()
_arg_device="specify the btrfs filesystem to scrub"
# THE DEFAULTS INITIALIZATION - OPTIONALS
_arg_multiple_devices="off"
_arg_percentage="100"
_arg_frequency="2629744"


# Function that prints general usage of the script.
# This is useful if users asks for it, or if there is an argument parsing error (unexpected / spurious arguments)
# and it makes sense to remind the user how the script is supposed to be called.
print_help()
{
        printf '%s\n' "Scrub btrfs data periodically, each time maximum p percentage is scrubbed. When scrub finishes it will not restart, unless f seconds have passed"
        printf 'Usage: %s [-d|--(no-)multiple_devices] [-p|--percentage <arg>] [-f|--frequency <arg>] [-h|--help] [<device>]\n' "$0"
        printf '\t%s\n' "-d, --multiple_devices, --no-multiple_devices: specify to scrub one device at a time, useful for raid5/6 (off by default)"
        printf '\t%s\n' "-p, --percentage: how much data to scrub, default is to scan all (default: '100')"
        printf '\t%s\n' "-f, --frequency: how frequently scrub should be run (default: '2629744') which is the average tropical month in seconds"
        printf '\t%s\n' "-h, --help: Prints help"
}


# The parsing of the command-line
parse_commandline()
{
        _positionals_count=0
        while test $# -gt 0
        do
                _key="$1"
                case "$_key" in
                        # The multiple_devices argurment doesn't accept a value,
                        # we expect the --multiple_devices or -d, so we watch for them.
                        -d|--no-multiple_devices|--multiple_devices)
                                _arg_multiple_devices="on"
                                test "${1:0:5}" = "--no-" && _arg_multiple_devices="off"
                                ;;
                        # We support getopts-style short arguments clustering,
                        # so as -d doesn't accept value, other short options may be appended to it, so we watch for -d*.
                        # After stripping the leading -d from the argument, we have to make sure
                        # that the first character that follows coresponds to a short option.
                        -d*)
                                _arg_multiple_devices="on"
                                _next="${_key##-d}"
                                if test -n "$_next" -a "$_next" != "$_key"
                                then
                                        { begins_with_short_option "$_next" && shift && set -- "-d" "-${_next}" "$@"; } || die "The short option '$_key' can't be decomposed to ${_key:0:2} and -${_key:2}, because ${_key:0:2} doesn't accept value and '-${_key:2:1}' doesn't correspond to a short option."
                                fi
                                ;;
                        # We support whitespace as a delimiter between option argument and its value.
                        # Therefore, we expect the --percentage or -p value.
                        # so we watch for --percentage and -p.
                        # Since we know that we got the long or short option,
                        # we just reach out for the next argument to get the value.
                        -p|--percentage)
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _arg_percentage="$2"
                                shift
                                ;;
                        # We support the = as a delimiter between option argument and its value.
                        # Therefore, we expect --percentage=value, so we watch for --percentage=*
                        # For whatever we get, we strip '--percentage=' using the ${var##--percentage=} notation
                        # to get the argument value
                        --percentage=*)
                                _arg_percentage="${_key##--percentage=}"
                                ;;
                        # We support getopts-style short arguments grouping,
                        # so as -p accepts value, we allow it to be appended to it, so we watch for -p*
                        # and we strip the leading -p from the argument string using the ${var##-p} notation.
                        -p*)
                                _arg_percentage="${_key##-p}"
                                ;;
                        # See the comment of option '--percentage' to see what's going on here - principle is the same.
                        -f|--frequency)
                                test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
                                _arg_frequency="$2"
                                shift
                                ;;
                        # See the comment of option '--percentage=' to see what's going on here - principle is the same.
                        --frequency=*)
                                _arg_frequency="${_key##--frequency=}"
                                ;;
                        # See the comment of option '-p' to see what's going on here - principle is the same.
                        -f*)
                                _arg_frequency="${_key##-f}"
                                ;;
                        # See the comment of option '--multiple_devices' to see what's going on here - principle is the same.
                        -h|--help)
                                print_help
                                exit 0
                                ;;
                        # See the comment of option '-d' to see what's going on here - principle is the same.
                        -h*)
                                print_help
                                exit 0
                                ;;
                        *)
                                _last_positional="$1"
                                _positionals+=("$_last_positional")
                                _positionals_count=$((_positionals_count + 1))
                                ;;
                esac
                shift
        done
}


# Check that we receive expected amount positional arguments.
# Return 0 if everything is OK, 1 if we have too little arguments
# and 2 if we have too much arguments
handle_passed_args_count()
{
        test "${_positionals_count}" -le 1 || _PRINT_HELP=yes die "FATAL ERROR: There were spurious positional arguments --- we expect between 0 and 1, but got ${_positionals_count} (the last one was: '${_last_positional}')." 1
}


# Take arguments that we have received, and save them in variables of given names.
# The 'eval' command is needed as the name of target variable is saved into another variable.
assign_positional_args()
{
        local _positional_name _shift_for=$1
        # We have an array of variables to which we want to save positional args values.
        # This array is able to hold array elements as targets.
        # As variables don't contain spaces, they may be held in space-separated string.
        _positional_names="_arg_device "

        shift "$_shift_for"
        for _positional_name in ${_positional_names}
        do
                test $# -gt 0 || break
                eval "$_positional_name=\${1}" || die "Error during argument parsing, possibly an Argbash bug." 1
                shift
        done
}

# Now call all the functions defined above that are needed to get the job done
parse_commandline "$@"
handle_passed_args_count
assign_positional_args 1 "${_positionals[@]}"

# OTHER STUFF GENERATED BY Argbash

### END OF CODE GENERATED BY Argbash (sortof) ### ])
# [ <-- needed because of Argbash

# ] <-- needed because of Argbash

#print arguments
echo "filesystem:$_arg_device"
echo "raid56:$_arg_multiple_devices"
echo "percentage:$_arg_percentage"
echo "frequency:$_arg_frequency"

#make sure that scrub is always stopped and statistics are printed when the program is terminated
trap "btrfs scrub cancel $_arg_device; btrfs scrub status $_arg_device; trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGQUIT EXIT

#find the array index of the active device, used in RAID56
active_device () {
  local active_device=$(btrfs scrub status -d "$1" | grep device | awk ' { if ( NF > x ) { x = NF; y = NR } }END{ print y }')
  (( active_device-- ))
  echo "$active_device"
}

current_position () {
  local position=$(btrfs scrub status -R "$1" | grep data_bytes_scrubbed | awk '{print $2}')
  if [ "$position" = "" ]; then
    position=0
  fi
  echo "$position"
}

current_status (){
  local status=""
  status=$(btrfs scrub status "$_arg_device" | grep Status: | awk '{print $2}')
  echo "$status"
}

#make sure that the program is scrubbing only one filesystem at a time

LCK="/tmp/btrfs.tmp"
exec 99>$LCK

flock -x 99


#make sure that date command understands btrfs scrub output
LC_TIME="C"

#find the last time that scrub was active. If frequency seconds are almost passed, scrub must be restarted
last_action_date=$(btrfs scrub status "$_arg_device" | grep -B1 "Status:" | grep Scrub | awk '{$1=$2=""; print $0}' |  awk '{$1=$1};1')
epoch=$( date -d "$last_action_date" +"%s" )
epoch=$(( epoch / _arg_frequency ))
now=$( date +"%s" )
now=$(( now / _arg_frequency ))
decision_to_restart_scrub=$(( now - epoch ))
echo "scrub must be restarted:$decision_to_restart_scrub"

#last scrub status
status=$( btrfs scrub status "$_arg_device" | grep Status: | awk '{print $2}' )

#find the total data that must be scrubbed, in raid56 we need the -T
if [ "$_arg_multiple_devices" = "off" ]; then
  data_to_scrub=$( btrfs fi usage -b "$_arg_device" | grep Used: | head -n 1 |awk '{print $2}' )
else
  data_to_scrub=$( btrfs fi usage -b -T "$_arg_device" | grep Used | tail -n 1 | awk '{print $2}' )
fi

#data already scrubbed
data_scrubbed=$( btrfs scrub status -R "$_arg_device" | grep data_bytes_scrubbed | awk '{print $2}' )


echo "total data:$data_to_scrub"
echo "data scrubbed:$data_scrubbed"
echo "status:$status"


if [ "$status" = "" ]; then
  status="finished"
  decision_to_restart_scrub=1
  echo "Scrub status not acquired! Restart scrub"
fi

#calculate how many data do we need to scrub in this run
limit=$(( data_scrubbed + data_to_scrub * _arg_percentage / 100 ))

if [ "$status" = "finished" ]; then
  limit=$(( data_to_scrub * _arg_percentage / 100 ))
fi

echo "limit:$limit"

#always try to resume previous scrub.
#if scrub must restart from scratch, activate it.
#in Raid56 mode,btrfs start until last device is done.

if [ "$_arg_multiple_devices" = "off" ]; then
  if [ "$status" != "finished" ]; then
    btrfs scrub resume -c 3 "$_arg_device"
  else
    if [ "$decision_to_restart_scrub" -gt 0 ]; then
      btrfs scrub start -c 3 "$_arg_device"
    fi
  fi
else
  #raid56 mode: resume last device
  readarray -t devices < <(btrfs scrub status -d "$_arg_device" | grep device | awk '{print $3}' )
  active_device="$(active_device "$_arg_device")"
  echo "active device:$active_device"
  if [ "$status" != "finished" ]; then
    btrfs scrub resume -c 3 "${devices[$active_device]}"
  else
    #start all devices to make sure that scrub is done also to them
    flag=false
    for index in "${!devices[@]}"
    do
      if [ "$index" -gt "$active_device" ]; then
        btrfs scrub start -c 3 "${devices[$index]}"
        flag=true
        break
      fi
    done
    #all raid56 devices were doen in previous scrub, thus we must restart from the begining
    if ! $flag; then
      if [ "$decision_to_restart_scrub" -gt 0 ]; then
        btrfs scrub start -c 3 "${devices[0]}"
      fi
    fi
  fi
fi

#scrub is resumed or started in this point!
#scrub statistics are updated every 5 seconds so sleep and check position periodically

sleep 6
current_position="$(current_position "$_arg_device")"

while [ "$current_position" -lt "$limit" ] && [ "$(current_status "$_arg_device")" != 'finished'  ]
do
  sleep 6
  current_position="$(current_position "$_arg_device")"
  #echo "Current Position:$current_position"
done

#loop has finished!
#trap will stop scrub and print statistics

#btrfs scrub cancel "$_arg_device"
#btrfs scrub status "$_arg_device"