multiple attempts for ssh command, fix fail connection when using pas…

…sword

entrypoint.sh

@@ -9,6 +9,8 @@ TARGET_CLEANUP="${TARGET_CLEANUP:-"true"}"
 CHECK_CONNECTION="${CHECK_CONNECTION:-"true"}"
 CHECK_CONNECTION_ATTEMPTS=${CHECK_CONNECTION_ATTEMPTS:-30}
 CHECK_CONNECTION_DELAY=${CHECK_CONNECTION_DELAY:-10}
+SSH_CMD_ATTEMPTS=${SSH_CMD_ATTEMPTS:-5}
+SSH_CMD_DELAY=${SSH_CMD_DELAY:-5}
 
 # Debug
 if [ "${DEBUG:-}" = "true" ]; then
@@ -21,7 +23,7 @@ if [[ ! remote_required ]] || [[ ! mamp_required ]] || [[ -z "${ASSETS_FOLDER+x}
 fi
 
 # Create ssh config file is using proxy connection
-if [[ -n "${BASTION_HOST}" && -n "${BASTION_HOST_USERNAME}" ]];then
+if [[ -n "${BASTION_HOST+x}" && -n "${BASTION_HOST_USERNAME+x}" ]];then
     ssh_config_file
 fi
 
@@ -37,14 +39,16 @@ fi
 # Create execution folder 
 echo "Create assets folder on target"
 TARGET_FOLDER="${TARGET_FOLDER:-"deliverest-${RANDOM}"}"
-$(ssh_cmd "mkdir -p ${TARGET_FOLDER}")
+mkdir_cmd="$(ssh_cmd mkdir -p ${TARGET_FOLDER})"
+exec_and_retry ${SSH_CMD_ATTEMPTS} ${SSH_CMD_DELAY} ${mkdir_cmd}
 
 # Copy asset
 echo "Copy assets folder to target"
-$(scp_to_cmd "${ASSETS_FOLDER}/*" "${TARGET_FOLDER}/")
+scp_cmd="$(scp_to_cmd "${ASSETS_FOLDER}/*" "${TARGET_FOLDER}/")"
+exec_and_retry ${SSH_CMD_ATTEMPTS} ${SSH_CMD_DELAY} ${scp_cmd}
 
 # Exec command
-$(ssh_cmd "$@")
+exec_and_retry ${SSH_CMD_ATTEMPTS} ${SSH_CMD_DELAY} "$(ssh_cmd $@)"
 
 # If remote workload includes a reboot this is the only way to ensure we can 
 # copy results if any or cleanup
@@ -60,11 +64,12 @@ fi
 if [[ ! -z "${TARGET_RESULTS+x}" ]]; then
     # If exec create some reuslts we define the env and they will be copied to OUTPUT_FOLDER
     OUTPUT_FOLDER="${OUTPUT_FOLDER:-"/output"}"
-    $(scp_from_cmd "${TARGET_FOLDER}/${TARGET_RESULTS}" "${OUTPUT_FOLDER}/")
+    scp_cmd="$(scp_from_cmd "${TARGET_FOLDER}/${TARGET_RESULTS}" "${OUTPUT_FOLDER}/)")"
+    exec_and_retry ${SSH_CMD_ATTEMPTS} ${SSH_CMD_DELAY} ${scp_cmd}
 fi
 
 if [ "${TARGET_CLEANUP:-}" = "true" ]; then
     # This will create the cmd based on OS env with the right syntax
     cmd="$(remove_folder ${TARGET_FOLDER})"
-    $(ssh_cmd ${cmd})
+    exec_and_retry ${SSH_CMD_ATTEMPTS} ${SSH_CMD_DELAY} "$(ssh_cmd $cmd)"
 fi

lib/common/remote.sh

@@ -27,7 +27,9 @@ connect_options() {
     options="$options -o UserKnownHostsFile=/dev/null"
     options="$options -o ServerAliveInterval=30"
     options="$options -o ServerAliveCountMax=1200"
-    options="$options -o BatchMode=yes"
+    if [[ -n "${TARGET_HOST_KEY_PATH+x}" ]]; then
+        options="$options -o BatchMode=yes"
+    fi
     options="$options -o ConnectTimeout=3"
     echo $options
 }
@@ -59,33 +61,16 @@ EOF
 # Check the connection to the host; "delay" in seconds, "repeats" in number of reps
 # Run as: check_connection <repeats int> <delay int>
 # e.g. check_connection 30 10
+
 check_connection() {
-    repeats=$1
-    while [[ $repeats -gt 0 ]]
-    do
-        $(ssh_cmd "pwd")
-        if [[ $? -gt 0 ]] 
-        then
-            echo "reps remaining: $repeats" >&2
-            ((repeats--))
-            sleep $2
-        else
-            break
-        fi
-    done
-    # fail or pass the check
-    if [[ $repeats -gt 0 ]]
-    then
-        return 0
-    else
-        return 1
-    fi          
+    check_cmd="$(ssh_cmd pwd)"
+    exec_and_retry $1 $2 $check_cmd
 }
 
 # Define remote connection
 uri () {
     local remote="${TARGET_HOST_USERNAME}@${TARGET_HOST}"
-    if [[ -n "${TARGET_HOST_DOMAIN}" ]]; then
+    if [[ -n "${TARGET_HOST_DOMAIN+x}" ]]; then
         remote="${TARGET_HOST_USERNAME}@${TARGET_HOST_DOMAIN}@${TARGET_HOST}"
     fi
     echo "${remote}" 
@@ -96,9 +81,9 @@ uri () {
 # $2 remote path
 scp_to_cmd () {
     cmd="scp -r $(connect_options) "
-    if [[ -n "${BASTION_HOST}" && -n "${BASTION_HOST_USERNAME}" ]]; then
+    if [[ -n "${BASTION_HOST+x}" && -n "${BASTION_HOST_USERNAME+x}" ]]; then
         echo "${cmd} -F ssh_config ${1} target_host:${2}"
-    elif [[ -n "${TARGET_HOST_KEY_PATH}" ]]; then
+    elif [[ -n "${TARGET_HOST_KEY_PATH+x}" ]]; then
         echo "${cmd} -i ${TARGET_HOST_KEY_PATH} ${1} $(uri):${2}"
     else
         echo "sshpass -p ${TARGET_HOST_PASSWORD} ${cmd} ${1} $(uri):${2}" 
@@ -110,9 +95,9 @@ scp_to_cmd () {
 # $2 local path
 scp_from_cmd () {
     cmd="scp -r $(connect_options) "
-    if [[ -n "${BASTION_HOST}" && -n "${BASTION_HOST_USERNAME}" ]]; then
+    if [[ -n "${BASTION_HOST+x}" && -n "${BASTION_HOST_USERNAME+x}" ]]; then
         echo "${cmd} -F ssh_config target_host:${1} ${2} "
-    elif [[ -n "${TARGET_HOST_KEY_PATH}" ]]; then
+    elif [[ -n "${TARGET_HOST_KEY_PATH+x}" ]]; then
         echo "${cmd} -i ${TARGET_HOST_KEY_PATH} $(uri):${1} ${2}"
     else
         echo "sshpass -p ${TARGET_HOST_PASSWORD} ${cmd} $(uri):${1} ${2}" 
@@ -122,9 +107,9 @@ scp_from_cmd () {
 # Generate SSH command
 ssh_cmd () {
     cmd="ssh $(connect_options) "
-    if [[ -n "${BASTION_HOST}" && -n "${BASTION_HOST_USERNAME}" ]]; then
+    if [[ -n "${BASTION_HOST+x}" && -n "${BASTION_HOST_USERNAME+x}" ]]; then
         cmd+="-F ssh_config target_host "
-    elif [[ -n "${TARGET_HOST_KEY_PATH}" ]]; then
+    elif [[ -n "${TARGET_HOST_KEY_PATH+x}" ]]; then
         cmd+="-i ${TARGET_HOST_KEY_PATH} $(uri) "
     else
         cmd="sshpass -p ${TARGET_HOST_PASSWORD} ${cmd} $(uri) "
@@ -137,4 +122,28 @@ ssh_cmd () {
         cmd+=" $@"
     fi
     echo "${cmd}"
-}
+}
+
+# Execute command and re-try if failed
+exec_and_retry() {
+    local retries="$1"
+    local wait="$2"
+    shift 2
+    local command="$@"
+
+    # Run the command, and save the exit code
+    $command
+    local exit_code=$?
+
+    # If the exit code is non-zero (i.e. command failed), and we have not
+    # reached the maximum number of retries, run the command again
+    if [[ $exit_code -ne 0 && $retries -gt 0 ]]; then
+    # Wait before retrying
+    sleep $wait
+
+    exec_and_retry $(($retries - 1)) $wait "$command"
+    else
+    # Return the exit code from the command
+    return $exit_code
+    fi
+}