From 746101ccb7c229abe88179c1f530b58c723feeb6 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 13:13:44 -0600 Subject: [PATCH 01/22] Faster switching Switch servers faster by not checking height/consensus of next server. Instead, just try to enable forging on it. Reduce sleep to 0.5 seconds when forging. Look at more of log when forging to make sure we catch the line as soon as possible --- check_consensus.sh | 110 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 50 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index 8ac6e1e..9fd9a9a 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -1,4 +1,4 @@ -## Version 0.9.2 +## Version 0.9.4 #!/bin/bash ## Check for config file @@ -27,16 +27,17 @@ done #Set text delay at 0 TXTDELAY=0 +FORGEDDELAY=0 # Set colors -red=`tput setaf 1` -green=`tput setaf 2` -yellow=`tput setaf 3` -cyan=`tput setaf 6` -resetColor=`tput sgr0` +RED=`tput setaf 1` +GREEN=`tput setaf 2` +YELLOW=`tput setaf 3` +CYAN=`tput setaf 6` +RESETCOLOR=`tput sgr0` ## Log start of script -date +"%Y-%m-%d %H:%M:%S || ${green}Starting MrV's consensus script${resetColor}" +date +"%Y-%m-%d %H:%M:%S || ${GREEN}Starting MrV's consensus script${RESETCOLOR}" # Set Lisk directory @@ -57,7 +58,7 @@ function SyncState() result=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync" | jq '.syncing') sleep 2 done - + date +"%Y-%m-%d %H:%M:%S || Looks like blockchain is finished syncing." } #--------------------------------------------------------------------------- @@ -66,37 +67,37 @@ function SyncState() while true; do ## Get forging status of server - FORGE=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') + FORGE=$(curl --connect-timeout 1 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then ## Log when a block is forged FORGEDBLOCKLOG=$(tail ~/lisk-main/logs/lisk.log -n 20| grep 'Forged new block') if [[ -n "$FORGEDBLOCKLOG" ]]; then - date +"%Y-%m-%d %H:%M:%S || ${green}$FORGEDBLOCKLOG${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" fi ## Get current server's height and consensus - SERVERLOCAL=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") + SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') - + ## If consensus is less than 51 and we are forging soon, try a reload to get new peers ## Management script should switch forging server during reload ## from Nerigal if [ "$CONSENSUSLOCAL" -lt "51" ]; then delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') - date +"%Y-%m-%d %H:%M:%S || ${yellow}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $pbk${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $pbk${RESETCOLOR}" if [[ $delegates == *"$pbk"* ]]; then - date +"%Y-%m-%d %H:%M:%S || ${red}You are forging soon, but your consensus is too low. Looking to switch server before reload.${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" for SERVER in ${SERVERS[@]} do ## Get next server's height and consensus SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") HEIGHT=$( echo "$SERVERINFO" | jq '.height') CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') - + ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch if [[ -n "$HEIGHT" ]]; then @@ -111,35 +112,35 @@ do if [ "$DISABLEFORGE" = "true" ]; then curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable - date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${RESETCOLOR}" ChangeDirectory bash lisk.sh reload sleep 20 SyncState break else - date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1 with low consensus before forging${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 with low consensus before forging${RESETCOLOR}" fi fi done fi fi ## Check log if node is recovering close to forging time - LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 2| grep 'starting recovery') + LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 10| grep 'starting recovery') if [[ -n "$LASTLINE" ]]; then delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') - date +"%Y-%m-%d %H:%M:%S || ${yellow}Node is recovering. Looking for delegate forging soon matching $pbk${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering. Looking for delegate forging soon matching $pbk${RESETCOLOR}" if [[ $delegates == *"$pbk"* ]]; then - date +"%Y-%m-%d %H:%M:%S || ${red}You are forging soon, but your node is recovering. Looking to switch server while recovering.${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" for SERVER in ${SERVERS[@]} do ## Get next server's height and consensus SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") HEIGHT=$( echo "$SERVERINFO" | jq '.height') CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') - + ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch if [[ -n "$HEIGHT" ]]; then @@ -154,58 +155,67 @@ do if [ "$DISABLEFORGE" = "true" ]; then curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable - date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS as your node is recovering.${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your node is recovering.${RESETCOLOR}" break else - date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1 that is recovering before forging${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 that is recovering before forging${RESETCOLOR}" fi fi done fi fi - + ## Check log for Inadequate consensus - LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 2| grep 'Inadequate') + LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 10| grep 'Inadequate') if [[ -n "$LASTLINE" ]]; then - date +"%Y-%m-%d %H:%M:%S || ${red}WARNING: $LASTLINE${resetColor}" - - for SERVER in ${SERVERS[@]} - do + date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: $LASTLINE${RESETCOLOR}" + + ## for SERVER in ${SERVERS[@]} + ## do ## Get next server's height and consensus - SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") - HEIGHT=$( echo "$SERVERINFO" | jq '.height') - CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') - + ## SERVERINFO=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") + ## HEIGHT=$( echo "$SERVERINFO" | jq '.height') + ## CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') + ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch - if [[ -n "$HEIGHT" ]]; - then - diff=$(( $HEIGHTLOCAL - $HEIGHT )) - else - diff="999" - fi + ## if [[ -n "$HEIGHT" ]]; + ## then + ## diff=$(( $HEIGHTLOCAL - $HEIGHT )) + ## else + ## diff="999" + ## fi ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate - if [ "$diff" -lt "3" ]; - then - DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') + ## if [ "$diff" -lt "3" ]; + ## then + DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') if [ "$DISABLEFORGE" = "true" ]; then - curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable - date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS to try and forge${resetColor}" + for SERVER in ${SERVERS[@]} + do + ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success') + if [ "$ENABLEFORGE" = "true" ]; + then + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS to try and forge.${RESETCOLOR}" + break ## Leave servers loop + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on $SERVER. Trying next server.${RESETCOLOR}" + fi + done else - date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1.${RESETCOLOR}" fi break - fi - done + ## fi + ## done fi (( ++TXTDELAY )) - if [[ "$TXTDELAY" -eq "30" ]]; ## Wait 30 seconds to update running status to not overcrowd log + if [[ "$TXTDELAY" -eq "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log then - date +"%Y-%m-%d %H:%M:%S || ${green}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${resetColor}" + date +"%Y-%m-%d %H:%M:%S || ${GREEN}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${RESETCOLOR}" TXTDELAY=0 fi - sleep 1 + sleep 0.5 else (( ++TXTDELAY )) if [[ "$TXTDELAY" -eq "30" ]]; ## Wait 30 seconds to update running status to not overcrowd log From 876608b486d78dc11df93a0742e3b1078bf4e932 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 13:16:17 -0600 Subject: [PATCH 02/22] Retry getting local forging status if unsuccessful --- check_consensus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_consensus.sh b/check_consensus.sh index 9fd9a9a..6e5f7f4 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -67,7 +67,7 @@ function SyncState() while true; do ## Get forging status of server - FORGE=$(curl --connect-timeout 1 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') + FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then ## Log when a block is forged From b1c98b43036bf5d3e33eac6212b5fba011a3369f Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 13:51:11 -0600 Subject: [PATCH 03/22] Add fork 5 check and only tail log once --- check_consensus.sh | 91 +++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index 6e5f7f4..085f98f 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -70,16 +70,28 @@ do FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then - ## Log when a block is forged - FORGEDBLOCKLOG=$(tail ~/lisk-main/logs/lisk.log -n 20| grep 'Forged new block') - if [[ -n "$FORGEDBLOCKLOG" ]]; - then - date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" - fi ## Get current server's height and consensus SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') + ## Get recent log + LOG=$(tail ~/lisk-main/logs/lisk.log -n 10) + + ## Only look for forged block in log if we didn't just log it + if [[ "$FORGEDDELAY" -eq "0" ]]; + then + ## Log when a block is forged + FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block') + + if [[ -n "$FORGEDBLOCKLOG" ]]; + then + date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" + FORGEDDELAY=20 + fi + elif [[ "$FORGEDDELAY" -gt "0" ]] + then + ((FORGEDDELAY--)) + fi ## If consensus is less than 51 and we are forging soon, try a reload to get new peers ## Management script should switch forging server during reload @@ -126,7 +138,7 @@ do fi fi ## Check log if node is recovering close to forging time - LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 10| grep 'starting recovery') + LASTLINE=$( echo "$LOG" | grep 'starting recovery') if [[ -n "$LASTLINE" ]]; then delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') @@ -165,49 +177,36 @@ do fi fi - ## Check log for Inadequate consensus - LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 10| grep 'Inadequate') - if [[ -n "$LASTLINE" ]]; + ## Check log for Inadequate consensus or fork 5 while forging + INADEQUATE=$( echo "$LOG" | grep 'Inadequate') + FORK5=$( echo "$LOG" | grep '"cause":5') + if [ -n "$INADEQUATE" ] || [ -n "$FORK5" ] ; then - date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: $LASTLINE${RESETCOLOR}" - - ## for SERVER in ${SERVERS[@]} - ## do - ## Get next server's height and consensus - ## SERVERINFO=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") - ## HEIGHT=$( echo "$SERVERINFO" | jq '.height') - ## CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') - - ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch - ## if [[ -n "$HEIGHT" ]]; - ## then - ## diff=$(( $HEIGHTLOCAL - $HEIGHT )) - ## else - ## diff="999" - ## fi - ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate - ## if [ "$diff" -lt "3" ]; - ## then - DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') - if [ "$DISABLEFORGE" = "true" ]; + if [ -n "$FORK5" ]; + then + date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Fork 5 in log.${RESETCOLOR}" + else + date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Inadequate consensus to forge.${RESETCOLOR}" + fi + + ## Disable forging on local server first. If successful, loop through servers until we are able to enable forging on one + DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') + if [ "$DISABLEFORGE" = "true" ]; + then + for SERVER in ${SERVERS[@]} + do + ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success') + if [ "$ENABLEFORGE" = "true" ]; then - for SERVER in ${SERVERS[@]} - do - ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success') - if [ "$ENABLEFORGE" = "true" ]; - then - date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS to try and forge.${RESETCOLOR}" - break ## Leave servers loop - else - date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on $SERVER. Trying next server.${RESETCOLOR}" - fi - done + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER to try and forge.${RESETCOLOR}" + break ## Leave servers loop else - date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1.${RESETCOLOR}" + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on $SERVER. Trying next server.${RESETCOLOR}" fi - break - ## fi - ## done + done + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1.${RESETCOLOR}" + fi fi (( ++TXTDELAY )) if [[ "$TXTDELAY" -eq "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log From f78500b720326230c691bae2b25c08879ff430ef Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 14:02:17 -0600 Subject: [PATCH 04/22] ShellCheck improvements --- check_consensus.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index 085f98f..e12e5fe 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -17,10 +17,10 @@ SERVERS=() size=$( echo "$CONFIGFILE" | jq '.servers | length') i=0 -while [ $i -le $size ] +while [ $i -le "$size" ] do SERVERS[$i]=$(echo "$CONFIGFILE" | jq -r --argjson i $i '.servers[$i]') - i=`expr $i + 1` + i=$((i + 1)) done ### ######################### @@ -30,11 +30,11 @@ TXTDELAY=0 FORGEDDELAY=0 # Set colors -RED=`tput setaf 1` -GREEN=`tput setaf 2` -YELLOW=`tput setaf 3` -CYAN=`tput setaf 6` -RESETCOLOR=`tput sgr0` +RED=$(tput setaf 1) +GREEN=$(tput setaf 2) +YELLOW=$(tput setaf 3) +CYAN=$(tput setaf 6) +RESETCOLOR=$(tput sgr0) ## Log start of script date +"%Y-%m-%d %H:%M:%S || ${GREEN}Starting MrV's consensus script${RESETCOLOR}" @@ -55,7 +55,7 @@ function SyncState() while [[ -z $result || $result != 'false' ]] do date +"%Y-%m-%d %H:%M:%S || Blockchain syncing" - result=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync" | jq '.syncing') + result=$(curl --connect-timeout 3 -s "http://$SRV1$PRT/api/loader/status/sync" | jq '.syncing') sleep 2 done @@ -103,7 +103,7 @@ do if [[ $delegates == *"$pbk"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" - for SERVER in ${SERVERS[@]} + for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") @@ -146,7 +146,7 @@ do if [[ $delegates == *"$pbk"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" - for SERVER in ${SERVERS[@]} + for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") @@ -193,7 +193,7 @@ do DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') if [ "$DISABLEFORGE" = "true" ]; then - for SERVER in ${SERVERS[@]} + for SERVER in "${SERVERS[@]}" do ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success') if [ "$ENABLEFORGE" = "true" ]; From e44ea0444956ebd41a11261c85ec7efd752a7e98 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 14:08:50 -0600 Subject: [PATCH 05/22] Double forged delay since sleep is 0.5 --- check_consensus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_consensus.sh b/check_consensus.sh index e12e5fe..f00f385 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -86,7 +86,7 @@ do if [[ -n "$FORGEDBLOCKLOG" ]]; then date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" - FORGEDDELAY=20 + FORGEDDELAY=40 fi elif [[ "$FORGEDDELAY" -gt "0" ]] then From d85de7efecf2e51c3bb54fe25befa06650832f38 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 14:33:22 -0600 Subject: [PATCH 06/22] Get current height and consensus only when needed --- check_consensus.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index f00f385..cf8d20b 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -70,10 +70,6 @@ do FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then - ## Get current server's height and consensus - SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") - HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') - CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') ## Get recent log LOG=$(tail ~/lisk-main/logs/lisk.log -n 10) @@ -103,6 +99,11 @@ do if [[ $delegates == *"$pbk"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" + ## Get current server's height and consensus + SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") + HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') + CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') + for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus @@ -145,7 +146,12 @@ do date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering. Looking for delegate forging soon matching $pbk${RESETCOLOR}" if [[ $delegates == *"$pbk"* ]]; then - date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" + ## Get current server's height and consensus + SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") + HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') + CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') + for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus @@ -211,6 +217,10 @@ do (( ++TXTDELAY )) if [[ "$TXTDELAY" -eq "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log then + ## Get current server's height and consensus + SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") + HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') + CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') date +"%Y-%m-%d %H:%M:%S || ${GREEN}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${RESETCOLOR}" TXTDELAY=0 fi From 9b54607819b339bdba4a4af82e3961ed2a7f1439 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 14:39:50 -0600 Subject: [PATCH 07/22] Revert last commit --- check_consensus.sh | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index cf8d20b..f00f385 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -70,6 +70,10 @@ do FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then + ## Get current server's height and consensus + SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") + HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') + CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') ## Get recent log LOG=$(tail ~/lisk-main/logs/lisk.log -n 10) @@ -99,11 +103,6 @@ do if [[ $delegates == *"$pbk"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" - ## Get current server's height and consensus - SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") - HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') - CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') - for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus @@ -146,12 +145,7 @@ do date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering. Looking for delegate forging soon matching $pbk${RESETCOLOR}" if [[ $delegates == *"$pbk"* ]]; then - date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" - ## Get current server's height and consensus - SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") - HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') - CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') - + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" for SERVER in "${SERVERS[@]}" do ## Get next server's height and consensus @@ -217,10 +211,6 @@ do (( ++TXTDELAY )) if [[ "$TXTDELAY" -eq "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log then - ## Get current server's height and consensus - SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync") - HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height') - CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus') date +"%Y-%m-%d %H:%M:%S || ${GREEN}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${RESETCOLOR}" TXTDELAY=0 fi From 5b382b66cde4cbe3c9348d843fe93a29bd514725 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Mon, 26 Dec 2016 20:42:30 -0600 Subject: [PATCH 08/22] Update check_consensus.sh --- check_consensus.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index f00f385..e62b766 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -25,8 +25,8 @@ done ### ######################### -#Set text delay at 0 -TXTDELAY=0 +#Set text delay and forging log delays +TXTDELAY=1 FORGEDDELAY=0 # Set colors @@ -209,19 +209,19 @@ do fi fi (( ++TXTDELAY )) - if [[ "$TXTDELAY" -eq "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log + if [[ "$TXTDELAY" -gt "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log then date +"%Y-%m-%d %H:%M:%S || ${GREEN}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${RESETCOLOR}" - TXTDELAY=0 + TXTDELAY=1 fi - sleep 0.5 + sleep 0.5 else (( ++TXTDELAY )) - if [[ "$TXTDELAY" -eq "30" ]]; ## Wait 30 seconds to update running status to not overcrowd log + if [[ "$TXTDELAY" -gt "30" ]]; ## Wait 30 seconds to update running status to not overcrowd log then date +"%Y-%m-%d %H:%M:%S || This server is not forging" - TXTDELAY=0 + TXTDELAY=1 fi - sleep 1 + sleep 1 fi done From ef2a45289bc8565473e25c8f526837b32305a51e Mon Sep 17 00:00:00 2001 From: mrv777 Date: Tue, 27 Dec 2016 16:38:40 -0600 Subject: [PATCH 09/22] Look for both fork and forged in recent log This could mean be forged on a fork so try another server...hopefully in time --- check_consensus.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index e62b766..963c6d4 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -177,14 +177,15 @@ do fi fi - ## Check log for Inadequate consensus or fork 5 while forging + ## Check log for Inadequate consensus or Fork & Forged while forging INADEQUATE=$( echo "$LOG" | grep 'Inadequate') - FORK5=$( echo "$LOG" | grep '"cause":5') - if [ -n "$INADEQUATE" ] || [ -n "$FORK5" ] ; + FORK=$( echo "$LOG" | grep 'Fork') + FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block') + if [ -n "$INADEQUATE" ] || ([ -n "$FORK" ] && [ -n "$FORGEDBLOCKLOG" ]); then - if [ -n "$FORK5" ]; + if [ -n "$FORK" ]; then - date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Fork 5 in log.${RESETCOLOR}" + date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Fork and Forged in log.${RESETCOLOR}" else date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Inadequate consensus to forge.${RESETCOLOR}" fi From 5f0302a76aa72b48b72d82e01ddfa9a6e2bc2b56 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Tue, 27 Dec 2016 16:47:41 -0600 Subject: [PATCH 10/22] Remove low consensus and we are forging soon check --- check_consensus.sh | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index 963c6d4..dd9e003 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -93,50 +93,6 @@ do ((FORGEDDELAY--)) fi - ## If consensus is less than 51 and we are forging soon, try a reload to get new peers - ## Management script should switch forging server during reload - ## from Nerigal - if [ "$CONSENSUSLOCAL" -lt "51" ]; - then - delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') - date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $pbk${RESETCOLOR}" - if [[ $delegates == *"$pbk"* ]]; - then - date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" - for SERVER in "${SERVERS[@]}" - do - ## Get next server's height and consensus - SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync") - HEIGHT=$( echo "$SERVERINFO" | jq '.height') - CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') - - ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch - if [[ -n "$HEIGHT" ]]; - then - diff=$(( $HEIGHTLOCAL - $HEIGHT )) - else - diff="999" - fi - ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate - if [ "$diff" -lt "3" ]; - then - DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') - if [ "$DISABLEFORGE" = "true" ]; - then - curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable - date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${RESETCOLOR}" - ChangeDirectory - bash lisk.sh reload - sleep 20 - SyncState - break - else - date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 with low consensus before forging${RESETCOLOR}" - fi - fi - done - fi - fi ## Check log if node is recovering close to forging time LASTLINE=$( echo "$LOG" | grep 'starting recovery') if [[ -n "$LASTLINE" ]]; From c04d67b0f000249944bb5c7033f853cce647d6d2 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Tue, 27 Dec 2016 16:57:46 -0600 Subject: [PATCH 11/22] Correct fork if statement --- check_consensus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_consensus.sh b/check_consensus.sh index dd9e003..d7bdbee 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -139,7 +139,7 @@ do FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block') if [ -n "$INADEQUATE" ] || ([ -n "$FORK" ] && [ -n "$FORGEDBLOCKLOG" ]); then - if [ -n "$FORK" ]; + if [ -n "$FORK" ] && [ -n "$FORGEDBLOCKLOG" ]; then date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Fork and Forged in log.${RESETCOLOR}" else From d35ed4712b1bea41e4c8443e5beca5cc367d9377 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Tue, 27 Dec 2016 20:06:14 -0600 Subject: [PATCH 12/22] Adde low consensus reload back in with better code --- check_consensus.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/check_consensus.sh b/check_consensus.sh index d7bdbee..e6067f6 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -165,6 +165,56 @@ do date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1.${RESETCOLOR}" fi fi + + ## If consensus is less than 51 and we are forging soon (but not one of the next 2), try a reload to get new peers + ## Management script should switch forging server during reload + ## from Nerigal + if [ "$CONSENSUSLOCAL" -lt "51" ]; + then + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $pbk${RESETCOLOR}" + DELEGATESNEXT=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers?limit=2" | jq '.delegates') + if [[ $DELEGATESNEXT != *"$pbk"* ]]; + then + DELEGATESSOON=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') + if [[ $DELEGATESSOON == *"$pbk"* ]]; + then + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" + for SERVER in "${SERVERS[@]}" + do + ## Get next server's height and consensus + SERVERINFO=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SERVER""$PRT"/api/loader/status/sync") + HEIGHT=$( echo "$SERVERINFO" | jq '.height') + CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus') + + ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch + if [[ -n "$HEIGHT" ]]; + then + diff=$(( $HEIGHTLOCAL - $HEIGHT )) + else + diff="999" + fi + ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate + if [ "$diff" -lt "3" ]; + then + DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') + if [ "$DISABLEFORGE" = "true" ]; + then + curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${RESETCOLOR}" + ChangeDirectory + bash lisk.sh reload + sleep 20 + SyncState + break + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 with low consensus before forging${RESETCOLOR}" + fi + fi + done + fi + fi + fi + (( ++TXTDELAY )) if [[ "$TXTDELAY" -gt "60" ]]; ## Wait 30 seconds to update running status to not overcrowd log then From fcd1125f58b67887e6cb49dbc8d36697495ca995 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 28 Dec 2016 13:10:29 -0600 Subject: [PATCH 13/22] Increase forging log delay --- check_consensus.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index e6067f6..aa68aea 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -11,7 +11,7 @@ LDIRECTORY=$( echo "$CONFIGFILE" | jq -r '.lisk_directory') SRV1=$( echo "$CONFIGFILE" | jq -r '.srv1') PRT=$( echo "$CONFIGFILE" | jq -r '.port') PRTS=$( echo "$CONFIGFILE" | jq -r '.https_port') -pbk=$( echo "$CONFIGFILE" | jq -r '.pbk') +PBK=$( echo "$CONFIGFILE" | jq -r '.PBK') SERVERS=() ### Get servers array size=$( echo "$CONFIGFILE" | jq '.servers | length') @@ -67,7 +67,7 @@ function SyncState() while true; do ## Get forging status of server - FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled') + FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$PBK| jq '.enabled') if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging then ## Get current server's height and consensus @@ -86,7 +86,7 @@ do if [[ -n "$FORGEDBLOCKLOG" ]]; then date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" - FORGEDDELAY=40 + FORGEDDELAY=60 fi elif [[ "$FORGEDDELAY" -gt "0" ]] then @@ -98,8 +98,8 @@ do if [[ -n "$LASTLINE" ]]; then delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') - date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering. Looking for delegate forging soon matching $pbk${RESETCOLOR}" - if [[ $delegates == *"$pbk"* ]]; + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering. Looking for delegate forging soon matching $PBK${RESETCOLOR}" + if [[ $delegates == *"$PBK"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}" for SERVER in "${SERVERS[@]}" @@ -112,12 +112,12 @@ do ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch if [[ -n "$HEIGHT" ]]; then - diff=$(( $HEIGHTLOCAL - $HEIGHT )) + DIFF=$(( $HEIGHTLOCAL - $HEIGHT )) else - diff="999" + DIFF="999" fi - ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate - if [ "$diff" -lt "3" ]; + ## if [ "$DIFF" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate + if [ "$DIFF" -lt "3" ]; then DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') if [ "$DISABLEFORGE" = "true" ]; @@ -171,12 +171,12 @@ do ## from Nerigal if [ "$CONSENSUSLOCAL" -lt "51" ]; then - date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $pbk${RESETCOLOR}" + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL. Looking for delegate forging soon matching $PBK${RESETCOLOR}" DELEGATESNEXT=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers?limit=2" | jq '.delegates') - if [[ $DELEGATESNEXT != *"$pbk"* ]]; + if [[ $DELEGATESNEXT != *"$PBK"* ]]; then DELEGATESSOON=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates') - if [[ $DELEGATESSOON == *"$pbk"* ]]; + if [[ $DELEGATESSOON == *"$PBK"* ]]; then date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}" for SERVER in "${SERVERS[@]}" @@ -189,17 +189,17 @@ do ## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch if [[ -n "$HEIGHT" ]]; then - diff=$(( $HEIGHTLOCAL - $HEIGHT )) + DIFF=$(( $HEIGHTLOCAL - $HEIGHT )) else - diff="999" + DIFF="999" fi - ## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate - if [ "$diff" -lt "3" ]; + ## if [ "$DIFF" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate + if [ "$DIFF" -lt "3" ]; then DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success') if [ "$DISABLEFORGE" = "true" ]; then - curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable + curl -s -S --connect-timeout 2 --retry 2 --retry-delay 0 --retry-max-time 4 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${RESETCOLOR}" ChangeDirectory bash lisk.sh reload From 2f3b84c582e80500c950ca77b666cebf9455db5f Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 28 Dec 2016 13:26:04 -0600 Subject: [PATCH 14/22] Better way to record forged blocks --- check_consensus.sh | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/check_consensus.sh b/check_consensus.sh index aa68aea..4717c8f 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -25,9 +25,9 @@ done ### ######################### -#Set text delay and forging log delays +#Set text delay and forging log TXTDELAY=1 -FORGEDDELAY=0 +LASTFORGED="" # Set colors RED=$(tput setaf 1) @@ -77,20 +77,13 @@ do ## Get recent log LOG=$(tail ~/lisk-main/logs/lisk.log -n 10) - ## Only look for forged block in log if we didn't just log it - if [[ "$FORGEDDELAY" -eq "0" ]]; - then - ## Log when a block is forged - FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block') - - if [[ -n "$FORGEDBLOCKLOG" ]]; - then - date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" - FORGEDDELAY=60 - fi - elif [[ "$FORGEDDELAY" -gt "0" ]] + ## Look for a forged block in logs + FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block') + ## Display in log if a new block forged and we didn't just display this one + if [ -n "$FORGEDBLOCKLOG" ] && [ "$LASTFORGED" != "$FORGEDBLOCKLOG" ]; then - ((FORGEDDELAY--)) + date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}" + LASTFORGED=$FORGEDBLOCKLOG fi ## Check log if node is recovering close to forging time From 863545948999f805857d61b317322945c906b1d0 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 28 Dec 2016 13:32:41 -0600 Subject: [PATCH 15/22] Public Key Correction --- check_consensus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_consensus.sh b/check_consensus.sh index 4717c8f..ddb7a32 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -11,7 +11,7 @@ LDIRECTORY=$( echo "$CONFIGFILE" | jq -r '.lisk_directory') SRV1=$( echo "$CONFIGFILE" | jq -r '.srv1') PRT=$( echo "$CONFIGFILE" | jq -r '.port') PRTS=$( echo "$CONFIGFILE" | jq -r '.https_port') -PBK=$( echo "$CONFIGFILE" | jq -r '.PBK') +PBK=$( echo "$CONFIGFILE" | jq -r '.pbk') SERVERS=() ### Get servers array size=$( echo "$CONFIGFILE" | jq '.servers | length') From 791383c8be29c68739afe940d0b4f10e018d4c99 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 28 Dec 2016 19:30:59 -0600 Subject: [PATCH 16/22] Add extra log entry for within 20 seconds --- check_consensus.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/check_consensus.sh b/check_consensus.sh index ddb7a32..aee6402 100644 --- a/check_consensus.sh +++ b/check_consensus.sh @@ -196,7 +196,7 @@ do date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low. We will try a reload.${RESETCOLOR}" ChangeDirectory bash lisk.sh reload - sleep 20 + sleep 15 SyncState break else @@ -205,6 +205,8 @@ do fi done fi + else + date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging within next 20 seconds, too soon to try a reload.${RESETCOLOR}" fi fi From c4924dc8b22cca4f0338ac8a308e0e814832cdc2 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Sat, 31 Dec 2016 19:28:13 -0600 Subject: [PATCH 17/22] Make sure forging was enable when none forging --- manage.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/manage.sh b/manage.sh index 64072d1..19695f3 100644 --- a/manage.sh +++ b/manage.sh @@ -1,4 +1,4 @@ -## Version 0.9.3 +## Version 0.9.4 #!/bin/bash ## Check for config file @@ -99,9 +99,15 @@ while true; do if [ "$DIFF" -lt "4" ] && [ "${SERVERSCONSENSUS[$NUM]}" -gt "50" ]; then date +"%Y-%m-%d %H:%M:%S || ${YELLOW}No node forging. Starting on $SERVER${RESETCOLOR}" - curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable - PREVIOUSFORGING=$NUM - break ## Exit loop once we find the first server at an acceptable height and consensus + ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success') + if [ "$ENABLEFORGE" = "true" ]; + then + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER to try and forge.${RESETCOLOR}" + PREVIOUSFORGING=$NUM + break ## Leave servers loop + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on $SERVER. Trying next server.${RESETCOLOR}" + fi fi ((NUM++)) done From 50833a8f68db461067b81242b84e4471f1aba4b5 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Sat, 31 Dec 2016 19:50:41 -0600 Subject: [PATCH 18/22] Check enable worked --- manage.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/manage.sh b/manage.sh index 19695f3..c7790b0 100644 --- a/manage.sh +++ b/manage.sh @@ -134,10 +134,15 @@ while true; do DIFF=$(( HIGHHEIGHT - ${SERVERSINFO[$index]} )) if [ "$DIFF" -lt "4" ] && [ "${SERVERSCONSENSUS[$NUM]}" -gt "50" ]; then - curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"${SERVERS[$index]}""$PRTS"/api/delegates/forging/enable - PREVIOUSFORGING=$index - date +"%Y-%m-%d %H:%M:%S || ${CYAN}Setting forging to ${SERVERS[$index]}${RESETCOLOR}" - break ## Exit loop once we find the first server at an acceptable height and consensus + ENABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"${SERVERS[$index]}""$PRTS"/api/delegates/forging/enable | jq '.success') + if [ "$ENABLEFORGE" = "true" ]; + then + PREVIOUSFORGING=$index + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Setting forging to ${SERVERS[$index]}${RESETCOLOR}" + break ## Exit loop once we find the first server at an acceptable height and consensus + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on ${SERVERS[$index]}. Trying next server.${RESETCOLOR}" + fi fi done fi @@ -178,10 +183,15 @@ while true; do DIFF=$(( HIGHHEIGHT - ${SERVERSINFO[$index]} )) if [ "$DIFF" -lt "4" ] && [ "${SERVERSCONSENSUS[$NUM]}" -gt "50" ]; then - curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"${SERVERS[$index]}""$PRTS"/api/delegates/forging/enable - FORGING=$index - date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to ${SERVERS[$index]}${RESETCOLOR}" - break ## Exit loop once we find the first server at an acceptable height and consensus + ENABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"${SERVERS[$index]}""$PRTS"/api/delegates/forging/enable | jq '.success') + if [ "$ENABLEFORGE" = "true" ]; + then + FORGING=$index + date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to ${SERVERS[$index]}${RESETCOLOR}" + break ## Exit loop once we find the first server at an acceptable height and consensus + else + date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on ${SERVERS[$index]}. Trying next server.${RESETCOLOR}" + fi fi done fi From 3bf0f09ce5defc388abbe23d7f7d01795a287315 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 4 Jan 2017 12:38:56 -0600 Subject: [PATCH 19/22] Change wait to 15 from 30 seconds --- manage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manage.sh b/manage.sh index c7790b0..32ef382 100644 --- a/manage.sh +++ b/manage.sh @@ -149,8 +149,8 @@ while true; do if [[ $PREVIOUSFORGING != "$FORGING" ]]; then - date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Different server forging! Previous=${SERVERS[$PREVIOUSFORGING]},Current=${SERVERS[$FORGING]}. Waiting 30 seconds${RESETCOLOR}" - sleep 24 + date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Different server forging! Previous=${SERVERS[$PREVIOUSFORGING]},Current=${SERVERS[$FORGING]}. Waiting 15 seconds${RESETCOLOR}" + sleep 9 else ## Same server still forging, check that everything still looks good on it date +"%Y-%m-%d %H:%M:%S || Highest Height: $HEIGHT" From 391bc0813beff2f4632d744dbff8bb1c0d8801b2 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 4 Jan 2017 14:04:17 -0600 Subject: [PATCH 20/22] Add logs option with multitail --- control.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/control.sh b/control.sh index bbd6816..8608899 100644 --- a/control.sh +++ b/control.sh @@ -1,5 +1,5 @@ ## control.sh -## Version 0.9.3 +## Version 0.9.4 ## Tested with jq 1.5.1 on Ubuntu 16.04.1 #!/bin/bash @@ -170,6 +170,20 @@ status() { fi } +logs() { + LOG_FILES="" + if [[ -e "$LOG_FILE" ]] ; then + LOG_FILES+=$LOG_FILE" " + fi + if [[ -e "$CONSENSUS_LOG_FILE" ]] ; then + LOG_FILES+=$CONSENSUS_LOG_FILE" " + fi + if [[ -e "$MANAGE_LOG_FILE" ]] ; then + LOG_FILES+=$MANAGE_LOG_FILE" " + fi + multitail -cT ansi $LOG_FILES +} + usage() { echo "Usage: $0 " echo "start -- starts consensus & height_rebuild scripts" @@ -181,6 +195,7 @@ usage() { echo "stoph -- stops height_rebuild script" echo "stopm -- stops manage script" echo "status -- check if scripts are running" + echo "logs -- display all logs (requires multitail)" echo "upgrade -- upgrades and runs both scripts" } @@ -220,6 +235,9 @@ case $1 in "status" ) status ;; +"logs" ) + logs +;; "upgrade" ) check_height_running check_consensus_running From 4610bfa9036ce0703fa3f4661252b81f3bdddb82 Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 4 Jan 2017 14:08:04 -0600 Subject: [PATCH 21/22] Enable colors for all log files --- control.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/control.sh b/control.sh index 8608899..d3a29d1 100644 --- a/control.sh +++ b/control.sh @@ -173,15 +173,15 @@ status() { logs() { LOG_FILES="" if [[ -e "$LOG_FILE" ]] ; then - LOG_FILES+=$LOG_FILE" " + LOG_FILES+="-cT ANSI "$LOG_FILE" " fi if [[ -e "$CONSENSUS_LOG_FILE" ]] ; then - LOG_FILES+=$CONSENSUS_LOG_FILE" " + LOG_FILES+="-cT ANSI "$CONSENSUS_LOG_FILE" " fi if [[ -e "$MANAGE_LOG_FILE" ]] ; then - LOG_FILES+=$MANAGE_LOG_FILE" " + LOG_FILES+="-cT ANSI "$MANAGE_LOG_FILE" " fi - multitail -cT ansi $LOG_FILES + multitail $LOG_FILES } usage() { From c90f05b5b6336ad9f3a5fda54789a2a11e135cfd Mon Sep 17 00:00:00 2001 From: mrv777 Date: Wed, 4 Jan 2017 14:10:49 -0600 Subject: [PATCH 22/22] Update to 0.9.4 --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28a8d01..8ca24bb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Lisk Delegate Scripts (BETA v0.9.3) +# Lisk Delegate Scripts (BETA v0.9.4) ## Thank you Thank you to cc001, corsaro, liberspirita, wannabe_RoteBaron, hagie, isabella, stoner19, punkrock, Nerigal, doweig, and anyone I might have missed for their help and/or contributions. @@ -12,6 +12,7 @@ This is the wrapper script for check_height_and_rebuild.sh and check_consensus.s * stop -- stops both scripts * stopc -- stops consensus script * stoph -- stops height_rebuild script +* logs -- display all log files (requires multitail) * upgrade -- upgrades and runs runs both scripts #### How to run: @@ -27,6 +28,7 @@ To check the logs and what the script is going: * `tail -f heightRebuild.log` * `tail -f consensus.log` +* `bash control.sh logs` (requires multitail) ## My consensus check script @@ -49,7 +51,7 @@ Compares the height of your 100 connected peers and gets the highest height. Th ### manage.sh **User does not need to directly do anything with this. control.sh interfaces with it automatically** -This script will check the block heights of all listed servers. It will attempt to have one and only one server forging and prefer to have server 1 do the forging. If the forging server swtiches by something other then this script, it will pause for 30 seconds and then check the status of the servers after that. +This script will check the block heights of all listed servers. It will attempt to have one and only one server forging and prefer to have server 1 do the forging. If the forging server swtiches by something other then this script, it will pause for 15 seconds and then check the status of the servers after that. #### How to run: @@ -60,5 +62,6 @@ This script will check the block heights of all listed servers. It will attempt To check the logs and what the script is going: * `tail -f manage.log` +* `bash control.sh logs` (requires multitail) *Donation Address: 3532362465127676136L*