0.9.4 Update

0.9.4 Update

README.md

@@ -1,4 +1,4 @@
-# Lisk Delegate Scripts  (BETA v0.9.3)
+# Lisk Delegate Scripts  (BETA v0.9.4)
 
 ## Thank you
 Thank you to cc001, corsaro, liberspirita, wannabe_RoteBaron, hagie, isabella, stoner19, punkrock, Nerigal, doweig, and anyone I might have missed for their help and/or contributions.
@@ -12,6 +12,7 @@ This is the wrapper script for check_height_and_rebuild.sh and check_consensus.s
 * stop              -- stops both scripts
 * stopc             -- stops consensus script
 * stoph             -- stops height_rebuild script
+* logs              -- display all log files (requires multitail)
 * upgrade           -- upgrades and runs runs both scripts
 
 #### How to run:
@@ -27,6 +28,7 @@ To check the logs and what the script is going:
 
 * `tail -f heightRebuild.log`
 * `tail -f consensus.log`
+* `bash control.sh logs` (requires multitail)
 
 ## My consensus check script
 
@@ -49,7 +51,7 @@ Compares the height of your 100 connected peers and gets the highest height.  Th
 
 ### manage.sh
 **User does not need to directly do anything with this.  control.sh interfaces with it automatically**
-This script will check the block heights of all listed servers.  It will attempt to have one and only one server forging and prefer to have server 1 do the forging.  If the forging server swtiches by something other then this script, it will pause for 30 seconds and then check the status of the servers after that.
+This script will check the block heights of all listed servers.  It will attempt to have one and only one server forging and prefer to have server 1 do the forging.  If the forging server swtiches by something other then this script, it will pause for 15 seconds and then check the status of the servers after that.
 
 #### How to run:
 
@@ -60,5 +62,6 @@ This script will check the block heights of all listed servers.  It will attempt
 To check the logs and what the script is going:
 
 * `tail -f manage.log`
+* `bash control.sh logs` (requires multitail)
 
 *Donation Address: 3532362465127676136L*

check_consensus.sh

@@ -1,4 +1,4 @@
-## Version 0.9.2
+## Version 0.9.4
 #!/bin/bash
 
 ## Check for config file
@@ -11,32 +11,33 @@ LDIRECTORY=$( echo "$CONFIGFILE" | jq -r '.lisk_directory')
 SRV1=$( echo "$CONFIGFILE" | jq -r '.srv1')
 PRT=$( echo "$CONFIGFILE" | jq -r '.port')
 PRTS=$( echo "$CONFIGFILE" | jq -r '.https_port')
-pbk=$( echo "$CONFIGFILE" | jq -r '.pbk')
+PBK=$( echo "$CONFIGFILE" | jq -r '.pbk')
 SERVERS=()
 ### Get servers array
 size=$( echo "$CONFIGFILE" | jq '.servers | length') 
 i=0
 
-while [ $i -le $size ]    
+while [ $i -le "$size" ]    
 do
 	SERVERS[$i]=$(echo "$CONFIGFILE" | jq -r --argjson i $i '.servers[$i]')
-    i=`expr $i + 1`
+	i=$((i + 1))
 done
 ###
 #########################
 
-#Set text delay at 0
-TXTDELAY=0
+#Set text delay and forging log
+TXTDELAY=1
+LASTFORGED=""
 
 # Set colors
-red=`tput setaf 1`
-green=`tput setaf 2`
-yellow=`tput setaf 3`
-cyan=`tput setaf 6`
-resetColor=`tput sgr0`
+RED=$(tput setaf 1)
+GREEN=$(tput setaf 2)
+YELLOW=$(tput setaf 3)
+CYAN=$(tput setaf 6)
+RESETCOLOR=$(tput sgr0)
 
 ## Log start of script
-date +"%Y-%m-%d %H:%M:%S || ${green}Starting MrV's consensus script${resetColor}"
+date +"%Y-%m-%d %H:%M:%S || ${GREEN}Starting MrV's consensus script${RESETCOLOR}"
 
 
 # Set Lisk directory
@@ -54,10 +55,10 @@ function SyncState()
 	while [[ -z $result || $result != 'false' ]]
 	do
 		date +"%Y-%m-%d %H:%M:%S || Blockchain syncing"
-		result=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync" | jq '.syncing')
+		result=$(curl --connect-timeout 3 -s "http://$SRV1$PRT/api/loader/status/sync" | jq '.syncing')
 		sleep 2
 	done
-	
+
 	date +"%Y-%m-%d %H:%M:%S || Looks like blockchain is finished syncing."
 }
 #---------------------------------------------------------------------------
@@ -66,153 +67,163 @@ function SyncState()
 while true;
 do
 	## Get forging status of server
-	FORGE=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$pbk| jq '.enabled')
+	FORGE=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/delegates/forging/status?publicKey="$PBK| jq '.enabled')
 	if [[ "$FORGE" == "true" ]]; ## Only check log and try to switch forging if needed, if server is currently forging
 	then
-		## Log when a block is forged
-		FORGEDBLOCKLOG=$(tail ~/lisk-main/logs/lisk.log -n 20| grep 'Forged new block')
-		if [[ -n "$FORGEDBLOCKLOG" ]];
-		then
-			date +"%Y-%m-%d %H:%M:%S || ${green}$FORGEDBLOCKLOG${resetColor}"
-		fi
 		## Get current server's height and consensus
-		SERVERLOCAL=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync")
+		SERVERLOCAL=$(curl --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -s "http://"$SRV1""$PRT"/api/loader/status/sync")
 		HEIGHTLOCAL=$( echo "$SERVERLOCAL" | jq '.height')
 		CONSENSUSLOCAL=$( echo "$SERVERLOCAL" | jq '.consensus')
+		## Get recent log
+		LOG=$(tail ~/lisk-main/logs/lisk.log -n 10)
 
-		## If consensus is less than 51 and we are forging soon, try a reload to get new peers
-		## Management script should switch forging server during reload
-		## from Nerigal
-		if [ "$CONSENSUSLOCAL" -lt "51" ];
+		## Look for a forged block in logs
+		FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block')
+		## Display in log if a new block forged and we didn't just display this one
+		if [ -n "$FORGEDBLOCKLOG" ] && [ "$LASTFORGED" != "$FORGEDBLOCKLOG" ];
+		then
+			date +"%Y-%m-%d %H:%M:%S || ${GREEN}$FORGEDBLOCKLOG${RESETCOLOR}"
+			LASTFORGED=$FORGEDBLOCKLOG
+		fi
+
+		## Check log if node is recovering close to forging time
+		LASTLINE=$( echo "$LOG" | grep 'starting recovery')
+		if [[ -n "$LASTLINE" ]];
 		then
 			delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates')
-			date +"%Y-%m-%d %H:%M:%S || ${yellow}Low consensus of $CONSENSUSLOCAL.  Looking for delegate forging soon matching $pbk${resetColor}"
-			if [[ $delegates == *"$pbk"* ]];
+			date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Node is recovering.  Looking for delegate forging soon matching $PBK${RESETCOLOR}"
+			if [[ $delegates == *"$PBK"* ]];
 			then
-				date +"%Y-%m-%d %H:%M:%S || ${red}You are forging soon, but your consensus is too low. Looking to switch server before reload.${resetColor}"
-				for SERVER in ${SERVERS[@]}
+			date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging soon, but your node is recovering. Looking to switch server while recovering.${RESETCOLOR}"
+				for SERVER in "${SERVERS[@]}"
 				do
 					## Get next server's height and consensus
 					SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync")
 					HEIGHT=$( echo "$SERVERINFO" | jq '.height')
 					CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus')
-				
+
 					## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch
 					if [[  -n "$HEIGHT" ]];
 					then
-						diff=$(( $HEIGHTLOCAL - $HEIGHT ))
+						DIFF=$(( $HEIGHTLOCAL - $HEIGHT ))
 					else
-						diff="999"
+						DIFF="999"
 					fi
-					## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate
-					if [ "$diff" -lt "3" ]; 
+					## if [ "$DIFF" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate
+					if [ "$DIFF" -lt "3" ]; 
 					then
 						DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success')
 						if [ "$DISABLEFORGE" = "true" ];
 						then
 							curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable
-							date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low.  We will try a reload.${resetColor}"
-							ChangeDirectory
-							bash lisk.sh reload
-							sleep 20
-							SyncState
+							date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your node is recovering.${RESETCOLOR}"
 							break
 						else
-							date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1 with low consensus before forging${resetColor}"
+							date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 that is recovering before forging${RESETCOLOR}"
 						fi
 					fi
 				done
 			fi
 		fi
-		## Check log if node is recovering close to forging time
-		LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 2| grep 'starting recovery')
-		if [[ -n "$LASTLINE" ]];
+
+		## Check log for Inadequate consensus or Fork & Forged while forging
+		INADEQUATE=$( echo "$LOG" | grep 'Inadequate')
+		FORK=$( echo "$LOG" | grep 'Fork')
+		FORGEDBLOCKLOG=$( echo "$LOG" | grep 'Forged new block')
+		if [ -n "$INADEQUATE" ] || ([ -n "$FORK" ] && [ -n "$FORGEDBLOCKLOG" ]);
 		then
-			delegates=$(curl --connect-timeout 3 -s "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates')
-			date +"%Y-%m-%d %H:%M:%S || ${yellow}Node is recovering.  Looking for delegate forging soon matching $pbk${resetColor}"
-			if [[ $delegates == *"$pbk"* ]];
+			if [ -n "$FORK" ] && [ -n "$FORGEDBLOCKLOG" ];
 			then
-			date +"%Y-%m-%d %H:%M:%S || ${red}You are forging soon, but your node is recovering. Looking to switch server while recovering.${resetColor}"
-				for SERVER in ${SERVERS[@]}
+				date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Fork and Forged in log.${RESETCOLOR}"
+			else
+				date +"%Y-%m-%d %H:%M:%S || ${RED}WARNING: Inadequate consensus to forge.${RESETCOLOR}"
+			fi
+
+			## Disable forging on local server first.  If successful, loop through servers until we are able to enable forging on one
+			DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success')
+			if [ "$DISABLEFORGE" = "true" ];
+			then
+				for SERVER in "${SERVERS[@]}"
 				do
-					## Get next server's height and consensus
-					SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync")
-					HEIGHT=$( echo "$SERVERINFO" | jq '.height')
-					CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus')
-
-					## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch
-					if [[  -n "$HEIGHT" ]];
+					ENABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 2 --retry-delay 0 --retry-max-time 2 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable | jq '.success')
+					if [ "$ENABLEFORGE" = "true" ];
 					then
-						diff=$(( $HEIGHTLOCAL - $HEIGHT ))
+						date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER to try and forge.${RESETCOLOR}"
+						break ## Leave servers loop
 					else
-						diff="999"
-					fi
-					## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate
-					if [ "$diff" -lt "3" ]; 
-					then
-						DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success')
-						if [ "$DISABLEFORGE" = "true" ];
-						then
-							curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable
-							date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS as your node is recovering.${resetColor}"
-							break
-						else
-							date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1 that is recovering before forging${resetColor}"
-						fi
+						date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to enable forging on $SERVER.  Trying next server.${RESETCOLOR}"
 					fi
 				done
+			else
+				date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1.${RESETCOLOR}"
 			fi
 		fi
 
-		## Check log for Inadequate consensus
-		LASTLINE=$(tail ~/lisk-main/logs/lisk.log -n 2| grep 'Inadequate')
-		if [[ -n "$LASTLINE" ]];
+		## If consensus is less than 51 and we are forging soon (but not one of the next 2), try a reload to get new peers
+		## Management script should switch forging server during reload
+		## from Nerigal
+		if [ "$CONSENSUSLOCAL" -lt "51" ];
 		then
-			date +"%Y-%m-%d %H:%M:%S || ${red}WARNING: $LASTLINE${resetColor}"
-
-			for SERVER in ${SERVERS[@]}
-			do
-				## Get next server's height and consensus
-				SERVERINFO=$(curl --connect-timeout 3 -s -S "http://"$SERVER""$PRT"/api/loader/status/sync")
-				HEIGHT=$( echo "$SERVERINFO" | jq '.height')
-				CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus')
-
-				## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch
-				if [[  -n "$HEIGHT" ]];
-				then
-					diff=$(( $HEIGHTLOCAL - $HEIGHT ))
-				else
-					diff="999"
-				fi
-				## if [ "$diff" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate
-				if [ "$diff" -lt "3" ]; 
+			date +"%Y-%m-%d %H:%M:%S || ${YELLOW}Low consensus of $CONSENSUSLOCAL.  Looking for delegate forging soon matching $PBK${RESETCOLOR}"
+			DELEGATESNEXT=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers?limit=2" | jq '.delegates')
+			if [[ $DELEGATESNEXT != *"$PBK"* ]];
+			then
+				DELEGATESSOON=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SRV1""$PRT"/api/delegates/getNextForgers" | jq '.delegates')
+				if [[ $DELEGATESSOON == *"$PBK"* ]];
 				then
-					DISABLEFORGE=$(curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success')
-					if [ "$DISABLEFORGE" = "true" ];
-					then
-						curl -s -S --connect-timeout 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable
-						date +"%Y-%m-%d %H:%M:%S || ${cyan}Switching to Server $SERVER with a consensus of $CONSENSUS to try and forge${resetColor}"
-					else
-						date +"%Y-%m-%d %H:%M:%S || ${red}Failed to disable forging on $SRV1${resetColor}"
-					fi
-					break
+					date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging in next 100 seconds, but your consensus is too low. Looking to switch server before reload.${RESETCOLOR}"
+					for SERVER in "${SERVERS[@]}"
+					do
+						## Get next server's height and consensus
+						SERVERINFO=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 "http://"$SERVER""$PRT"/api/loader/status/sync")
+						HEIGHT=$( echo "$SERVERINFO" | jq '.height')
+						CONSENSUS=$( echo "$SERVERINFO" | jq '.consensus')
+
+						## Make sure next server is not more than 3 blocks behind this server and consensus is better, then switch
+						if [[  -n "$HEIGHT" ]];
+						then
+							DIFF=$(( $HEIGHTLOCAL - $HEIGHT ))
+						else
+							DIFF="999"
+						fi
+						## if [ "$DIFF" -lt "3" ] && [ "$CONSENSUS" -gt "$CONSENSUSLOCAL" ]; ## Removed for now as I believe consensus read from API isn't updated every second to be fully accurate
+						if [ "$DIFF" -lt "3" ]; 
+						then
+							DISABLEFORGE=$(curl -s -S --connect-timeout 1 --retry 3 --retry-delay 0 --retry-max-time 3 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SRV1""$PRTS"/api/delegates/forging/disable | jq '.success')
+							if [ "$DISABLEFORGE" = "true" ];
+							then
+								curl -s -S --connect-timeout 2 --retry 2 --retry-delay 0 --retry-max-time 4 -k -H "Content-Type: application/json" -X POST -d '{"secret":"'"$SECRET"'"}' https://"$SERVER""$PRTS"/api/delegates/forging/enable
+								date +"%Y-%m-%d %H:%M:%S || ${CYAN}Switching to Server $SERVER with a consensus of $CONSENSUS as your consensus is too low.  We will try a reload.${RESETCOLOR}"
+								ChangeDirectory
+								bash lisk.sh reload
+								sleep 15
+								SyncState
+								break
+							else
+								date +"%Y-%m-%d %H:%M:%S || ${RED}Failed to disable forging on $SRV1 with low consensus before forging${RESETCOLOR}"
+							fi
+						fi
+					done
 				fi
-			done
+			else
+				date +"%Y-%m-%d %H:%M:%S || ${RED}You are forging within next 20 seconds, too soon to try a reload.${RESETCOLOR}"
+			fi
 		fi
+
 		(( ++TXTDELAY ))
-		if [[ "$TXTDELAY" -eq "30" ]];  ## Wait 30 seconds to update running status to not overcrowd log
+		if [[ "$TXTDELAY" -gt "60" ]];  ## Wait 30 seconds to update running status to not overcrowd log
 		then
-			date +"%Y-%m-%d %H:%M:%S || ${green}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${resetColor}"
-			TXTDELAY=0
+			date +"%Y-%m-%d %H:%M:%S || ${GREEN}Still working at block $HEIGHTLOCAL with a consensus of $CONSENSUSLOCAL${RESETCOLOR}"
+			TXTDELAY=1
 		fi
-			sleep 1
+		sleep 0.5
 	else
 		(( ++TXTDELAY ))
-		if [[ "$TXTDELAY" -eq "30" ]];  ## Wait 30 seconds to update running status to not overcrowd log
+		if [[ "$TXTDELAY" -gt "30" ]];  ## Wait 30 seconds to update running status to not overcrowd log
 		then
 			date +"%Y-%m-%d %H:%M:%S || This server is not forging"
-			TXTDELAY=0
+			TXTDELAY=1
 		fi
-			sleep 1
+		sleep 1
 	fi
 done