From b42b6c0370a27747370def4503f50d410774a787 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 13 Apr 2021 09:48:06 -0400
Subject: [PATCH 01/26] Update blacklist.sh

---
 blacklist.sh | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 7b87bdb..432d106 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -1,18 +1,29 @@
 #!/bin/bash
+echo "Blacklist update started" > /config/scripts/blacklist-processing.txt ; 
+date >> /config/scripts/blacklist-processing.txt ;
 
-real_list=`grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}'`
-[[ -z "$real_list" ]] && { echo "aborting"; exit -1; } || echo "Updating $real_list"
+real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}'); [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list";
 
-ipset_list='temporary-list'
+ipset_list='temporary-list' ;
 
-sudo /sbin/ipset -! destroy $ipset_list
-sudo /sbin/ipset create $ipset_list hash:net
+sudo /sbin/ipset -! destroy $ipset_list ;
+sudo /sbin/ipset create $ipset_list hash:net ;
 
-for url in 'https://www.spamhaus.org/drop/edrop.txt' 'http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt' 'https://check.torproject.org/cgi-bin/TorBulkExitList.py?ip=1.1.1.1';
-do
- echo "Fetching and processing $url"
- curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 sudo ipset -q add $ipset_list
-done
+for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
+do echo "Fetching and processing $url" ;
+ { echo "Processing blacklist" ; 
+  date ; 
+  echo $url ;
+ } >> /config/scripts/blacklist-processing.txt ;
+ curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 sudo ipset -exist add $ipset_list ;
+done ;
 
-sudo /sbin/ipset swap $ipset_list $real_list
-sudo /sbin/ipset destroy $ipset_list
+sudo /sbin/ipset swap $ipset_list "$real_list" ;
+
+{ echo "Blacklist update finished" ; 
+ date ; 
+ echo "Blacklist contents" ; 
+ sudo /sbin/ipset list -s "$real_list" ;
+ } >> /config/scripts/blacklist-processing.txt ;
+ 
+sudo /sbin/ipset destroy $ipset_list ;

From a9ac1d8d4721a86236b1f1a3fb8fc41f07518325 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 13 Apr 2021 10:22:55 -0400
Subject: [PATCH 02/26] Update config.gateway.json

---
 config.gateway.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.gateway.json b/config.gateway.json
index 3f02356..adcbedd 100644
--- a/config.gateway.json
+++ b/config.gateway.json
@@ -4,7 +4,8 @@
             "task": {
                 "blacklist": {
                     "executable": {
-                        "path": "/config/scripts/blacklist.sh"
+						"path": "/bin/bash",
+                        "arguments": "/config/scripts/blacklist.sh"
                     },
                     "interval": "24h"
                 }

From 6ab3525dd54d8c32b2122156d1cba458a125237f Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 13 Apr 2021 10:23:14 -0400
Subject: [PATCH 03/26] Update config.gateway.json

---
 config.gateway.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.gateway.json b/config.gateway.json
index adcbedd..d343f2f 100644
--- a/config.gateway.json
+++ b/config.gateway.json
@@ -4,7 +4,7 @@
             "task": {
                 "blacklist": {
                     "executable": {
-						"path": "/bin/bash",
+			"path": "/bin/bash",
                         "arguments": "/config/scripts/blacklist.sh"
                     },
                     "interval": "24h"

From 212586cf16ebff65ed86cf054cf03f087657c532 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Mon, 19 Apr 2021 13:35:59 -0400
Subject: [PATCH 04/26] Update blacklist.sh

---
 blacklist.sh | 74 +++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 432d106..69bb44e 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -1,29 +1,61 @@
 #!/bin/bash
-echo "Blacklist update started" > /config/scripts/blacklist-processing.txt ; 
-date >> /config/scripts/blacklist-processing.txt ;
+{
+echo "Blacklist update started"
+date
+}  > /config/scripts/blacklist-processing.txt
 
-real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}'); [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list";
+real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
+[[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
 
-ipset_list='temporary-list' ;
+ipset_list="temporary-list"
 
-sudo /sbin/ipset -! destroy $ipset_list ;
-sudo /sbin/ipset create $ipset_list hash:net ;
+usgupt=$(uptime | awk '{print $4}')
 
-for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
-do echo "Fetching and processing $url" ;
- { echo "Processing blacklist" ; 
-  date ; 
-  echo $url ;
- } >> /config/scripts/blacklist-processing.txt ;
- curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 sudo ipset -exist add $ipset_list ;
-done ;
+if [ $usgupt == "min," ]
+then
+	sbin/ipset restore -f /config/scripts/blacklist-backup.bak
+	/sbin/ipset swap $ipset_list "$real_list"
+	/sbin/ipset -! destroy $ipset_list
+	{
+	echo "USG uptime is less than one hour, loading previous version of blacklist"
+	date
+	echo "Blacklist contents"
+	/sbin/ipset list -s "$real_list"
+	} >> /config/scripts/blacklist-processing.txt
+else
+	/sbin/ipset -! destroy $ipset_list
+	/sbin/ipset create $ipset_list hash:net
 
-sudo /sbin/ipset swap $ipset_list "$real_list" ;
+	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
+	do
+	echo "Fetching and processing $url"
+	{
+	echo "Processing blacklist"
+	date
+	echo $url
+	} >> /config/scripts/blacklist-processing.txt
+	curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 ipset -! add $ipset_list
+	done
 
-{ echo "Blacklist update finished" ; 
- date ; 
- echo "Blacklist contents" ; 
- sudo /sbin/ipset list -s "$real_list" ;
- } >> /config/scripts/blacklist-processing.txt ;
+	tlcontents=$(sudo /sbin/ipset list temporary-list | grep -A1 "Members:" | sed -n '2p')
+
+	if [ -z $tlcontents ]
+	then 
+		{
+		echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place."
+		date
+		} >> /config/scripts/blacklist-processing.txt
+	else 
+		/sbin/ipset save $ipset_list -f /config/scripts/blacklist-backup.bak
+		/sbin/ipset swap $ipset_list "$real_list"
+	fi
+
+	{
+	echo "Blacklist update finished"
+	date
+	echo "Blacklist contents"
+	/sbin/ipset list -s "$real_list"
+	} >> /config/scripts/blacklist-processing.txt
  
-sudo /sbin/ipset destroy $ipset_list ;
+	/sbin/ipset destroy $ipset_list
+fi

From 9055ffed43206aca76c464fb03e336a6e78810e3 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Mon, 19 Apr 2021 13:36:40 -0400
Subject: [PATCH 05/26] Update config.gateway.json

---
 config.gateway.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/config.gateway.json b/config.gateway.json
index d343f2f..3f02356 100644
--- a/config.gateway.json
+++ b/config.gateway.json
@@ -4,8 +4,7 @@
             "task": {
                 "blacklist": {
                     "executable": {
-			"path": "/bin/bash",
-                        "arguments": "/config/scripts/blacklist.sh"
+                        "path": "/config/scripts/blacklist.sh"
                     },
                     "interval": "24h"
                 }

From ccfa93c680edba750a8122ec84d0cd47c84e4790 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Mon, 19 Apr 2021 13:37:12 -0400
Subject: [PATCH 06/26] Update blacklist.sh

---
 blacklist.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blacklist.sh b/blacklist.sh
index 69bb44e..44f3997 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -4,7 +4,7 @@ echo "Blacklist update started"
 date
 }  > /config/scripts/blacklist-processing.txt
 
-real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
+real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}')
 [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
 
 ipset_list="temporary-list"

From 8ea2be98ec32f2f79ed3b40ab799ef43091b73fc Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Mon, 19 Apr 2021 13:42:00 -0400
Subject: [PATCH 07/26] Update blacklist.sh

---
 blacklist.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/blacklist.sh b/blacklist.sh
index 44f3997..855ebc1 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -46,6 +46,10 @@ else
 		date
 		} >> /config/scripts/blacklist-processing.txt
 	else 
+		{
+		echo "Blacklist is updated and backed up"
+		date
+		} >> /config/scripts/blacklist-processing.txt
 		/sbin/ipset save $ipset_list -f /config/scripts/blacklist-backup.bak
 		/sbin/ipset swap $ipset_list "$real_list"
 	fi

From 62adbbecafd930ba3849439d3dc684dedb70b975 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 20 Apr 2021 11:16:39 -0400
Subject: [PATCH 08/26] Update blacklist.sh

---
 blacklist.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 855ebc1..4c78029 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -13,7 +13,7 @@ usgupt=$(uptime | awk '{print $4}')
 
 if [ $usgupt == "min," ]
 then
-	sbin/ipset restore -f /config/scripts/blacklist-backup.bak
+	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
 	/sbin/ipset swap $ipset_list "$real_list"
 	/sbin/ipset -! destroy $ipset_list
 	{
@@ -28,13 +28,13 @@ else
 
 	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
 	do
-	echo "Fetching and processing $url"
-	{
-	echo "Processing blacklist"
-	date
-	echo $url
-	} >> /config/scripts/blacklist-processing.txt
-	curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 ipset -! add $ipset_list
+		echo "Fetching and processing $url"
+		{
+		echo "Processing blacklist"
+		date
+		echo $url
+		} >> /config/scripts/blacklist-processing.txt
+		curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list
 	done
 
 	tlcontents=$(sudo /sbin/ipset list temporary-list | grep -A1 "Members:" | sed -n '2p')

From a0d530f7384fbcab3c44c97b541532617f2d6ec6 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 9 Nov 2021 10:39:26 -0500
Subject: [PATCH 09/26] Update blacklist.sh

Updated the script to put the blacklist creation steps into a function so that an if, then, elif, then, else statement could be created.  This new series of checks determines if the blacklist backup file exists on a reboot.  If it does, it restores that backup list to speed up provisioning.  If it does not then it creates a new list on reboot to ensure the device is protected rather than waiting until the next scheduled interval to process and create the list.  The list will still be processed and updated at the scheduled interval as well.  Additional logging is included to state whether the list is being restored from a backup or processed as new.
---
 blacklist.sh | 47 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 4c78029..4c7e9b7 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -4,25 +4,16 @@ echo "Blacklist update started"
 date
 }  > /config/scripts/blacklist-processing.txt
 
-real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}')
+real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
 [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
 
 ipset_list="temporary-list"
 
 usgupt=$(uptime | awk '{print $4}')
 
-if [ $usgupt == "min," ]
-then
-	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
-	/sbin/ipset swap $ipset_list "$real_list"
-	/sbin/ipset -! destroy $ipset_list
-	{
-	echo "USG uptime is less than one hour, loading previous version of blacklist"
-	date
-	echo "Blacklist contents"
-	/sbin/ipset list -s "$real_list"
-	} >> /config/scripts/blacklist-processing.txt
-else
+backupexists="/config/scripts/blacklist-backup.bak"
+
+process_blacklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
 
@@ -37,7 +28,7 @@ else
 		curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list
 	done
 
-	tlcontents=$(sudo /sbin/ipset list temporary-list | grep -A1 "Members:" | sed -n '2p')
+	tlcontents=$(/sbin/ipset list temporary-list | grep -A1 "Members:" | sed -n '2p')
 
 	if [ -z $tlcontents ]
 	then 
@@ -62,4 +53,32 @@ else
 	} >> /config/scripts/blacklist-processing.txt
  
 	/sbin/ipset destroy $ipset_list
+}
+
+if [ $usgupt == "min," ] && [ -e $backupexists ]
+then
+	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
+	/sbin/ipset swap $ipset_list "$real_list"
+	/sbin/ipset -! destroy $ipset_list
+	{
+	echo "USG uptime is less than one hour, and backup list is found" 
+	echo "Loading previous version of blacklist. This will speed up provisioning"
+	date
+	echo "Blacklist contents"
+	/sbin/ipset list -s "$real_list"
+	} >> /config/scripts/blacklist-processing.txt
+elif [ $usgupt == "min," ] && [ ! -e $backupexists ]
+then
+	{
+	echo "USG uptime is less than one hour, but backup list is not found"
+	echo "Proceeding to create new blacklist. This will delay provisioning, but ensure you are protected"
+	date
+	} >> /config/scripts/blacklist-processing.txt
+	process_blacklist
+else
+	{
+	echo "Routine processing of blacklist started"
+	date
+	} >> /config/scripts/blacklist-processing.txt
+	process_blacklist
 fi

From 3bfecd26358886dcd3c571c092a10f3e3d7ea83c Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 30 Nov 2021 14:04:07 -0500
Subject: [PATCH 10/26] Update blacklist.sh

Added additional code to do a list comparison and write the results to the log file.  This will show changes between each run of the script on what IPs were added or removed from the new list created by the script.  This comparison will not run on a reboot of the USG to reduce provisioning time.  Also changed "FireHOL" back to "Dynamic Threat List" to make sure @brontide instructions continue to work with this script.
---
 blacklist.sh | 49 +++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 4c7e9b7..0aee34c 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -1,10 +1,9 @@
 #!/bin/bash
 {
 echo "Blacklist update started"
-date
 }  > /config/scripts/blacklist-processing.txt
 
-real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
+real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}')
 [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
 
 ipset_list="temporary-list"
@@ -28,9 +27,9 @@ process_blacklist () {
 		curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list
 	done
 
-	tlcontents=$(/sbin/ipset list temporary-list | grep -A1 "Members:" | sed -n '2p')
+	tlcontents=$(/sbin/ipset list $ipset_list | grep -A1 "Members:" | sed -n '2p')
 
-	if [ -z $tlcontents ]
+	if [ -z "$tlcontents" ]
 	then 
 		{
 		echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place."
@@ -51,11 +50,48 @@ process_blacklist () {
 	echo "Blacklist contents"
 	/sbin/ipset list -s "$real_list"
 	} >> /config/scripts/blacklist-processing.txt
+	
+	if [ "$usgupt" != "min," ]
+	then
+		{
+		echo "Blacklist changes compared to previous run"
+		} >> /config/scripts/blacklist-processing.txt
+		
+		for Nip in $(/sbin/ipset list "$real_list" | awk '/^[1-9]/ { print }')
+		do
+			if /sbin/ipset test $ipset_list "$Nip"; [ $? != 0 ]
+			then
+				{
+				echo "ADDED $Nip to the list"
+				} >> /config/scripts/blacklist-processing.txt
+			fi
+		done
+	fi
+	
+	if [ "$usgupt" != "min," ]
+	then
+		for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }')
+		do
+			if /sbin/ipset test "$real_list" "$Oip"; [ $? != 0 ]
+			then
+				{
+				echo "REMOVED $Oip from the list"
+				} >> /config/scripts/blacklist-processing.txt
+			fi
+		done
+		
+		{
+		echo "Blacklist comparision complete"
+		echo "Blacklist processing finished"
+		date
+		} >> /config/scripts/blacklist-processing.txt
+		
+	fi
  
 	/sbin/ipset destroy $ipset_list
 }
 
-if [ $usgupt == "min," ] && [ -e $backupexists ]
+if [ "$usgupt" == "min," ] && [ -e $backupexists ]
 then
 	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
 	/sbin/ipset swap $ipset_list "$real_list"
@@ -67,11 +103,12 @@ then
 	echo "Blacklist contents"
 	/sbin/ipset list -s "$real_list"
 	} >> /config/scripts/blacklist-processing.txt
-elif [ $usgupt == "min," ] && [ ! -e $backupexists ]
+elif [ "$usgupt" == "min," ] && [ ! -e $backupexists ]
 then
 	{
 	echo "USG uptime is less than one hour, but backup list is not found"
 	echo "Proceeding to create new blacklist. This will delay provisioning, but ensure you are protected"
+	echo "Blacklist changes will not be compared as this is the first creation of the list"
 	date
 	} >> /config/scripts/blacklist-processing.txt
 	process_blacklist

From 04162242b7360e9e8cbcf7e9fa46cb34c16286a8 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 30 Nov 2021 15:19:44 -0500
Subject: [PATCH 11/26] Update blacklist.sh

Minor verbiage changes to logging messages.
---
 blacklist.sh | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index 0aee34c..b3ac21a 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -36,17 +36,15 @@ process_blacklist () {
 		date
 		} >> /config/scripts/blacklist-processing.txt
 	else 
+		/sbin/ipset save $ipset_list -f /config/scripts/blacklist-backup.bak
+		/sbin/ipset swap $ipset_list "$real_list"
 		{
 		echo "Blacklist is updated and backed up"
 		date
 		} >> /config/scripts/blacklist-processing.txt
-		/sbin/ipset save $ipset_list -f /config/scripts/blacklist-backup.bak
-		/sbin/ipset swap $ipset_list "$real_list"
 	fi
 
 	{
-	echo "Blacklist update finished"
-	date
 	echo "Blacklist contents"
 	/sbin/ipset list -s "$real_list"
 	} >> /config/scripts/blacklist-processing.txt
@@ -81,27 +79,34 @@ process_blacklist () {
 		done
 		
 		{
-		echo "Blacklist comparision complete"
-		echo "Blacklist processing finished"
-		date
+		echo "Blacklist comparison complete"
 		} >> /config/scripts/blacklist-processing.txt
 		
 	fi
+	
+	{
+	echo "Blacklist processing finished"
+	date
+	} >> /config/scripts/blacklist-processing.txt
  
 	/sbin/ipset destroy $ipset_list
 }
 
 if [ "$usgupt" == "min," ] && [ -e $backupexists ]
 then
-	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
-	/sbin/ipset swap $ipset_list "$real_list"
-	/sbin/ipset -! destroy $ipset_list
 	{
 	echo "USG uptime is less than one hour, and backup list is found" 
 	echo "Loading previous version of blacklist. This will speed up provisioning"
 	date
+	} >> /config/scripts/blacklist-processing.txt
+	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
+	/sbin/ipset swap $ipset_list "$real_list"
+	/sbin/ipset -! destroy $ipset_list
+	{
 	echo "Blacklist contents"
 	/sbin/ipset list -s "$real_list"
+	echo "Restoration of blacklist backup complete"
+	date
 	} >> /config/scripts/blacklist-processing.txt
 elif [ "$usgupt" == "min," ] && [ ! -e $backupexists ]
 then

From f15a8d0becbd5952f9c2219e071d9cd9307cead0 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 30 Nov 2021 22:05:31 -0500
Subject: [PATCH 12/26] Update blacklist.sh

More logging to track if no changes were made and also logging total changes made in comparison of previous list to new list during script execution.
---
 blacklist.sh | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index b3ac21a..071b0e9 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -57,11 +57,16 @@ process_blacklist () {
 		
 		for Nip in $(/sbin/ipset list "$real_list" | awk '/^[1-9]/ { print }')
 		do
-			if /sbin/ipset test $ipset_list "$Nip"; [ $? != 0 ]
+			NTotal=$((NTotal+1));
+			
+			if ! /sbin/ipset test $ipset_list "$Nip"
 			then
+				NChanges=$((NChanges+1));
 				{
 				echo "ADDED $Nip to the list"
 				} >> /config/scripts/blacklist-processing.txt
+			else
+				NoneAdded=$((NoneAdded+1));
 			fi
 		done
 	fi
@@ -70,14 +75,31 @@ process_blacklist () {
 	then
 		for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }')
 		do
-			if /sbin/ipset test "$real_list" "$Oip"; [ $? != 0 ]
+			OTotal=$((OTotal+1));
+			
+			if ! /sbin/ipset test "$real_list" "$Oip"
 			then
+				OChanges=$((OChanges+1));
 				{
 				echo "REMOVED $Oip from the list"
 				} >> /config/scripts/blacklist-processing.txt
+			else
+				NoneRemoved=$((NoneRemoved+1));
 			fi
 		done
 		
+		if [ $((NTotal + OTotal)) == $((NoneAdded + NoneRemoved)) ]
+		then
+			{
+			echo "No changes"
+			} >> /config/scripts/blacklist-processing.txt
+		else
+			TChanges=$((NChanges + OChanges));
+			{
+			echo "$TChanges total changes"
+			} >> /config/scripts/blacklist-processing.txt
+		fi
+		
 		{
 		echo "Blacklist comparison complete"
 		} >> /config/scripts/blacklist-processing.txt

From d73796ef431187b8af5d1d5222e06daaa1ed92dc Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 30 Nov 2021 22:23:11 -0500
Subject: [PATCH 13/26] Update README.md

Updated readme to include step from @bloqhed to create symbolic link so script will run on USG reboot as well as scheduled interval.
---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 71f630b..ec348b0 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,17 @@ correctly.
    sudo curl -o /config/scripts/blacklist.sh https://raw.githubusercontent.com/brontide/usg-blacklist/master/blacklist.sh
    sudo chmod 755 /config/scripts/blacklist.sh
    ```
+1. Create symbolic link so script runs on controller reboot in addition to scheduled interval
+
+   ```
+   sudo ln -s /config/scripts/blacklist.sh /config/scripts/post-config.d/blacklist.sh
+   ```
 1. Create/update config.gateway.json on your controller to run this script periodically.
+1. Reboot USG to force immediate script execution or SSH into USG and run below command to force immediate script execution
+
+   ```
+   sudo /config/scripts/blacklist.sh
+   ```
 
 # View counters.
 

From 0852962628ee2315ee7d812373fb1b00d0af3b75 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Wed, 1 Dec 2021 15:27:54 -0500
Subject: [PATCH 14/26] Update blacklist.sh

Another minor logging change to include total additions and removals from list.  Also changing list name to "FireHOL" instead of "Dynamic Threat List" as FireHOL is where all my lists come from, and @brontide has archived his repo so the merge will not happen.
---
 blacklist.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/blacklist.sh b/blacklist.sh
index 071b0e9..edf6785 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -3,7 +3,7 @@
 echo "Blacklist update started"
 }  > /config/scripts/blacklist-processing.txt
 
-real_list=$(grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}')
+real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
 [[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
 
 ipset_list="temporary-list"
@@ -96,6 +96,8 @@ process_blacklist () {
 		else
 			TChanges=$((NChanges + OChanges));
 			{
+			echo "$NChanges additions"
+			echo "$OChanges removals"
 			echo "$TChanges total changes"
 			} >> /config/scripts/blacklist-processing.txt
 		fi

From 5c96f35a1ac434c18d86c51c0339e3677d61cbf1 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Wed, 1 Dec 2021 15:54:35 -0500
Subject: [PATCH 15/26] Update README.md

Updated "Dynamic Threat List" to "FireHOL" to be consistent with my script.  Added in parentheses the firewall rule names shown in the 6.2.x and above controller firewall area for clarification.
---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index ec348b0..f3404c1 100644
--- a/README.md
+++ b/README.md
@@ -4,13 +4,13 @@ Having migrated from EdgeRouter to USG I wanted to bring over one script that ke
 from several reputable sources.  The script itself is quite simple but requires setup within the conttroller to work
 correctly.
 
-1. Setup a firewall IPv4 group called "Dynamic Threat List".  The name is important because it's used by the script.
-1. Setup firewall WAN_LOCAL, WAN_OUT rules to drop traffic from/to this group.
+1. Setup a firewall IPv4 group called "FireHOL".  The name is important because it's used by the script.
+1. Setup firewall WAN_IN (Internet In), WAN_LOCAL (Internet Local), WAN_OUT (Internet Out) rules to drop traffic from/to this group.
 1. Install the script into /config/scripts on the USG.  Please check the files before running.
    
    ```
-   sudo curl -o /config/scripts/blacklist.sh https://raw.githubusercontent.com/brontide/usg-blacklist/master/blacklist.sh
-   sudo chmod 755 /config/scripts/blacklist.sh
+   sudo curl -o /config/scripts/blacklist.sh https://raw.githubusercontent.com/FastEddy1114/usg-blacklist/master/blacklist.sh
+   sudo chmod +x /config/scripts/blacklist.sh
    ```
 1. Create symbolic link so script runs on controller reboot in addition to scheduled interval
 

From bdde11ef3679f7be34f5f31a88e0c3eaca4c848d Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Fri, 3 Dec 2021 13:15:52 -0500
Subject: [PATCH 16/26] Update blacklist.sh

Minor code efficiency update to consolidate if statements in list comparison code.
---
 blacklist.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/blacklist.sh b/blacklist.sh
index edf6785..c67e596 100644
--- a/blacklist.sh
+++ b/blacklist.sh
@@ -69,10 +69,7 @@ process_blacklist () {
 				NoneAdded=$((NoneAdded+1));
 			fi
 		done
-	fi
-	
-	if [ "$usgupt" != "min," ]
-	then
+		
 		for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }')
 		do
 			OTotal=$((OTotal+1));
@@ -105,9 +102,8 @@ process_blacklist () {
 		{
 		echo "Blacklist comparison complete"
 		} >> /config/scripts/blacklist-processing.txt
-		
 	fi
-	
+		
 	{
 	echo "Blacklist processing finished"
 	date

From 36effccca16b6a6e8e708f3bc61a65c2237a77a6 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Fri, 3 Dec 2021 13:18:08 -0500
Subject: [PATCH 17/26] Update README.md

Clarification on config.gateway.json time intervals.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f3404c1..565bb02 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ correctly.
    ```
    sudo ln -s /config/scripts/blacklist.sh /config/scripts/post-config.d/blacklist.sh
    ```
-1. Create/update config.gateway.json on your controller to run this script periodically.
+1. Create/update config.gateway.json on your controller to run this script periodically.  Time interval specified in config.gateway.json file is always based on 00:00:00 (midnight) being the starting point.
 1. Reboot USG to force immediate script execution or SSH into USG and run below command to force immediate script execution
 
    ```

From a7fb2987fca878cb53ba0731614e288e3abfec36 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 6 Dec 2022 10:17:00 -0500
Subject: [PATCH 18/26] Rename file

Renaming file from blacklist.sh to blocklist.sh.  This update also includes all references inside the file to blacklist are now updated to blocklist.  Additional logging directed to console by using echo so if script is run manually you have a better idea of what step it is on and what it is doing.
---
 blocklist.sh | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 161 insertions(+)
 create mode 100644 blocklist.sh

diff --git a/blocklist.sh b/blocklist.sh
new file mode 100644
index 0000000..bffc768
--- /dev/null
+++ b/blocklist.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+{
+echo "Blocklist update started"
+}  > /config/scripts/blocklist-processing.txt
+
+real_list=$(grep -B2 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
+[[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Will update FireHOL list ID $real_list"
+
+ipset_list="temporary-list"
+
+usgupt=$(uptime | awk '{print $4}')
+
+backupexists="/config/scripts/blocklist-backup.bak"
+
+process_blocklist () {
+	/sbin/ipset -! destroy $ipset_list
+	/sbin/ipset create $ipset_list hash:net
+
+	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
+	do
+		echo "Fetching and processing $url"
+		{
+		echo "Processing blocklist"
+		date
+		echo $url
+		} >> /config/scripts/blocklist-processing.txt
+		curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list
+	done
+
+	tlcontents=$(/sbin/ipset list $ipset_list | grep -A1 "Members:" | sed -n '2p')
+
+	if [ -z "$tlcontents" ]
+	then 
+		echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place."
+		{
+		echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place."
+		date
+		} >> /config/scripts/blocklist-processing.txt
+	else 
+		/sbin/ipset save $ipset_list -f /config/scripts/blocklist-backup.bak
+		/sbin/ipset swap $ipset_list "$real_list"
+		echo "Blocklist is updated and backed up"
+		{
+		echo "Blocklist is updated and backed up"
+		date
+		} >> /config/scripts/blocklist-processing.txt
+	fi
+
+	{
+	echo "Blocklist contents"
+	/sbin/ipset list -s "$real_list"
+	} >> /config/scripts/blocklist-processing.txt
+	
+	if [ "$usgupt" != "min," ]
+	then
+		echo "Processing changes compared to previous run"
+		echo "To see the changes check the log located at /config/scripts/blocklist-processing.txt"
+		{
+		echo "Blocklist changes compared to previous run"
+		} >> /config/scripts/blocklist-processing.txt
+		
+		for Nip in $(/sbin/ipset list "$real_list" | awk '/^[1-9]/ { print }')
+		do
+			NTotal=$((NTotal+1));
+			
+			if ! /sbin/ipset test $ipset_list "$Nip"
+			then
+				NChanges=$((NChanges+1));
+				{
+				echo "ADDED $Nip to the list"
+				} >> /config/scripts/blocklist-processing.txt
+			else
+				NoneAdded=$((NoneAdded+1));
+			fi
+		done
+		
+		for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }')
+		do
+			OTotal=$((OTotal+1));
+			
+			if ! /sbin/ipset test "$real_list" "$Oip"
+			then
+				OChanges=$((OChanges+1));
+				{
+				echo "REMOVED $Oip from the list"
+				} >> /config/scripts/blocklist-processing.txt
+			else
+				NoneRemoved=$((NoneRemoved+1));
+			fi
+		done
+		
+		if [ $((NTotal + OTotal)) == $((NoneAdded + NoneRemoved)) ]
+		then
+			{
+			echo "No changes"
+			} >> /config/scripts/blocklist-processing.txt
+		else
+			TChanges=$((NChanges + OChanges));
+			{
+			echo "$NChanges additions"
+			echo "$OChanges removals"
+			echo "$TChanges total changes"
+			} >> /config/scripts/blocklist-processing.txt
+		fi
+		
+		echo "Blocklist comparison complete"
+		{
+		echo "Blocklist comparison complete"
+		} >> /config/scripts/blocklist-processing.txt
+	fi
+		
+	{
+	echo "Blocklist processing finished"
+	date
+	} >> /config/scripts/blocklist-processing.txt
+ 
+	/sbin/ipset destroy $ipset_list
+	echo "Blocklist processing finished"
+}
+
+if [ "$usgupt" == "min," ] && [ -e $backupexists ]
+then
+	echo "USG uptime is less than one hour, and backup list is found" 
+	echo "Loading previous version of blocklist. This will speed up provisioning"
+	{
+	echo "USG uptime is less than one hour, and backup list is found" 
+	echo "Loading previous version of blocklist. This will speed up provisioning"
+	date
+	} >> /config/scripts/blocklist-processing.txt
+	/sbin/ipset restore -f /config/scripts/blocklist-backup.bak
+	/sbin/ipset swap $ipset_list "$real_list"
+	/sbin/ipset -! destroy $ipset_list
+	{
+	echo "Blocklist contents"
+	/sbin/ipset list -s "$real_list"
+	echo "Restoration of blocklist backup complete"
+	date
+	} >> /config/scripts/blocklist-processing.txt
+	echo "Restoration of blocklist backup complete"
+elif [ "$usgupt" == "min," ] && [ ! -e $backupexists ]
+then
+	echo "USG uptime is less than one hour, but backup list is not found"
+	echo "Proceeding to create new blocklist. This will delay provisioning, but ensure you are protected"
+	echo "Blocklist changes will not be compared as this is the first creation of the list"
+	{
+	echo "USG uptime is less than one hour, but backup list is not found"
+	echo "Proceeding to create new blocklist. This will delay provisioning, but ensure you are protected"
+	echo "Blocklist changes will not be compared as this is the first creation of the list"
+	date
+	} >> /config/scripts/blocklist-processing.txt
+	process_blocklist
+	echo "First time creation of blocklist complete"
+else
+	echo "Routine processing of blocklist started"
+	{
+	echo "Routine processing of blocklist started"
+	date
+	} >> /config/scripts/blocklist-processing.txt
+	process_blocklist
+	echo "Routine processing of blocklist complete"
+fi

From bcaeb2528577b2147712f59c14bd46a7dd472ff9 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 6 Dec 2022 10:18:59 -0500
Subject: [PATCH 19/26] Delete blacklist.sh

Remove file as it is now titled blocklist.sh
---
 blacklist.sh | 146 ---------------------------------------------------
 1 file changed, 146 deletions(-)
 delete mode 100644 blacklist.sh

diff --git a/blacklist.sh b/blacklist.sh
deleted file mode 100644
index c67e596..0000000
--- a/blacklist.sh
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/bin/bash
-{
-echo "Blacklist update started"
-}  > /config/scripts/blacklist-processing.txt
-
-real_list=$(grep -B1 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}')
-[[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Updating $real_list"
-
-ipset_list="temporary-list"
-
-usgupt=$(uptime | awk '{print $4}')
-
-backupexists="/config/scripts/blacklist-backup.bak"
-
-process_blacklist () {
-	/sbin/ipset -! destroy $ipset_list
-	/sbin/ipset create $ipset_list hash:net
-
-	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
-	do
-		echo "Fetching and processing $url"
-		{
-		echo "Processing blacklist"
-		date
-		echo $url
-		} >> /config/scripts/blacklist-processing.txt
-		curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list
-	done
-
-	tlcontents=$(/sbin/ipset list $ipset_list | grep -A1 "Members:" | sed -n '2p')
-
-	if [ -z "$tlcontents" ]
-	then 
-		{
-		echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place."
-		date
-		} >> /config/scripts/blacklist-processing.txt
-	else 
-		/sbin/ipset save $ipset_list -f /config/scripts/blacklist-backup.bak
-		/sbin/ipset swap $ipset_list "$real_list"
-		{
-		echo "Blacklist is updated and backed up"
-		date
-		} >> /config/scripts/blacklist-processing.txt
-	fi
-
-	{
-	echo "Blacklist contents"
-	/sbin/ipset list -s "$real_list"
-	} >> /config/scripts/blacklist-processing.txt
-	
-	if [ "$usgupt" != "min," ]
-	then
-		{
-		echo "Blacklist changes compared to previous run"
-		} >> /config/scripts/blacklist-processing.txt
-		
-		for Nip in $(/sbin/ipset list "$real_list" | awk '/^[1-9]/ { print }')
-		do
-			NTotal=$((NTotal+1));
-			
-			if ! /sbin/ipset test $ipset_list "$Nip"
-			then
-				NChanges=$((NChanges+1));
-				{
-				echo "ADDED $Nip to the list"
-				} >> /config/scripts/blacklist-processing.txt
-			else
-				NoneAdded=$((NoneAdded+1));
-			fi
-		done
-		
-		for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }')
-		do
-			OTotal=$((OTotal+1));
-			
-			if ! /sbin/ipset test "$real_list" "$Oip"
-			then
-				OChanges=$((OChanges+1));
-				{
-				echo "REMOVED $Oip from the list"
-				} >> /config/scripts/blacklist-processing.txt
-			else
-				NoneRemoved=$((NoneRemoved+1));
-			fi
-		done
-		
-		if [ $((NTotal + OTotal)) == $((NoneAdded + NoneRemoved)) ]
-		then
-			{
-			echo "No changes"
-			} >> /config/scripts/blacklist-processing.txt
-		else
-			TChanges=$((NChanges + OChanges));
-			{
-			echo "$NChanges additions"
-			echo "$OChanges removals"
-			echo "$TChanges total changes"
-			} >> /config/scripts/blacklist-processing.txt
-		fi
-		
-		{
-		echo "Blacklist comparison complete"
-		} >> /config/scripts/blacklist-processing.txt
-	fi
-		
-	{
-	echo "Blacklist processing finished"
-	date
-	} >> /config/scripts/blacklist-processing.txt
- 
-	/sbin/ipset destroy $ipset_list
-}
-
-if [ "$usgupt" == "min," ] && [ -e $backupexists ]
-then
-	{
-	echo "USG uptime is less than one hour, and backup list is found" 
-	echo "Loading previous version of blacklist. This will speed up provisioning"
-	date
-	} >> /config/scripts/blacklist-processing.txt
-	/sbin/ipset restore -f /config/scripts/blacklist-backup.bak
-	/sbin/ipset swap $ipset_list "$real_list"
-	/sbin/ipset -! destroy $ipset_list
-	{
-	echo "Blacklist contents"
-	/sbin/ipset list -s "$real_list"
-	echo "Restoration of blacklist backup complete"
-	date
-	} >> /config/scripts/blacklist-processing.txt
-elif [ "$usgupt" == "min," ] && [ ! -e $backupexists ]
-then
-	{
-	echo "USG uptime is less than one hour, but backup list is not found"
-	echo "Proceeding to create new blacklist. This will delay provisioning, but ensure you are protected"
-	echo "Blacklist changes will not be compared as this is the first creation of the list"
-	date
-	} >> /config/scripts/blacklist-processing.txt
-	process_blacklist
-else
-	{
-	echo "Routine processing of blacklist started"
-	date
-	} >> /config/scripts/blacklist-processing.txt
-	process_blacklist
-fi

From 29618a44b7af1ba617808e752f3a33fd75b6318e Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 6 Dec 2022 10:23:21 -0500
Subject: [PATCH 20/26] Update README.md

update references from blacklist to blocklist as the filename has changed.
---
 README.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 565bb02..92bf28c 100644
--- a/README.md
+++ b/README.md
@@ -1,27 +1,27 @@
-# Dynamic IP/CIDR Blacklisting for the Unifi USG router
+# Dynamic IP/CIDR Blocklist for the Unifi USG router
 
-Having migrated from EdgeRouter to USG I wanted to bring over one script that kept a daily dynamic blacklist updated
+Having a USG I wanted to create a script that kept a daily dynamic blocklist updated
 from several reputable sources.  The script itself is quite simple but requires setup within the conttroller to work
 correctly.
 
-1. Setup a firewall IPv4 group called "FireHOL".  The name is important because it's used by the script.
+1. Setup a firewall IPv4 group called "FireHOL" with one place holder IPv4 address or subnet such as "192.168.0.0/16" as this address will always be in the list anyway as it is a bogon.  The name is important because it's used by the script.
 1. Setup firewall WAN_IN (Internet In), WAN_LOCAL (Internet Local), WAN_OUT (Internet Out) rules to drop traffic from/to this group.
 1. Install the script into /config/scripts on the USG.  Please check the files before running.
    
    ```
-   sudo curl -o /config/scripts/blacklist.sh https://raw.githubusercontent.com/FastEddy1114/usg-blacklist/master/blacklist.sh
-   sudo chmod +x /config/scripts/blacklist.sh
+   sudo curl -o /config/scripts/blocklist.sh https://raw.githubusercontent.com/FastEddy1114/usg-blacklist/master/blocklist.sh
+   sudo chmod +x /config/scripts/blocklist.sh
    ```
-1. Create symbolic link so script runs on controller reboot in addition to scheduled interval
+1. Create symbolic link so script runs on USG reboot in addition to scheduled interval
 
    ```
-   sudo ln -s /config/scripts/blacklist.sh /config/scripts/post-config.d/blacklist.sh
+   sudo ln -s /config/scripts/blocklist.sh /config/scripts/post-config.d/blocklist.sh
    ```
 1. Create/update config.gateway.json on your controller to run this script periodically.  Time interval specified in config.gateway.json file is always based on 00:00:00 (midnight) being the starting point.
 1. Reboot USG to force immediate script execution or SSH into USG and run below command to force immediate script execution
 
    ```
-   sudo /config/scripts/blacklist.sh
+   sudo /config/scripts/blocklist.sh
    ```
 
 # View counters.

From ec1558bc1660c0a1d24a3ad7e78d4b33ad763799 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 6 Dec 2022 10:24:21 -0500
Subject: [PATCH 21/26] Update config.gateway.json

update blacklist to blocklist and adjust interval to 12h from 24h
---
 config.gateway.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config.gateway.json b/config.gateway.json
index 3f02356..1f0d805 100644
--- a/config.gateway.json
+++ b/config.gateway.json
@@ -2,11 +2,11 @@
     "system": {
         "task-scheduler": {
             "task": {
-                "blacklist": {
+                "blocklist": {
                     "executable": {
-                        "path": "/config/scripts/blacklist.sh"
+                        "path": "/config/scripts/blocklist.sh"
                     },
-                    "interval": "24h"
+                    "interval": "12h"
                 }
             }
         }

From aae1501063f215b510aa3341de51de4a053603c0 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 6 Dec 2022 11:27:41 -0500
Subject: [PATCH 22/26] Update blocklist.sh

Added if [ -e $backupexists ]; then backupexists="TRUE"; else backupexists="FALSE"; fi.  Adjusted if [ "$usgupt" != "min," ] && [ -e $backupexists ] test to if [ "$usgupt" != "min," ] && [ "$backupexists" == "TRUE" ].  Running the test the other way the [ -e ] portion was always defaulting to true even if the file did not exist which was causing the change comparison to run when it should not.  The change comparison should only run if a previous backup file exists indicating a prior successful run of the script, and the USG uptime is over 59 minutes.
---
 blocklist.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/blocklist.sh b/blocklist.sh
index bffc768..2e6f3c5 100644
--- a/blocklist.sh
+++ b/blocklist.sh
@@ -12,6 +12,13 @@ usgupt=$(uptime | awk '{print $4}')
 
 backupexists="/config/scripts/blocklist-backup.bak"
 
+if [ -e $backupexists ]
+then
+	backupexists="TRUE"
+else
+	backupexists="FALSE"
+fi
+
 process_blocklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
@@ -51,7 +58,7 @@ process_blocklist () {
 	/sbin/ipset list -s "$real_list"
 	} >> /config/scripts/blocklist-processing.txt
 	
-	if [ "$usgupt" != "min," ]
+	if [ "$usgupt" != "min," ] && [ "$backupexists" == "TRUE" ]
 	then
 		echo "Processing changes compared to previous run"
 		echo "To see the changes check the log located at /config/scripts/blocklist-processing.txt"
@@ -118,7 +125,7 @@ process_blocklist () {
 	echo "Blocklist processing finished"
 }
 
-if [ "$usgupt" == "min," ] && [ -e $backupexists ]
+if [ "$usgupt" == "min," ] && [ "$backupexists" = "TRUE" ]
 then
 	echo "USG uptime is less than one hour, and backup list is found" 
 	echo "Loading previous version of blocklist. This will speed up provisioning"
@@ -137,7 +144,7 @@ then
 	date
 	} >> /config/scripts/blocklist-processing.txt
 	echo "Restoration of blocklist backup complete"
-elif [ "$usgupt" == "min," ] && [ ! -e $backupexists ]
+elif [ "$usgupt" == "min," ] && [ "$backupexists" == "FALSE" ]
 then
 	echo "USG uptime is less than one hour, but backup list is not found"
 	echo "Proceeding to create new blocklist. This will delay provisioning, but ensure you are protected"

From 1ce3a6fc80bafed18def8d6c82593c070b432035 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Fri, 31 Mar 2023 12:18:32 -0400
Subject: [PATCH 23/26] Update blocklist.sh

Disable previous to current list compare.  Not really worth the extra processing time and CPU load.
---
 blocklist.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/blocklist.sh b/blocklist.sh
index 2e6f3c5..803de99 100644
--- a/blocklist.sh
+++ b/blocklist.sh
@@ -23,7 +23,7 @@ process_blocklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
 
-	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset
+	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_1d.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/stopforumspam_toxic.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/myip.ipset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/tor_exits.ipset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/iblocklist_onion_router.netset
 	do
 		echo "Fetching and processing $url"
 		{
@@ -58,6 +58,7 @@ process_blocklist () {
 	/sbin/ipset list -s "$real_list"
 	} >> /config/scripts/blocklist-processing.txt
 	
+<<Disabled
 	if [ "$usgupt" != "min," ] && [ "$backupexists" == "TRUE" ]
 	then
 		echo "Processing changes compared to previous run"
@@ -115,6 +116,7 @@ process_blocklist () {
 		echo "Blocklist comparison complete"
 		} >> /config/scripts/blocklist-processing.txt
 	fi
+Disabled
 		
 	{
 	echo "Blocklist processing finished"

From b8f84bb734a60d187eb1897ad758a4a1bf744a4c Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Tue, 5 Sep 2023 19:24:20 -0400
Subject: [PATCH 24/26] Update blocklist.sh

Updated URLs for iplists as FireHOL github repository is no longer being updated.  URLs now pull directly from firehol.org site which is still getting updated.
---
 blocklist.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blocklist.sh b/blocklist.sh
index 803de99..882568f 100644
--- a/blocklist.sh
+++ b/blocklist.sh
@@ -23,7 +23,7 @@ process_blocklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
 
-	for url in https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_webclient.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_1d.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/stopforumspam_toxic.netset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/myip.ipset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/tor_exits.ipset https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/iblocklist_onion_router.netset
+	for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/firehol_level3.netset https://iplists.firehol.org/files/firehol_webclient.netset https://iplists.firehol.org/files/firehol_abusers_1d.netset https://iplists.firehol.org/files/myip.ipset https://iplists.firehol.org/files/tor_exits.ipset https://iplists.firehol.org/files/iblocklist_onion_router.netset
 	do
 		echo "Fetching and processing $url"
 		{

From 3b749fd85905659c40341f4a5b5a7c30b94ad22a Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Thu, 28 Dec 2023 10:27:25 -0500
Subject: [PATCH 25/26] Update blocklist.sh

Removed FireHOL Level 3, too many false positives for me in the list.
---
 blocklist.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blocklist.sh b/blocklist.sh
index 882568f..1d700c3 100644
--- a/blocklist.sh
+++ b/blocklist.sh
@@ -23,7 +23,7 @@ process_blocklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
 
-	for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/firehol_level3.netset https://iplists.firehol.org/files/firehol_webclient.netset https://iplists.firehol.org/files/firehol_abusers_1d.netset https://iplists.firehol.org/files/myip.ipset https://iplists.firehol.org/files/tor_exits.ipset https://iplists.firehol.org/files/iblocklist_onion_router.netset
+	for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/firehol_webclient.netset https://iplists.firehol.org/files/firehol_abusers_1d.netset https://iplists.firehol.org/files/myip.ipset https://iplists.firehol.org/files/tor_exits.ipset https://iplists.firehol.org/files/iblocklist_onion_router.netset
 	do
 		echo "Fetching and processing $url"
 		{

From 7759a9bb29997b155c15bc23caf88a99a73b3869 Mon Sep 17 00:00:00 2001
From: FastEddy1114 <47258351+FastEddy1114@users.noreply.github.com>
Date: Mon, 1 Apr 2024 20:31:55 -0400
Subject: [PATCH 26/26] Update blocklist.sh

Updated blocklist URLs
---
 blocklist.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blocklist.sh b/blocklist.sh
index 1d700c3..fd78699 100644
--- a/blocklist.sh
+++ b/blocklist.sh
@@ -23,7 +23,7 @@ process_blocklist () {
 	/sbin/ipset -! destroy $ipset_list
 	/sbin/ipset create $ipset_list hash:net
 
-	for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/firehol_webclient.netset https://iplists.firehol.org/files/firehol_abusers_1d.netset https://iplists.firehol.org/files/myip.ipset https://iplists.firehol.org/files/tor_exits.ipset https://iplists.firehol.org/files/iblocklist_onion_router.netset
+	for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/iblocklist_onion_router.netset https://iplists.firehol.org/files/ciarmy.ipset https://iplists.firehol.org/files/tor_exits.ipset
 	do
 		echo "Fetching and processing $url"
 		{