diff --git a/README.md b/README.md index 71f630b..92bf28c 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,28 @@ -# Dynamic IP/CIDR Blacklisting for the Unifi USG router +# Dynamic IP/CIDR Blocklist for the Unifi USG router -Having migrated from EdgeRouter to USG I wanted to bring over one script that kept a daily dynamic blacklist updated +Having a USG I wanted to create a script that kept a daily dynamic blocklist updated from several reputable sources. The script itself is quite simple but requires setup within the conttroller to work correctly. -1. Setup a firewall IPv4 group called "Dynamic Threat List". The name is important because it's used by the script. -1. Setup firewall WAN_LOCAL, WAN_OUT rules to drop traffic from/to this group. +1. Setup a firewall IPv4 group called "FireHOL" with one place holder IPv4 address or subnet such as "192.168.0.0/16" as this address will always be in the list anyway as it is a bogon. The name is important because it's used by the script. +1. Setup firewall WAN_IN (Internet In), WAN_LOCAL (Internet Local), WAN_OUT (Internet Out) rules to drop traffic from/to this group. 1. Install the script into /config/scripts on the USG. Please check the files before running. ``` - sudo curl -o /config/scripts/blacklist.sh https://raw.githubusercontent.com/brontide/usg-blacklist/master/blacklist.sh - sudo chmod 755 /config/scripts/blacklist.sh + sudo curl -o /config/scripts/blocklist.sh https://raw.githubusercontent.com/FastEddy1114/usg-blacklist/master/blocklist.sh + sudo chmod +x /config/scripts/blocklist.sh + ``` +1. Create symbolic link so script runs on USG reboot in addition to scheduled interval + + ``` + sudo ln -s /config/scripts/blocklist.sh /config/scripts/post-config.d/blocklist.sh + ``` +1. Create/update config.gateway.json on your controller to run this script periodically. Time interval specified in config.gateway.json file is always based on 00:00:00 (midnight) being the starting point. +1. Reboot USG to force immediate script execution or SSH into USG and run below command to force immediate script execution + + ``` + sudo /config/scripts/blocklist.sh ``` -1. Create/update config.gateway.json on your controller to run this script periodically. # View counters. diff --git a/blacklist.sh b/blacklist.sh deleted file mode 100644 index 7b87bdb..0000000 --- a/blacklist.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -real_list=`grep -B1 "Dynamic Threat List" /config/config.boot | head -n 1 | awk '{print $2}'` -[[ -z "$real_list" ]] && { echo "aborting"; exit -1; } || echo "Updating $real_list" - -ipset_list='temporary-list' - -sudo /sbin/ipset -! destroy $ipset_list -sudo /sbin/ipset create $ipset_list hash:net - -for url in 'https://www.spamhaus.org/drop/edrop.txt' 'http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt' 'https://check.torproject.org/cgi-bin/TorBulkExitList.py?ip=1.1.1.1'; -do - echo "Fetching and processing $url" - curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 sudo ipset -q add $ipset_list -done - -sudo /sbin/ipset swap $ipset_list $real_list -sudo /sbin/ipset destroy $ipset_list diff --git a/blocklist.sh b/blocklist.sh new file mode 100644 index 0000000..fd78699 --- /dev/null +++ b/blocklist.sh @@ -0,0 +1,170 @@ +#!/bin/bash +{ +echo "Blocklist update started" +} > /config/scripts/blocklist-processing.txt + +real_list=$(grep -B2 "FireHOL" /config/config.boot | head -n 1 | awk '{print $2}') +[[ -z "$real_list" ]] && { echo "aborting"; exit 1; } || echo "Will update FireHOL list ID $real_list" + +ipset_list="temporary-list" + +usgupt=$(uptime | awk '{print $4}') + +backupexists="/config/scripts/blocklist-backup.bak" + +if [ -e $backupexists ] +then + backupexists="TRUE" +else + backupexists="FALSE" +fi + +process_blocklist () { + /sbin/ipset -! destroy $ipset_list + /sbin/ipset create $ipset_list hash:net + + for url in https://iplists.firehol.org/files/firehol_level1.netset https://iplists.firehol.org/files/firehol_level2.netset https://iplists.firehol.org/files/iblocklist_onion_router.netset https://iplists.firehol.org/files/ciarmy.ipset https://iplists.firehol.org/files/tor_exits.ipset + do + echo "Fetching and processing $url" + { + echo "Processing blocklist" + date + echo $url + } >> /config/scripts/blocklist-processing.txt + curl "$url" | awk '/^[1-9]/ { print $1 }' | xargs -n1 /sbin/ipset -! add $ipset_list + done + + tlcontents=$(/sbin/ipset list $ipset_list | grep -A1 "Members:" | sed -n '2p') + + if [ -z "$tlcontents" ] + then + echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place." + { + echo "Temporary list is empty, not backing up or swapping list. Leaving current list and contents in place." + date + } >> /config/scripts/blocklist-processing.txt + else + /sbin/ipset save $ipset_list -f /config/scripts/blocklist-backup.bak + /sbin/ipset swap $ipset_list "$real_list" + echo "Blocklist is updated and backed up" + { + echo "Blocklist is updated and backed up" + date + } >> /config/scripts/blocklist-processing.txt + fi + + { + echo "Blocklist contents" + /sbin/ipset list -s "$real_list" + } >> /config/scripts/blocklist-processing.txt + +<> /config/scripts/blocklist-processing.txt + + for Nip in $(/sbin/ipset list "$real_list" | awk '/^[1-9]/ { print }') + do + NTotal=$((NTotal+1)); + + if ! /sbin/ipset test $ipset_list "$Nip" + then + NChanges=$((NChanges+1)); + { + echo "ADDED $Nip to the list" + } >> /config/scripts/blocklist-processing.txt + else + NoneAdded=$((NoneAdded+1)); + fi + done + + for Oip in $(/sbin/ipset list $ipset_list | awk '/^[1-9]/ { print }') + do + OTotal=$((OTotal+1)); + + if ! /sbin/ipset test "$real_list" "$Oip" + then + OChanges=$((OChanges+1)); + { + echo "REMOVED $Oip from the list" + } >> /config/scripts/blocklist-processing.txt + else + NoneRemoved=$((NoneRemoved+1)); + fi + done + + if [ $((NTotal + OTotal)) == $((NoneAdded + NoneRemoved)) ] + then + { + echo "No changes" + } >> /config/scripts/blocklist-processing.txt + else + TChanges=$((NChanges + OChanges)); + { + echo "$NChanges additions" + echo "$OChanges removals" + echo "$TChanges total changes" + } >> /config/scripts/blocklist-processing.txt + fi + + echo "Blocklist comparison complete" + { + echo "Blocklist comparison complete" + } >> /config/scripts/blocklist-processing.txt + fi +Disabled + + { + echo "Blocklist processing finished" + date + } >> /config/scripts/blocklist-processing.txt + + /sbin/ipset destroy $ipset_list + echo "Blocklist processing finished" +} + +if [ "$usgupt" == "min," ] && [ "$backupexists" = "TRUE" ] +then + echo "USG uptime is less than one hour, and backup list is found" + echo "Loading previous version of blocklist. This will speed up provisioning" + { + echo "USG uptime is less than one hour, and backup list is found" + echo "Loading previous version of blocklist. This will speed up provisioning" + date + } >> /config/scripts/blocklist-processing.txt + /sbin/ipset restore -f /config/scripts/blocklist-backup.bak + /sbin/ipset swap $ipset_list "$real_list" + /sbin/ipset -! destroy $ipset_list + { + echo "Blocklist contents" + /sbin/ipset list -s "$real_list" + echo "Restoration of blocklist backup complete" + date + } >> /config/scripts/blocklist-processing.txt + echo "Restoration of blocklist backup complete" +elif [ "$usgupt" == "min," ] && [ "$backupexists" == "FALSE" ] +then + echo "USG uptime is less than one hour, but backup list is not found" + echo "Proceeding to create new blocklist. This will delay provisioning, but ensure you are protected" + echo "Blocklist changes will not be compared as this is the first creation of the list" + { + echo "USG uptime is less than one hour, but backup list is not found" + echo "Proceeding to create new blocklist. This will delay provisioning, but ensure you are protected" + echo "Blocklist changes will not be compared as this is the first creation of the list" + date + } >> /config/scripts/blocklist-processing.txt + process_blocklist + echo "First time creation of blocklist complete" +else + echo "Routine processing of blocklist started" + { + echo "Routine processing of blocklist started" + date + } >> /config/scripts/blocklist-processing.txt + process_blocklist + echo "Routine processing of blocklist complete" +fi diff --git a/config.gateway.json b/config.gateway.json index 3f02356..1f0d805 100644 --- a/config.gateway.json +++ b/config.gateway.json @@ -2,11 +2,11 @@ "system": { "task-scheduler": { "task": { - "blacklist": { + "blocklist": { "executable": { - "path": "/config/scripts/blacklist.sh" + "path": "/config/scripts/blocklist.sh" }, - "interval": "24h" + "interval": "12h" } } }