Merge pull request clong#588 from clong/monitor_eth0_eth1

Monitor both eth0 and eth1 with Suricata+Zeek
gregclermont · Jan 12, 2021 · 95717a7 · 95717a7
2 parents 6237582 + c52627e
commit 95717a7
Show file tree

Hide file tree

Showing 14 changed files with 236 additions and 101 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,4 +13,5 @@ inventory.yml.bak
 *.box
 manifest.xml
 HyperV/.vagrant/*
-logger_variables.sh
+logger_variables.sh
+ESXi/Packer/variables.json
diff --git a/ESXi/Packer/ubuntu1804_esxi.json b/ESXi/Packer/ubuntu1804_esxi.json
@@ -26,36 +26,45 @@
                 "<enter><wait>"
             ],
             "boot_wait": "10s",
-            "keep_registered": true,
-            "remote_datastore": "{{user `esxi_datastore`}}",
-            "remote_host": "{{user `esxi_host`}}",
-            "remote_username": "{{user `esxi_username`}}",
-            "remote_password": "{{user `esxi_password`}}",
-            "remote_type": "esx5",
-            "vnc_disable_password": true,
-            "vnc_port_min": 5900,
-            "vnc_port_max": 5980,
+            "cpus": "{{ user `cpus` }}",
             "disk_size": "{{user `disk_size`}}",
             "guest_os_type": "ubuntu-64",
             "http_directory": "{{user `http_directory`}}",
             "iso_checksum": "{{user `iso_checksum`}}",
             "iso_url": "{{user `mirror`}}/{{user `mirror_directory`}}/{{user `iso_name`}}",
+            "keep_registered": true,
             "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now",
             "ssh_password": "vagrant",
             "ssh_port": 22,
             "ssh_username": "vagrant",
             "ssh_timeout": "10000s",
-            "pause_before_connecting": "10m",
+            "memory": "{{ user `memory` }}",
+            "pause_before_connecting": "1m",
+            "remote_datastore": "{{user `esxi_datastore`}}",
+            "remote_host": "{{user `esxi_host`}}",
+            "remote_username": "{{user `esxi_username`}}",
+            "remote_password": "{{user `esxi_password`}}",
+            "remote_type": "esx5",
+            "skip_export": true,
             "tools_upload_flavor": "linux",
             "type": "vmware-iso",
             "vm_name": "Ubuntu1804",
-            "memory": "{{ user `memory` }}",
-            "cpus": "{{ user `cpus` }}",
             "vmx_data": {
                 "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet` }}",
                 "cpuid.coresPerSocket": "1",
-                "ethernet0.pciSlotNumber": "32"
-            }
+                "ethernet0.pciSlotNumber": "32",
+                "tools.syncTime": "0",
+                "time.synchronize.continue": "0",
+                "time.synchronize.restore": "0",
+                "time.synchronize.resume.disk": "0",
+                "time.synchronize.shrink": "0",
+                "time.synchronize.tools.startup": "0",
+                "time.synchronize.tools.enable": "0",
+                "time.synchronize.resume.host": "0"
+            },
+            "vnc_disable_password": true,
+            "vnc_port_min": 5900,
+            "vnc_port_max": 5980
         }
     ],
     "provisioners": [

diff --git a/ESXi/Packer/windows_10_esxi.json b/ESXi/Packer/windows_10_esxi.json
@@ -1,31 +1,10 @@
 {
   "builders": [
     {
-      "vnc_disable_password": true,
-      "keep_registered": true,
-      "remote_datastore": "{{user `esxi_datastore`}}",
-      "remote_host": "{{user `esxi_host`}}",
-      "remote_username": "{{user `esxi_username`}}",
-      "remote_password": "{{user `esxi_password`}}",
-      "remote_type": "esx5",
-      "type": "vmware-iso",
-      "vm_name":"Windows10",
-      "communicator": "winrm",
-      "iso_url": "{{user `iso_url`}}",
-      "iso_checksum": "{{user `iso_checksum`}}",
-      "headless": false,
       "boot_wait": "6m",
       "boot_command": "",
-      "winrm_username": "vagrant",
-      "winrm_password": "vagrant",
-      "winrm_timeout": "4h",
-      "shutdown_timeout": "2h",
-      "shutdown_command": "a:/sysprep.bat",
-      "guest_os_type": "windows9-64",
-      "disk_size": "{{user `disk_size`}}",
-      "vnc_port_min": 5900,
-      "vnc_port_max": 5980,
-      "version": 11,
+      "communicator": "winrm",
+      "disk_size": 61440,
       "floppy_files": [
         "{{user `autounattend`}}",
         "../../Packer/floppy/WindowsPowershell.lnk",
@@ -40,12 +19,42 @@
         "../../Packer/scripts/unattend.xml",
         "../../Packer/scripts/sysprep.bat"
       ],
+      "guest_os_type": "windows9-64",
+      "keep_registered": true,
+      "headless": false,
+      "iso_url": "{{user `iso_url`}}",
+      "iso_checksum": "{{user `iso_checksum`}}",
+      "shutdown_timeout": "2h",
+      "shutdown_command": "a:/sysprep.bat",
+      "skip_export": true,
+      "remote_datastore": "{{user `esxi_datastore`}}",
+      "remote_host": "{{user `esxi_host`}}",
+      "remote_username": "{{user `esxi_username`}}",
+      "remote_password": "{{user `esxi_password`}}",
+      "remote_type": "esx5",
+      "type": "vmware-iso",
+      "version": 11,
+      "vm_name": "Windows10",
+      "vnc_disable_password": true,
+      "vnc_port_min": 5900,
+      "vnc_port_max": 5980,
       "vmx_data": {
         "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet`}}",
         "memsize": "2048",
         "numvcpus": "2",
-        "scsi0.virtualDev": "lsisas1068"
-      }
+        "scsi0.virtualDev": "lsisas1068",
+        "tools.syncTime": "0",
+        "time.synchronize.continue": "0",
+        "time.synchronize.restore": "0",
+        "time.synchronize.resume.disk": "0",
+        "time.synchronize.shrink": "0",
+        "time.synchronize.tools.startup": "0",
+        "time.synchronize.tools.enable": "0",
+        "time.synchronize.resume.host": "0"
+      },
+    "winrm_username": "vagrant",
+    "winrm_password": "vagrant",
+    "winrm_timeout": "4h"
     }
   ],
   "provisioners": [
@@ -93,8 +102,7 @@
     "esxi_password": "",
     "iso_checksum": "sha256:ab4862ba7d1644c27f27516d24cb21e6b39234eb3301e5f1fb365a78b22f79b3",
     "iso_url": "https://software-download.microsoft.com/download/pr/18362.30.190401-1528.19h1_release_svc_refresh_CLIENTENTERPRISEEVAL_OEMRET_x64FRE_en-us.iso",
-    "autounattend": "../../Packer/answer_files/10/Autounattend.xml",
-    "disk_size": "61440"
+    "autounattend": "../../Packer/answer_files/10/Autounattend.xml"
   }
 }
 
diff --git a/ESXi/Packer/windows_2016_esxi.json b/ESXi/Packer/windows_2016_esxi.json
@@ -1,28 +1,9 @@
 {
   "builders": [
     {
-      "vnc_disable_password": true,
-      "keep_registered": true,
-      "remote_datastore": "{{user `esxi_datastore`}}",
-      "remote_host": "{{user `esxi_host`}}",
-      "remote_username": "{{user `esxi_username`}}",
-      "remote_password": "{{user `esxi_password`}}",
-      "remote_type": "esx5",
-      "vm_name":"WindowsServer2016",
-      "type": "vmware-iso",
-      "communicator": "winrm",
-      "iso_url": "{{user `iso_url`}}",
-      "iso_checksum": "{{user `iso_checksum`}}",
-      "headless": false,
       "boot_wait": "2m",
-      "winrm_username": "vagrant",
-      "winrm_password": "vagrant",
-      "winrm_timeout": "4h",
-      "shutdown_timeout": "2h",
-      "shutdown_command": "a:/sysprep.bat",
-      "guest_os_type": "windows8srv-64",
+      "communicator": "winrm",
       "disk_size": 61440,
-      "version": 11,
       "floppy_files": [
         "{{user `autounattend`}}",
         "../../Packer/floppy/WindowsPowershell.lnk",
@@ -35,12 +16,40 @@
         "../../Packer/scripts/microsoft-updates.bat",
         "../../Packer/scripts/win-updates.ps1"
       ],
+      "guest_os_type": "windows8srv-64",
+      "headless": false,
+      "iso_url": "{{user `iso_url`}}",
+      "iso_checksum": "{{user `iso_checksum`}}",
+      "keep_registered": true,
+      "shutdown_timeout": "2h",
+      "shutdown_command": "a:/sysprep.bat",
+      "skip_export": true,
+      "remote_datastore": "{{user `esxi_datastore`}}",
+      "remote_host": "{{user `esxi_host`}}",
+      "remote_username": "{{user `esxi_username`}}",
+      "remote_password": "{{user `esxi_password`}}",
+      "remote_type": "esx5",
+      "type": "vmware-iso",
+      "version": 11,
+      "vm_name": "WindowsServer2016",
+      "vnc_disable_password": true,
       "vmx_data": {
         "ethernet0.networkName": "{{user `esxi_network_with_dhcp_and_internet`}}",
         "memsize": "2048",
         "numvcpus": "2",
-        "scsi0.virtualDev": "lsisas1068"
-      }
+        "scsi0.virtualDev": "lsisas1068",
+        "tools.syncTime": "0",
+        "time.synchronize.continue": "0",
+        "time.synchronize.restore": "0",
+        "time.synchronize.resume.disk": "0",
+        "time.synchronize.shrink": "0",
+        "time.synchronize.tools.startup": "0",
+        "time.synchronize.tools.enable": "0",
+        "time.synchronize.resume.host": "0"
+      },
+      "winrm_username": "vagrant",
+      "winrm_password": "vagrant",
+      "winrm_timeout": "4h"
     }
   ],
   "provisioners": [

diff --git a/ESXi/main.tf b/ESXi/main.tf
@@ -29,9 +29,8 @@ resource "esxi_guest" "logger" {
 
     provisioner "remote-exec" {
     inline = [
-      "sudo ifconfig eth1 up || echo 'eth1 up'",
-      "sudo ifconfig eth2 up || echo 'eth2 up'",
-      "sudo route add default gw 192.168.76.1 || echo 'route exists'"
+      "sudo ifconfig eth0 up && echo 'eth0 up' || echo 'unable to bring eth0 interface up",
+      "sudo ifconfig eth1 up && echo 'eth1 up' || echo 'unable to bring eth1 interface up"
     ]
 
     connection {
@@ -72,7 +71,6 @@ resource "esxi_guest" "dc" {
   guestos    = "windows9srv-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "4096"
   numvcpus           = "2"
@@ -101,7 +99,6 @@ resource "esxi_guest" "wef" {
   guestos    = "windows9srv-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "2048"
   numvcpus           = "2"
@@ -130,7 +127,6 @@ resource "esxi_guest" "win10" {
   guestos    = "windows9-64"
 
   boot_disk_type = "thin"
-  boot_disk_size = "35"
 
   memsize            = "2048"
   numvcpus           = "2"

diff --git a/ESXi/variables.tf b/ESXi/variables.tf
@@ -1,7 +1,8 @@
 #
 #  See https://www.terraform.io/intro/getting-started/variables.html for more details.
 #
-#  Change these defaults to fit your needs!
+#  Don't change the variables in this file! 
+#  Instead, create a terrform.tfvars file to override them.
 
 variable "esxi_hostname" {
   default = ""

diff --git a/Vagrant/logger_bootstrap.sh b/Vagrant/logger_bootstrap.sh
@@ -377,6 +377,11 @@ install_zeek() {
   crudini --set $NODECFG proxy host localhost
 
   # Setup $CPUS numbers of Zeek workers
+  crudini --set $NODECFG worker-eth0 type worker
+  crudini --set $NODECFG worker-eth0 host localhost
+  crudini --set $NODECFG worker-eth0 interface eth0
+  crudini --set $NODECFG worker-eth0 lb_method pf_ring
+  crudini --set $NODECFG worker-eth0 lb_procs "$(nproc)"
   crudini --set $NODECFG worker-eth1 type worker
   crudini --set $NODECFG worker-eth1 host localhost
   crudini --set $NODECFG worker-eth1 interface eth1
@@ -391,7 +396,7 @@ install_zeek() {
   # Configure the Splunk inputs
   mkdir -p /opt/splunk/etc/apps/Splunk_TA_bro/local && touch /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager index zeek
-  crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager sourcetype bro:json
+  crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager sourcetype zeek:json
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager whitelist '.*\.log$'
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager blacklist '.*(communication|stderr)\.log$'
   crudini --set /opt/splunk/etc/apps/Splunk_TA_bro/local/inputs.conf monitor:///opt/zeek/spool/manager disabled 0
@@ -464,12 +469,11 @@ install_suricata() {
   suricata-update enable-source ptresearch/attackdetection
 
   # Configure the Splunk inputs
-  mkdir -p /opt/splunk/etc/apps/SplunkLightForwarder/local && touch /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata index suricata
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata sourcetype suricata:json
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata whitelist 'eve.json'
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/inputs.conf monitor:///var/log/suricata disabled 0
-  crudini --set /opt/splunk/etc/apps/SplunkLightForwarder/local/props.conf json_suricata TRUNCATE 0
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata index suricata
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata sourcetype suricata:json
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata whitelist 'eve.json'
+  crudini --set /opt/splunk/etc/apps/search/local/inputs.conf monitor:///var/log/suricata disabled 0
+  crudini --set /opt/splunk/etc/apps/search/local/props.conf suricata:json TRUNCATE 0
 
   # Update suricata and restart
   suricata-update

diff --git a/Vagrant/resources/GPO/rdp_users/manifest.xml b/Vagrant/resources/GPO/rdp_users/manifest.xml
diff --git a/Vagrant/resources/splunk_server/logger_dashboard.xml b/Vagrant/resources/splunk_server/logger_dashboard.xml
@@ -69,13 +69,14 @@
       <title>Zeek Network Traffic by Type</title>
       <chart>
         <search>
-          <query>index=zeek | stats count by _time, tag::eventtype | timechart span=1h count by tag::eventtype</query>
+          <query>| tstats count where index=zeek by source, _time span=1h prestats=t | timechart span=1h count by source useother=f</query>
           <earliest>-24h@h</earliest>
           <latest>now</latest>
         </search>
         <option name="charting.chart">column</option>
         <option name="charting.chart.stackMode">stacked</option>
         <option name="charting.drilldown">none</option>
+        <option name="refresh.display">progressbar</option>
       </chart>
     </panel>
   </row>
@@ -125,16 +126,18 @@
       <table>
         <title>http://findingbad.blogspot.com/2020/05/hunting-for-beacons-part-2.html</title>
         <search>
-          <query>index=zeek (dest_port=443 OR dest_port=80)  
-| rename orig_bytes as bytes_out resp_bytes as bytes_in 
-| stats count(bytes_out) as "beacon_count" values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out |eventstats sum(beacon_count) as total_count dc(bytes_out) as unique_count by src_ip,dest_ip 
-| eval beacon_avg=('beacon_count' / 'total_count') 
-| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out 
-| head 100
-| eval incount=mvcount(bytes_in) 
-| eventstats avg(beacon_count) as overall_average 
-| eval beacon_percentage=('beacon_count' / 'overall_average') 
-| sort - beacon_percentage</query>
+          <query>index=zeek (dest_port=443 OR dest_port=80)  dest_ip!=192.168.0.0/16
+| rename orig_bytes as bytes_out resp_bytes as bytes_in
+| stats count(bytes_out) as "beacon_count" values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out |eventstats sum(beacon_count) as total_count dc(bytes_out) as unique_count by src_ip,dest_ip
+| eval beacon_avg=('beacon_count' / 'total_count')
+| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out
+| eval beacon_avg=('beacon_count' / 'total_count')
+| stats values(beacon_count) as beacon_count values(unique_count) as unique_count values(beacon_avg) as beacon_avg values(total_count) as total_count values(bytes_in) as bytes_in by src_ip,dest_ip,bytes_out
+| eval incount=mvcount(bytes_in)
+| eventstats avg(beacon_count) as overall_average
+| eval beacon_percentage=('beacon_count' / 'overall_average')
+| sort - beacon_percentage
+| fields - incount,overall_average</query>
           <earliest>-24h@h</earliest>
           <latest>now</latest>
         </search>
@@ -215,4 +218,4 @@
       </chart>
     </panel>
   </row>
-</dashboard>
+</dashboard>
diff --git a/Vagrant/resources/suricata/suricata.yaml b/Vagrant/resources/suricata/suricata.yaml
@@ -124,11 +124,14 @@ logging:
       facility: local5
       format: "[%i] <%d> -- "
 af-packet:
+  - interface: eth0
+    cluster-id: 98
+    cluster-type: cluster_flow
+    defrag: yes
   - interface: eth1
     cluster-id: 99
     cluster-type: cluster_flow
     defrag: yes
-  - interface: default
 pcap-file:
   checksum-checks: auto
 app-layer:

diff --git a/ci/build_machine_bootstrap.sh b/ci/build_machine_bootstrap.sh
@@ -79,8 +79,8 @@ ufw --force enable
 echo "[$(date +%H:%M:%S)]: Installing Vagrant..."
 mkdir /opt/vagrant
 cd /opt/vagrant || exit 1
-wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.10/vagrant_2.2.10_x86_64.deb
-dpkg -i vagrant_2.2.10_x86_64.deb
+wget --progress=bar:force https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.deb
+dpkg -i vagrant_2.2.14_x86_64.deb
 echo "[$(date +%H:%M:%S)]: Installing vagrant-reload plugin..."
 vagrant plugin install vagrant-reload