Adds dnsmonitor and dnstwist scan updates

src/pe_source/dnsmonitor.py

@@ -69,32 +69,33 @@ def run_dnsMonitor(self):
         domain_df = get_monitored_domains(token)
 
         failed = []
+        warnings = []
         # Iterate through each org
-        for org in pe_orgs_final:
+        for org_idx, org in enumerate(pe_orgs_final):
             org_name = org["name"]
             org_uid = org["organizations_uid"]
             org_code = org["cyhy_db_name"]
-            LOGGER.info("\nRunning DNSMonitor on %s", org_code)
+            LOGGER.info(f"Running DNSMonitor on {org_code} ({org_idx+1} of {len(pe_orgs_final)})")
 
             # Get respective domain IDs
             domain_ids = domain_df[domain_df["org"] == org_name]
-            LOGGER.info("Found %s root domains being monitored.", len(domain_ids))
+            LOGGER.info(f"Found {len(domain_ids)} root domains being monitored for {org_code}")
             domain_ids = str(domain_ids["domainId"].tolist())
 
             # Get Alerts for a specific org based on the list of domain IDs
             if domain_ids == "[]":
-                LOGGER.error("Can't match org to any domains...")
-                failed.append(f"{org_code} - No domains")
+                LOGGER.warning(f"No domains being monitored by DNSMonitor for {org_code}")
+                warnings.append(f"{org_code} - No domains being monitored")
                 continue
             else:
                 alerts_df = get_domain_alerts(token, domain_ids, START_DATE, END_DATE)
                 LOGGER.info("Fetched %s alerts.", len(alerts_df.index))
 
-                # If no alerts, continue
-                if alerts_df.empty:
-                    LOGGER.error("No alerts for %s", org_code)
-                    failed.append(f"{org_code} - No alerts")
-                    continue
+            # If no alerts, continue
+            if alerts_df.empty:
+                LOGGER.warning(f"No DNSMonitor alerts found for {org_code}")
+                warnings.append(f"{org_code} - No alerts found")
+                continue
 
             for alert_index, alert_row in alerts_df.iterrows():
                 # Get subdomain_uid
@@ -165,11 +166,11 @@ def run_dnsMonitor(self):
             try:
                 execute_dnsmonitor_data(dom_perm_df) # api ver.
                 # execute_dnsmonitor_data(dom_perm_df, "domain_permutations") # tsql ver.
-                # LOGGER.info("Success inserting into domain_permutations - %s", org_code) # tsql ver.
+                # LOGGER.info("Success inserting into domain_permutations - %s", org_code)
             except Exception as e:
                 LOGGER.error("Failed inserting into domain_permutations - %s", org_code)
                 LOGGER.error(e)
-                failed.append(f"{org_code} - Failed inserting into dom_perms")
+                failed.append(f"{org_code} - Failed inserting into domain_permutations")
 
             # Format dataframe and insert into domain_alerts table
             alerts_df = alerts_df.rename(columns={"date_observed": "date"})
@@ -188,12 +189,26 @@ def run_dnsMonitor(self):
             try:
                 execute_dnsmonitor_alert_data(domain_alerts) # api ver.
                 # execute_dnsmonitor_alert_data(domain_alerts, "domain_alerts") # tsql ver.
-                # LOGGER.info("Success inserting into domain_alerts - %s", org_code) # tsql ver.
+                # LOGGER.info("Success inserting into domain_alerts - %s", org_code)
             except Exception as e:
                 LOGGER.error("Failed inserting into domain_alerts - %s", org_code)
                 LOGGER.error(e)
-                failed.append(f"{org_code} - Failed inserting into dom_alerts")
+                failed.append(f"{org_code} - Failed inserting into domain_alerts")
+
+        # Output any warnings
+        if len(warnings) > 0:
+            print("Warnings: %s", warnings)
 
         # Output any failures
         if len(failed) > 0:
-            LOGGER.error("Failures: %s", failed)
+            print("Failures: %s", failed)
+
+        # Output summary stats
+        num_no_domain_monitor = sum('No domains being monitored' in s for s in warnings)
+        num_no_alerts = sum('No alerts found' in s for s in warnings)
+        num_success = len(pe_orgs_final) - num_no_domain_monitor - num_no_alerts - len(failed)
+        num_fail = len(failed)
+        LOGGER.info(f"{num_no_domain_monitor}/{len(pe_orgs_final)} orgs do not have domains being monitored by DNSMonitor")
+        LOGGER.info(f"{num_no_alerts}/{len(pe_orgs_final)} orgs have domains being monitored, but didn't have any new alerts")
+        LOGGER.info(f"{num_success}/{len(pe_orgs_final)} orgs had new DNSMonitor findings and successfully added them to the database")
+        LOGGER.info(f"{num_fail}/{len(pe_orgs_final)} orgs had a significant failure during the DNSMonitor scan")

src/pe_source/dnstwistscript.py

@@ -1,18 +1,38 @@
 """Use DNS twist to fuzz domain names and cross check with a blacklist."""
 # Standard Python Libraries
+import os 
+import sys
+import django
 import contextlib
 import datetime
 import json
 import logging
 import pathlib
 import traceback
+import uuid
+
+# Dynamically resolve the root directory of the Django project
+current_file_path = os.path.dirname(os.path.abspath(__file__))  # Path of the current script
+project_root = os.path.join(current_file_path, '../pe_reports/pe_reports_django_project')  # Adjust relative to `pe_reports`
+# Add the resolved project root to sys.path
+sys.path.append(project_root)
+
+# Set the Django settings module
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pe_reports_django.settings')
+
+# Initialize Django
+django.setup()
+
+from dmz_mini_dl.models import DataSource as MDLDataSource, SubDomains as MDLSubDomains, Organization as MDLOrganization, DomainPermutations as MDLDomainPermutations
+from home.models import DataSource, SubDomains, DnsRecords, Organizations
 
 # Third-Party Libraries
 import dnstwist
 import dshield
 import psycopg2.extras as extras
 import requests
 
+
 from .data.pe_db.db_query_source import (
     addSubdomain,
     connect,
@@ -291,6 +311,44 @@ def run_dnstwist(orgs_list):
                 LOGGER.info("Failure inserting data into database.")
                 failures.append(org_name)
                 LOGGER.info(traceback.format_exc())
+            try:
+                mdl_organization = MDLOrganization.objects.get(acronym=org['cyhy_db_name'])
+                for domain in domain_list:
+                    mdl_data_source = None
+                    mdl_subdomain = None
+                    try:
+                        data_source = DataSource.objects.get(data_source_uid=domain['data_source_uid'])
+                        mdl_data_source = MDLDataSource.objects.get(name=data_source.name)
+                        sub_domain = SubDomains.objects.get(sub_domain_uid=domain['sub_domain_uid'])
+                        mdl_subdomain = MDLSubDomains.objects.get(sub_domain=sub_domain.sub_domain)
+                    except MDLSubDomains.DoesNotExist:
+                        mdl_subdomain = None
+                    except MDLDataSource.DoesNotExist:
+                        mdl_domain_permutation = None
+                    except Exception:
+                        print('unknown error occurred')
+                    mdl_domain_permutation = MDLDomainPermutations.objects.create(
+                        organization=mdl_organization,
+                        data_source=mdl_data_source,
+                        sub_domain=mdl_subdomain,
+                        domain_permutation=domain['domain_permutation'],
+                        ipv4=domain['ipv4'],
+                        ipv6=domain['ipv6'],
+                        mail_server=domain['mail_server'],
+                        name_server=domain['name_server'],
+                        fuzzer=domain['fuzzer'],
+                        blocklist_attack_count=domain['blocklist_attack_count'],
+                        blocklist_report_count=domain['blocklist_report_count'],
+                        malicious=domain['malicious'],
+                        ssdeep_score=domain['ssdeep_score'],
+                        dshield_record_count=domain['dshield_record_count'],
+                        dshield_attack_count=domain['dshield_attack_count'],
+                        date_active=domain['date_active']
+                        )
+                    print('mdl perm',mdl_domain_permutation)
+            except Exception as error:
+                print('Error inserting data into data lake', error)
+                LOGGER.info("Failure inserting data into data lake")
 
     PE_conn.close()
     if failures != []:
@@ -299,4 +357,4 @@ def run_dnstwist(orgs_list):
 
 
 if __name__ == "__main__":
-    run_dnstwist("all")
+    run_dnstwist("all")