notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('LINKS.csv', 'r') as f:\n",
    "    reader = csv.reader(f)\n",
    "    queue = 1 \n",
    "    total_emails = 0\n",
    "    for row in reader:\n",
    "        start = time.time()\n",
    "        emails = []\n",
    "        city = row[0]\n",
    "        state = row[1]\n",
    "        URL = row[2]\n",
    "        print(city + \", \" + state + \" (\" + str(queue) + \" of n)\")\n",
    "        print(\"Searching LINK (\" + URL + \")\")\n",
    "        page = requests.get(URL)\n",
    "        soup = BeautifulSoup(page.content, 'html.parser')\n",
    "        mailtos = soup.select('a[href^=mailto]')\n",
    "        for i in mailtos:\n",
    "            href=i['href']\n",
    "            try:\n",
    "                str1, str2 = href.split(':')\n",
    "            except ValueError:\n",
    "                break\n",
    "            emails.append(str2)\n",
    "        total_emails = total_emails + len(emails)\n",
    "        print(\"Found \" + str(len(emails)) + \" emails (Total \"+ str(total_emails))\n",
    "        email_string = ';'.join(emails)\n",
    "        if len(emails) > 0:\n",
    "            for email in emails:\n",
    "                print(email)\n",
    "        queue += 1\n",
    "        time.sleep(1.5)\n",
    "        data = [city, state, email_string]\n",
    "        with open('EMAILS.csv', 'a') as e:\n",
    "            writer = csv.writer(e)\n",
    "            writer.writerow(data)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('ALL.csv') as email_list:\n",
    "#     reader = csv.reader(email_list, delimiter=',')\n",
    "#     for row in reader:\n",
    "#         city = row[0]\n",
    "#         state = row[1]\n",
    "#         email_list = row[2].split(';')\n",
    "#         email_list_clean = []\n",
    "#         for email in email_list:\n",
    "#             email = email.lower().strip()\n",
    "#             email = email.split('?')\n",
    "#             clean_email = email[0]\n",
    "#             email_list_clean.append(clean_email)\n",
    "#         email_list_clean = list(set(email_list_clean))\n",
    "#         for clean_email in email_list_clean:\n",
    "#             if len(clean_email) > 0:\n",
    "#                 row = [city, state, clean_email]\n",
    "#                 with open(\"MASTER.csv\",'a') as output:\n",
    "#                     writer = csv.writer(output, delimiter=',')\n",
    "#                     writer.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DPW\n",
    "\n",
    "single = ['public']\n",
    "\n",
    "keywords = ['dpw','pw','pwd','publicworks', 'pubwork','works',\n",
    "            'road','highway','street','pothole',\n",
    "            'foreman'\n",
    "            ]\n",
    "\n",
    "with open('MASTER.csv') as master:\n",
    "    rows = csv.reader(master, delimiter=',')\n",
    "    count = 0\n",
    "    for row in rows:\n",
    "        city = row[0]\n",
    "        state = row[1]\n",
    "        email = row[2]\n",
    "        if any(x in email for x in keywords):\n",
    "            count += 1\n",
    "            row = [city, state, email]\n",
    "            print(email)\n",
    "            # with open(\"DPW-emails.csv\",'a') as output:\n",
    "            #     writer = csv.writer(output, delimiter=',')\n",
    "            #     writer.writerow(row)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords = ['clerk','treasurer','mayor','info','admin','manager']\n",
    "\n",
    "with open('MASTER.csv') as master:\n",
    "    rows = csv.reader(master, delimiter=',')\n",
    "    count = 0\n",
    "    for row in rows:\n",
    "        city = row[0]\n",
    "        state = row[1]\n",
    "        email = row[2]\n",
    "        if any(x in email for x in keywords):\n",
    "            count += 1\n",
    "            row = [city, state, email]\n",
    "            print(email)\n",
    "            # with open(\"DPW-emails.csv\",'a') as output:\n",
    "            #     writer = csv.writer(output, delimiter=',')\n",
    "            #     writer.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "query = 'QUERY'\n",
    "num_results = 1\n",
    "url = 'https://www.google.com/search?q=' + query + '&num=' + str(num_results)\n",
    "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}\n",
    "response = requests.get(url=url, headers=headers)\n",
    "soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "links = []\n",
    "\n",
    "for result in soup.find_all('div', attrs={'class': 'g'}):\n",
    "    link = result.find('a', href=True)\n",
    "    links.append(link['href'])\n",
    "links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googleapiclient.discovery import build"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from googleapiclient.errors import HttpError\n",
    "from googleapiclient.discovery import build\n",
    "\n",
    "\n",
    "# Replace with your own API key and Custom Search Engine ID\n",
    "api_key = \"AIzaSyD70FOl-LfKRNnHKwsEnTmQUYlQfobtztw\"\n",
    "cse_id = \"f4c2ce78f48524ca3\"\n",
    "\n",
    "# Create a Custom Search API client\n",
    "service = build(\"customsearch\", \"v1\", developerKey=api_key)\n",
    "\n",
    "# Open the input CSV file for reading\n",
    "with open(\"INPUTS.csv\", \"r\") as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "\n",
    "    # Open the output CSV file for writing and add a new \"link\" and \"emails\" field\n",
    "    with open(\"OUTPUTS.csv\", \"w\") as output_file:\n",
    "        fieldnames = reader.fieldnames + [\"link\", \"emails\"]\n",
    "        writer = csv.DictWriter(output_file, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "\n",
    "        # Iterate over each row in the input CSV file\n",
    "        for row in reader:\n",
    "\n",
    "             # Print a separator line after each query is completed\n",
    "            print(\"=\" * 80)\n",
    "\n",
    "            # Get the search query from the \"query\" column\n",
    "            query = row[\"query\"]\n",
    "\n",
    "            try:\n",
    "                # Perform the search\n",
    "                print(f\"Performing search for query: {query}\")\n",
    "                start_time = time.time()\n",
    "                res = service.cse().list(q=query, cx=cse_id).execute()\n",
    "                elapsed_time = time.time() - start_time\n",
    "                print(f\"Search completed in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                # Get the top search result and save it to the \"link\" column\n",
    "                if \"items\" in res and len(res[\"items\"]) > 0:\n",
    "                    link = res[\"items\"][0][\"link\"]\n",
    "                    row[\"link\"] = link\n",
    "                    # print(f\"Top search result for query '{query}': {link}\")\n",
    "\n",
    "                    # Search for email addresses in the HTML of the top search result\n",
    "                    # print(f\"Searching for email addresses in '{link}'\")\n",
    "                    start_time = time.time()\n",
    "                    page = requests.get(link, timeout=3)\n",
    "                    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "                    email_pattern = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"\n",
    "                    email_tags = soup.select('a[href^=mailto]')\n",
    "                    emails = []\n",
    "                    for email_tag in email_tags:\n",
    "                        email = email_tag.get_text().strip()\n",
    "                        if re.match(email_pattern, email):\n",
    "                            emails.append(email)\n",
    "                            print(email)\n",
    "                    elapsed_time = time.time() - start_time\n",
    "                    row[\"emails\"] = \";\".join(emails)\n",
    "                    print(f\"Found {len(emails)} email addresses in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                # Write the row to the output CSV file\n",
    "                writer.writerow(row)\n",
    "\n",
    "            except requests.exceptions.Timeout:\n",
    "                # Handle timeouts by printing a message and continuing to the next query\n",
    "                print(f\"Timed out searching for query '{query}'\")\n",
    "                continue\n",
    "\n",
    "            except HttpError as error:\n",
    "                # Handle API errors by printing an error message and continuing to the next query\n",
    "                print(f\"An error occurred for query '{query}': {error}\")\n",
    "                continue\n",
    "\n",
    "            # Pause for 1 second before the next search to avoid rate limiting the Google Search API\n",
    "            time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Open the input CSV file for reading\n",
    "with open(\"INPUTS.csv\", \"r\") as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "\n",
    "    # Open the output CSV file for writing and add a new \"link\" and \"emails\" field\n",
    "    with open(\"OUTPUTS.csv\", \"w\") as output_file:\n",
    "        fieldnames = reader.fieldnames + [\"link\", \"emails\"]\n",
    "        writer = csv.DictWriter(output_file, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "\n",
    "        # Iterate over each row in the input CSV file\n",
    "        for row in reader:\n",
    "\n",
    "            # Print a separator line after each query is completed\n",
    "            print(\"=\" * 80)\n",
    "\n",
    "            # Get the search query from the \"query\" column\n",
    "            query = row[\"query\"] + \"PLACEHOLDER\"\n",
    "\n",
    "            try:\n",
    "                # Perform the search\n",
    "                print(f\"Performing search for query: {query}\")\n",
    "                start_time = time.time()\n",
    "\n",
    "                num_results = 1\n",
    "                url = 'https://www.google.com/search?q=' + query + '&num=' + str(num_results)\n",
    "                headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}\n",
    "                response = requests.get(url=url, headers=headers)\n",
    "                soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "                elapsed_time = time.time() - start_time\n",
    "                print(f\"Search completed in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                # Get the top search result and save it to the \"link\" column\n",
    "                links = []\n",
    "                for result in soup.find_all('div', attrs={'class': 'g'}):\n",
    "                    link = result.find('a', href=True)\n",
    "                    links.append(link['href'])\n",
    "\n",
    "                if len(links) > 0:\n",
    "                    link = links[0]\n",
    "                    row[\"link\"] = link\n",
    "                    print(f\"Top search result for query '{query}': {link}\")\n",
    "\n",
    "                    # Search for email addresses in the HTML of the top search result\n",
    "                    # print(f\"Searching for email addresses in '{link}'\")\n",
    "                    start_time = time.time()\n",
    "                    page = requests.get(link, timeout=3)\n",
    "                    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "                    email_pattern = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"\n",
    "                    email_tags = soup.select('a[href^=mailto]')\n",
    "                    emails = []\n",
    "                    for email_tag in email_tags:\n",
    "                        email = email_tag.get_text().strip()\n",
    "                        if re.match(email_pattern, email):\n",
    "                            emails.append(email)\n",
    "                            print(email)\n",
    "                    elapsed_time = time.time() - start_time\n",
    "                    row[\"emails\"] = \";\".join(emails)\n",
    "                    print(f\"Found {len(emails)} email addresses in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                # Write the row to the output CSV file\n",
    "                writer.writerow(row)\n",
    "\n",
    "            except requests.exceptions.Timeout:\n",
    "                # Handle timeouts by printing a message and continuing to the next query\n",
    "                print(f\"Timed out searching for query '{query}'\")\n",
    "                continue\n",
    "\n",
    "            # Pause for 1 second before the next search to avoid rate limiting the Google Search API\n",
    "            time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Open the input CSV file for reading\n",
    "with open(\"INPUTS.csv\", \"r\") as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "\n",
    "    # Open the output CSV file for writing and add a new \"link\" and \"emails\" field\n",
    "    with open(\"OUTPUTS.csv\", \"w\") as output_file:\n",
    "        fieldnames = reader.fieldnames + [\"query\", \"link\", \"email\"]\n",
    "        writer = csv.DictWriter(output_file, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "\n",
    "        # Iterate over each row in the input CSV file\n",
    "        for row in reader:\n",
    "\n",
    "            # Get the location query from the \"query\" column\n",
    "            location_query = row[\"query\"]\n",
    "\n",
    "            # Create an array of search variations\n",
    "            search_variations = [\n",
    "                \" public works department\",\n",
    "                \" engineering department\",\n",
    "                \" utilities\",\n",
    "                \" public works directory\"\n",
    "            ]\n",
    "\n",
    "            # Iterate over each search variation\n",
    "            for variation in search_variations:\n",
    "\n",
    "                # Print a separator line after each query is completed\n",
    "                print(\"-\" * 40)\n",
    "\n",
    "                # Get the search query by combining the location and variation\n",
    "                query = location_query + variation\n",
    "\n",
    "                try:\n",
    "                    # Perform the search\n",
    "                    print(f\"Performing search for query: {query}\")\n",
    "                    start_time = time.time()\n",
    "\n",
    "                    num_results = 1\n",
    "                    url = 'https://www.google.com/search?q=' + query + '&num=' + str(num_results)\n",
    "                    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}\n",
    "                    response = requests.get(url=url, headers=headers)\n",
    "                    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "                    elapsed_time = time.time() - start_time\n",
    "                    # print(f\"Search completed in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                    # Get the top search result and save it to the \"link\" column\n",
    "                    links = []\n",
    "                    for result in soup.find_all('div', attrs={'class': 'g'}):\n",
    "                        link = result.find('a', href=True)\n",
    "                        links.append(link['href'])\n",
    "\n",
    "                    if len(links) > 0:\n",
    "                        link = links[0]\n",
    "                        # print(f\"Top search result for query '{query}': {link}\")\n",
    "\n",
    "                        # Search for email addresses in the HTML of the top search result\n",
    "                        start_time = time.time()\n",
    "                        page = requests.get(link, timeout=3)\n",
    "\n",
    "                        soup = BeautifulSoup(page.content, 'html.parser')\n",
    "                        email_pattern = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"\n",
    "                        email_tags = soup.select('a[href^=mailto]')\n",
    "                        emails = []\n",
    "                        for email_tag in email_tags:\n",
    "                            email = email_tag.get_text().strip()\n",
    "                            \n",
    "                            if re.match(email_pattern, email):\n",
    "                                emails.append(email)\n",
    "                                print(email)\n",
    "                                # Write the row to the output CSV file\n",
    "                                writer.writerow({\n",
    "                                    \"city\": row[\"city\"],\n",
    "                                    \"state\": row[\"state\"],\n",
    "                                    \"pop\": row[\"pop\"],\n",
    "                                    \"query\": query,\n",
    "                                    \"link\": link,\n",
    "                                    \"email\": email\n",
    "                                })\n",
    "                        elapsed_time = time.time() - start_time\n",
    "                        print(f\"Found {len(emails)} email addresses in {elapsed_time:.2f} seconds\")\n",
    "\n",
    "                except requests.exceptions.Timeout:\n",
    "                # Handle timeouts by printing a message and continuing to the next query\n",
    "                    print(f\"Timed out searching for query '{query}'\")\n",
    "                    continue\n",
    "\n",
    "                # Pause for 1 second before the next search to avoid rate limiting the Google Search API\n",
    "                time.sleep(.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def fetch_emails(link):\n",
    "    try:\n",
    "        page = requests.get(link, timeout=3)\n",
    "    except requests.RequestException as e:\n",
    "        print(f\"Error fetching page '{link}': {e}\")\n",
    "        return []\n",
    "\n",
    "    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "    email_pattern = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"\n",
    "    email_tags = soup.select('a[href^=mailto]')\n",
    "    emails = [email_tag.get_text().strip() for email_tag in email_tags if re.match(email_pattern, email_tag.get_text().strip())]\n",
    "\n",
    "    return emails\n",
    "\n",
    "# Open the input CSV file for reading\n",
    "with open(\"INPUTS.csv\", \"r\") as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "\n",
    "    # Open the output CSV file for writing and add a new \"link\" and \"emails\" field\n",
    "    with open(\"OUTPUTS.csv\", \"w\") as output_file:\n",
    "        fieldnames = reader.fieldnames + [\"link\", \"email\"]\n",
    "        writer = csv.DictWriter(output_file, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "\n",
    "        # Create a requests session to reuse the TCP connection\n",
    "        session = requests.Session()\n",
    "        session.headers.update({'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'})\n",
    "\n",
    "        # Iterate over each row in the input CSV file\n",
    "        for row in reader:\n",
    "            location_query = row[\"query\"]\n",
    "            search_variations = [\n",
    "                \" public works department\",\n",
    "                \" engineering department\",\n",
    "                \" utilities\",\n",
    "                \" public works directory\"\n",
    "            ]\n",
    "\n",
    "            # Iterate over each search variation\n",
    "            for index, variation in enumerate(search_variations):\n",
    "                print(\"-\" * 40)\n",
    "                query = location_query + variation\n",
    "                print(f\"Performing search for query {index + 1}/{len(search_variations)}: {query}\")\n",
    "\n",
    "                try:\n",
    "                    start_time = time.time()\n",
    "                    num_results = 1\n",
    "                    url = 'https://www.google.com/search?q=' + query + '&num=' + str(num_results)\n",
    "                    response = session.get(url=url)\n",
    "                    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "                    links = [result.find('a', href=True)['href'] for result in soup.find_all('div', attrs={'class': 'g'}) if result.find('a', href=True)]\n",
    "                    if len(links) > 0:\n",
    "                        link = links[0]\n",
    "\n",
    "                        emails = fetch_emails(link)\n",
    "                        for email in emails:\n",
    "                            writer.writerow({\n",
    "                                \"city\": row[\"city\"],\n",
    "                                \"state\": row[\"state\"],\n",
    "                                \"pop\": row[\"pop\"],\n",
    "                                \"query\": query,\n",
    "                                \"link\": link,\n",
    "                                \"email\": email\n",
    "                            })\n",
    "\n",
    "                        print(f\"Found {len(emails)} email addresses\")\n",
    "\n",
    "                except requests.exceptions.Timeout:\n",
    "                    print(f\"Timed out searching for query '{query}'\")\n",
    "                    continue\n",
    "\n",
    "                # Pause for 0.1 seconds before the next search to avoid rate limiting the Google Search API\n",
    "                time.sleep(0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed location: Azusa, CA\n",
      "Locations processed: 1, Emails scraped: 0\n",
      "Processed location: Newark, OH\n",
      "Locations processed: 2, Emails scraped: 0\n",
      "Processed location: Lincoln, CA\n",
      "Locations processed: 3, Emails scraped: 0\n",
      "Processed location: East Brunswick, NJ\n",
      "Locations processed: 4, Emails scraped: 0\n",
      "Processed location: Euclid, OH\n",
      "Locations processed: 5, Emails scraped: 0\n",
      "Processed location: Cerritos, CA\n",
      "Locations processed: 6, Emails scraped: 0\n",
      "Processed location: Rockwall, TX\n",
      "Locations processed: 7, Emails scraped: 0\n",
      "Processed location: Wake Forest, NC\n",
      "Locations processed: 8, Emails scraped: 0\n",
      "Processed location: Biloxi, MS\n",
      "Locations processed: 9, Emails scraped: 0\n",
      "Processed location: Lawrence, IN\n",
      "Locations processed: 10, Emails scraped: 0\n",
      "Processed location: Jeffersonville, IN\n",
      "Locations processed: 11, Emails scraped: 0\n",
      "Processed location: Dublin, OH\n",
      "Locations processed: 12, Emails scraped: 0\n",
      "Processed location: Ceres, CA\n",
      "Locations processed: 13, Emails scraped: 0\n",
      "Processed location: Winter Haven, FL\n",
      "Locations processed: 14, Emails scraped: 0\n",
      "Processed location: Sheboygan, WI\n",
      "Locations processed: 15, Emails scraped: 0\n",
      "Processed location: Coral Gables, FL\n",
      "Locations processed: 16, Emails scraped: 0\n",
      "Processed location: Bedford, TX\n",
      "Locations processed: 17, Emails scraped: 0\n",
      "Processed location: St Louis Park, MN\n",
      "Locations processed: 18, Emails scraped: 0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-88eca3c4c056>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-1-88eca3c4c056>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m                     \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubmit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_row\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch_variations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocations_processed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0memails_scraped\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, exc_type, exc_val, exc_tb)\u001b[0m\n\u001b[1;32m    635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 637\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    638\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    639\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, cancel_futures)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_threads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m     \u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_base\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1051\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1054\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1071\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1072\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1073\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1074\u001b[0m                 \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1075\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed location: Everett, MA\n",
      "Locations processed: 19, Emails scraped: 0\n",
      "Processed location: Barnstable, MA\n",
      "Locations processed: 20, Emails scraped: 0\n",
      "Processed location: West Orange, NJ\n",
      "Locations processed: 21, Emails scraped: 0\n",
      "Processed location: Poway, CA\n",
      "Locations processed: 22, Emails scraped: 0\n",
      "Processed location: Titusville, FL\n",
      "Locations processed: 23, Emails scraped: 0\n",
      "Processed location: Hattiesburg, MS\n",
      "Locations processed: 24, Emails scraped: 0\n",
      "Processed location: Glenview, IL\n",
      "Locations processed: 25, Emails scraped: 0\n",
      "Processed location: Washington Township, NJ\n",
      "Locations processed: 26, Emails scraped: 0\n",
      "Processed location: Niagara Falls, NY\n",
      "Locations processed: 27, Emails scraped: 0\n",
      "Processed location: Monroe Township, NJ\n",
      "Locations processed: 28, Emails scraped: 0\n",
      "Processed location: Cedar Hill, TX\n",
      "Locations processed: 29, Emails scraped: 0\n",
      "Processed location: Roswell, NM\n",
      "Locations processed: 30, Emails scraped: 0\n",
      "Processed location: Stillwater , OK\n",
      "Locations processed: 31, Emails scraped: 0\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from threading import Lock\n",
    "from multiprocessing import Value\n",
    "\n",
    "def find_emails(url):\n",
    "    try:\n",
    "        page = requests.get(url, timeout=3)\n",
    "        soup = BeautifulSoup(page.content, 'html.parser')\n",
    "        email_pattern = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"\n",
    "        email_tags = soup.select('a[href^=mailto]')\n",
    "        emails = []\n",
    "        for email_tag in email_tags:\n",
    "            email = email_tag.get_text().strip()\n",
    "            if re.match(email_pattern, email):\n",
    "                emails.append(email)\n",
    "        return emails\n",
    "    except requests.exceptions.RequestException:\n",
    "        return []\n",
    "\n",
    "def perform_search(query, num_results=1):\n",
    "    url = f'https://www.google.com/search?q={query}&num={num_results}'\n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}\n",
    "    response = requests.get(url=url, headers=headers)\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "    links = [result.find('a', href=True)['href'] for result in soup.find_all('div', attrs={'class': 'g'})]\n",
    "    return links\n",
    "\n",
    "def process_row(row, search_variations, writer, lock, locations_processed, emails_scraped):\n",
    "    location_query = row[\"query\"]\n",
    "    for variation in search_variations:\n",
    "        query = location_query + variation\n",
    "        try:\n",
    "            links = perform_search(query)\n",
    "            if links:\n",
    "                link = links[0]\n",
    "                emails = find_emails(link)\n",
    "                with lock:\n",
    "                    for email in emails:\n",
    "                        writer.writerow({\n",
    "                            \"city\": row[\"city\"],\n",
    "                            \"state\": row[\"state\"],\n",
    "                            \"pop\": row[\"pop\"],\n",
    "                            \"query\": query,\n",
    "                            \"link\": link,\n",
    "                            \"email\": email\n",
    "                        })\n",
    "                        emails_scraped.value += 1\n",
    "        except requests.exceptions.RequestException:\n",
    "            continue\n",
    "    with lock:\n",
    "        locations_processed.value += 1\n",
    "        print(f\"Processed location: {row['city']}, {row['state']}\")\n",
    "        print(f\"Locations processed: {locations_processed.value}, Emails scraped: {emails_scraped.value}\")\n",
    "\n",
    "def main():\n",
    "    search_variations = [\n",
    "        \" public works department\",\n",
    "        \" engineering department\",\n",
    "        \" utilities\",\n",
    "        \" public works directory\"\n",
    "    ]\n",
    "\n",
    "    with open(\"INPUTS.csv\", \"r\") as csv_file:\n",
    "        reader = csv.DictReader(csv_file)\n",
    "        with open(\"EMAILS.csv\", \"w\") as output_file:\n",
    "            fieldnames = reader.fieldnames + [\"query\", \"link\", \"email\"]\n",
    "            writer = csv.DictWriter(output_file, fieldnames=fieldnames)\n",
    "            writer.writeheader()\n",
    "\n",
    "            lock = Lock()\n",
    "            locations_processed = Value('i', 0)\n",
    "            emails_scraped = Value('i', 0)\n",
    "\n",
    "            with ThreadPoolExecutor(max_workers=4) as executor:\n",
    "                for row in reader:\n",
    "                    executor.submit(process_row, row, search_variations, writer, lock, locations_processed, emails_scraped)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "eff1bbf364d39ae6a6577f8cc8c33f80caf078d85856d233c0bf52485b936d90"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit ('lasers': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}