3_feature_engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering\n",
    "Here, in this notebook, we will \n",
    "\n",
    "1. Create our final labels\n",
    "2. Do some basic feature engineering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import sys\n",
    "\n",
    "from collections import Counter\n",
    "from loguru import logger\n",
    "from pathlib import Path\n",
    "from time import time, strftime, gmtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Configuring the logger\n",
    "config = {\"handlers\": [{\"sink\": sys.stdout,\"colorize\": True,\n",
    "          \"format\": \"<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>\"}]}\n",
    "logger.configure(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Folder\n",
    "data_folder = Path.home() / 'Data' / 'cc_news'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Getting our keywords from each document "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2020-01-12 22:47:55\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m16\u001b[0m - \u001b[1mIt took 00:00:05 to run this script!\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "start_time = time()\n",
    "with open(data_folder / 'spark_output' / 'df_final.csv', 'r') as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "    keyword_counter = Counter()\n",
    "    for i, row in enumerate(reader):\n",
    "        keywords = row['keywords']\n",
    "        if keywords != \"\" or keywords is not None:\n",
    "            try:\n",
    "                split_keywords = re.split('[,;-]', keywords)\n",
    "            except TypeError:\n",
    "                pass\n",
    "            for k in split_keywords:\n",
    "                if k != '' and len(k)>2:\n",
    "                    keyword_counter.update({k.strip().lower(): 1})\n",
    "elapsed = strftime(\"%H:%M:%S\", gmtime(time() - start_time))\n",
    "logger.info(f'It took {elapsed} to run this script!')\n",
    "os.system(\"play /usr/share/sounds/sound-icons/trumpet-1.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually Create a short list of main topic keywords that we will use as a lookup\n",
    "main_topics = {(\"government / politics\", \"politics\", \"government\", \"us government news\", \"government news\"): \"politics\",\n",
    "               (\"economics\", \"economy\", \"trade\", \"business\", \"company news\",\"finance\", \"market reports\",\n",
    "                \"stock\", \"stocks\" \"company\", \"companies\", \"tech\", \"technology\",\n",
    "                \"mergers / acquisitions / takeovers\", \"corporate events\", \"equity markets\",\n",
    "                \"international trade\",\"equities markets\", \"markets\", \"market\", \"emerging market countries\"): \"business_economy\",\n",
    "               (\"world\",\"world news\", \"europe\", \"western europe\",\"asia\", \"asia / pacific\", \n",
    "                \"africa\",\"west africa\",\"east africa\", \"southern africa\", \"european union\"\n",
    "                \"latin america\", \"americas\", \"mid east\", \"middle east\"): \"world\",\n",
    "                (\"entertainment\",\"entertainment news\", \"celebrity\", \"gossip\", \"music\"): \"entertainment\",\n",
    "                ('sport', 'sports', 'league', 'game', 'soccer', 'basketball', 'football', 'games', 'score', 'scores', \"nfl\", \"premier league\"): \"sports\",\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2020-01-12 22:50:42\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m37\u001b[0m - \u001b[1mIt took 00:00:10 to run this script!\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here we are going to get the most comment key words that co-exists with the main ones above.\n",
    "# We will throw them into a counter based in the key  \n",
    "# hard label each document if at least 2 of the keywords exist in the list above\n",
    "start_time = time()\n",
    "with open(data_folder / 'spark_output' / 'df_final.csv', 'r') as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "    counters = {\"politics\" : Counter(), \n",
    "                \"world\": Counter(),\n",
    "                \"business_economy\":Counter(),\n",
    "                'sports':Counter(),\n",
    "                'entertainment':Counter(),\n",
    "                'other': Counter()\n",
    "               }\n",
    "    \n",
    "    exclude = [\"news\", 'local', \"general news\", \"united states\", \"travel\",\n",
    "              \"breaking news\", \"breaking\", \"major events\", \"major news\", \n",
    "               \"current events\", \"latest news\", \"current_news\", \"picture available\", \"pictures\", \"picture\"]\n",
    "    main_topic_kw = [k for key in main_topics.keys() for k in key] + exclude\n",
    "    for row in reader:\n",
    "        keywords = row['keywords']\n",
    "        if keywords != \"\" or keywords is not None:\n",
    "            try:\n",
    "                split_keywords = re.split('[,;-]', keywords)\n",
    "                split_keywords = [s.strip().lower() for s in split_keywords]\n",
    "                for k, v in main_topics.items():\n",
    "                    if len(set(split_keywords).intersection(set(k))) > 1:\n",
    "                        for kw in split_keywords:\n",
    "                            if kw != '' and len(kw)>2 and kw not in main_topic_kw:\n",
    "                                counters[v].update({kw: 1})\n",
    "                    else:\n",
    "                        for kw in split_keywords:\n",
    "                            if kw != '' and len(kw)>2 and kw not in main_topic_kw:\n",
    "                                counters['other'].update({kw: 1})\n",
    "\n",
    "            except TypeError:\n",
    "                pass\n",
    "    \n",
    "elapsed = strftime(\"%H:%M:%S\", gmtime(time() - start_time))\n",
    "logger.info(f'It took {elapsed} to run this script!')\n",
    "os.system(\"play /usr/share/sounds/sound-icons/trumpet-1.wav\") # Beeps an alert. May not run on your machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "politics [('usa', 209), ('diplomacy / foreign policy', 145), ('us house of representatives', 137), ('international / national security', 114), ('trump', 113), ('judicial process / court cases / court decisions', 113), ('impeachment', 107), ('donald trump', 104), ('us senate', 103), ('ukraine', 102), ('lawmaking', 101), ('crime / law / justice', 100), ('crime', 92), ('russia', 90), ('elections / voting', 64), ('presidential elections', 61), ('exclusive', 46), ('video', 45), ('nancy pelosi', 42), ('graphics', 41), ('stadium', 40), ('european union', 39), ('match', 38), ('biofuels', 36), ('region', 34), ('message', 32), ('health / medicine', 31), ('oil and gas (trbc)', 30), ('conflicts / war / peace', 30), ('joe biden', 30)] \n",
      "\n",
      "world [('united kingdom', 256), ('france', 162), ('european union', 152), ('china (prc)', 151), ('diplomacy / foreign policy', 151), ('germany', 145), ('financials (legacy)', 113), ('japan', 110), ('euro zone', 108), ('financials (trbc)', 104), ('italy', 104), ('judicial process / court cases / court decisions', 102), ('governance', 102), ('hong kong', 102), ('auto and truck manufacturers (trbc)', 100), ('cyclical consumer goods and services (trbc)', 100), ('deal', 99), ('banking and investment services (trbc)', 98), ('central / eastern europe', 96), ('legal and judicial affairs', 95), ('debt / fixed income markets', 85), ('business and finance', 84), ('crime / law / justice', 80), ('transportation (trbc)', 78), ('international / national security', 77), ('regulation', 75), ('automobiles / auto parts (legacy)', 75), ('automobiles and auto parts (trbc)', 75), ('nigeria', 74), ('labour / personnel', 74)] \n",
      "\n",
      "business_economy [('commerce', 155), ('china (prc)', 129), ('france', 113), ('financials (legacy)', 112), ('governance', 109), ('deal', 109), ('financials (trbc)', 107), ('united kingdom', 106), ('cyclical consumer goods and services (trbc)', 106), ('legal and judicial affairs', 97), ('hong kong', 96), ('regulation', 94), ('company', 93), ('banking and investment services (trbc)', 92), ('business and finance', 90), ('auto and truck manufacturers (trbc)', 90), ('japan', 88), ('italy', 86), ('debt / fixed income markets', 85), ('deals', 83), ('china', 78), ('reports', 77), ('content produced in bangalore', 76), ('nigeria', 73), ('south africa', 72), ('usa', 71), ('labour / personnel', 67), ('national government debt', 66), ('euro zone', 66), ('industrials (trbc)', 65)] \n",
      "\n",
      "sports [('team', 163), ('american football', 113), ('point', 89), ('rebound', 76), ('sky', 74), ('prep', 73), ('college', 70), ('win', 60), ('season', 60), ('lead', 56), ('assist', 52), ('stadium', 46), ('arsenal', 46), ('live', 44), ('liverpool', 44), ('coach', 43), ('steal', 39), ('player', 39), ('match', 39), ('message', 36), ('fixtures', 35), ('free throw', 34), ('school', 34), ('results', 34), ('region', 32), ('oklahoma city', 32), ('okc', 32), ('tournament', 30), ('barcelona', 29), ('video', 29)] \n",
      "\n",
      "entertainment [('bollywood news', 69), ('movies', 46), ('latest bollywood news', 33), ('bollywood gossip', 33), ('latest bollywood gossip', 33), ('latest bollywood headlines', 33), ('latest bollywood events', 33), ('bollywood news today', 33), ('blog', 33), ('bollywood', 29), ('arts', 21), ('rochester', 19), ('hollywood news', 18), ('events', 18), ('calendar', 18), ('new york', 18), ('alternative news', 18), ('journalism', 18), ('television news', 16), ('celebitchy', 15), ('celebrities', 15), ('famous', 15), ('royals', 15), ('royal', 15), ('liberal politics', 15), ('oscar', 15), ('pop culture', 15), ('post', 15), ('bitchy', 15), ('humor', 15)] \n",
      "\n",
      "other [('donald trump', 6322), ('crime', 5981), ('impeachment', 5155), ('law', 4611), ('christmas', 3501), ('police', 3453), ('india', 3279), ('commerce', 3254), ('stocks', 2939), ('work', 2926), ('china', 2856), ('education', 2741), ('company', 2661), ('criminal law', 2484), ('cnn', 2467), ('school', 2466), ('obituaries', 2456), ('food', 2319), ('trump', 2312), ('barcelona', 2214), ('state', 2194), ('weather', 2027), ('real madrid', 2018), ('health', 2001), ('christianity', 1968), ('congress', 1928), ('nancy pelosi', 1882), ('primera divisiã³n', 1870), ('video', 1853), ('african all stars', 1790)] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Get print out of the counters\n",
    "for k, counter in counters.items():\n",
    "    print(k, counters[k].most_common(30), \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seems legit!\n",
    "\n",
    "## 2. Labeling our documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Switching to pandas: Reading our data\n",
    "df_final = pd.read_csv(data_folder / 'spark_output' / 'df_final.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function will create a hard label for each document if at least 2 keywords exist in the keyword list\n",
    "def label_records_via_keywords(keywords):\n",
    "    if keywords != \"\" or keywords is not None:\n",
    "        try:\n",
    "            split_keywords = re.split('[,;-]', keywords)\n",
    "            split_keywords = [s.strip().lower() for s in split_keywords]\n",
    "            for k, v in main_topics.items():\n",
    "                if len(set(split_keywords).intersection(set(k))) > 1:\n",
    "                    return v\n",
    "        except:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>dom_cat</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>keywords</th>\n",
       "      <th>news_keywords</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://chicago.cbslocal.com/2019/12/18/man-st...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Man Stabbed After His Granddaughter And Three ...</td>\n",
       "      <td>A man's granddaughter is one of four teens acc...</td>\n",
       "      <td>grandfather, granddaughter, teen, teens, home ...</td>\n",
       "      <td>grandfather, granddaughter, teen, teens, home ...</td>\n",
       "      <td>Man Stabbed After His Granddaughter And Three ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.releasewire.com/press-releases/rele...</td>\n",
       "      <td>news</td>\n",
       "      <td>Commence Industrial CRM Achieves Sales Optimiz...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Commence Industrial CRM Achieves Sales Optimiz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.kark.com/news/national-news/1-dead...</td>\n",
       "      <td>news</td>\n",
       "      <td>1 dead, 2 injured in Oregon shopping center st...</td>\n",
       "      <td>Multiple people were stabbed at the Murray Hil...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1 dead, 2 injured in Oregon shopping center st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.ii.co.uk/etfs/sg-wti-x2-daily-long...</td>\n",
       "      <td>financial_news</td>\n",
       "      <td>LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...</td>\n",
       "      <td>Real-time share price updates and latest news ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.peicanada.com/island_deaths/frances...</td>\n",
       "      <td>news</td>\n",
       "      <td>Frances Salsman | Island Deaths | peicanada.com</td>\n",
       "      <td>SALSMAN, Frances (Fran) Townsend At the Prince...</td>\n",
       "      <td>island_deaths, death_notices</td>\n",
       "      <td>island_deaths, death_notices</td>\n",
       "      <td>Frances Salsman | Island Deaths | peicanada.co...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url         dom_cat  \\\n",
       "0  https://chicago.cbslocal.com/2019/12/18/man-st...          sports   \n",
       "1  http://www.releasewire.com/press-releases/rele...            news   \n",
       "2  https://www.kark.com/news/national-news/1-dead...            news   \n",
       "3  https://www.ii.co.uk/etfs/sg-wti-x2-daily-long...  financial_news   \n",
       "4  http://www.peicanada.com/island_deaths/frances...            news   \n",
       "\n",
       "                                               title  \\\n",
       "0  Man Stabbed After His Granddaughter And Three ...   \n",
       "1  Commence Industrial CRM Achieves Sales Optimiz...   \n",
       "2  1 dead, 2 injured in Oregon shopping center st...   \n",
       "3  LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...   \n",
       "4    Frances Salsman | Island Deaths | peicanada.com   \n",
       "\n",
       "                                         description  \\\n",
       "0  A man's granddaughter is one of four teens acc...   \n",
       "1                                                NaN   \n",
       "2  Multiple people were stabbed at the Murray Hil...   \n",
       "3  Real-time share price updates and latest news ...   \n",
       "4  SALSMAN, Frances (Fran) Townsend At the Prince...   \n",
       "\n",
       "                                            keywords  \\\n",
       "0  grandfather, granddaughter, teen, teens, home ...   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3                                                NaN   \n",
       "4                       island_deaths, death_notices   \n",
       "\n",
       "                                       news_keywords  \\\n",
       "0  grandfather, granddaughter, teen, teens, home ...   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3                                                NaN   \n",
       "4                       island_deaths, death_notices   \n",
       "\n",
       "                                                text  \n",
       "0  Man Stabbed After His Granddaughter And Three ...  \n",
       "1  Commence Industrial CRM Achieves Sales Optimiz...  \n",
       "2  1 dead, 2 injured in Oregon shopping center st...  \n",
       "3  LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...  \n",
       "4  Frances Salsman | Island Deaths | peicanada.co...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply Our Function to each row\n",
    "df_final['kw_label'] = df_final.apply(lambda row: label_records_via_keywords(row['keywords']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>dom_cat</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>keywords</th>\n",
       "      <th>news_keywords</th>\n",
       "      <th>text</th>\n",
       "      <th>kw_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>https://www.nwitimes.com/news/local/govt-and-p...</td>\n",
       "      <td>news</td>\n",
       "      <td>Visclosky set to vote today for both impeachme...</td>\n",
       "      <td>U.S. Rep. Pete Visclosky will join the Democra...</td>\n",
       "      <td>, indiana, national-government, local-governme...</td>\n",
       "      <td>, indiana, national-government, local-governme...</td>\n",
       "      <td>Visclosky set to vote today for both impeachme...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>https://www.news4jax.com/news/politics/2019/12...</td>\n",
       "      <td>news</td>\n",
       "      <td>Sports betting's rapid expansion faces more te...</td>\n",
       "      <td>The line for the ticket windows at a Rhode Isl...</td>\n",
       "      <td>Sports,politics,Government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sports betting's rapid expansion faces more te...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>https://www.reuters.com/article/us-basketball-...</td>\n",
       "      <td>news</td>\n",
       "      <td>\\n                Suns' Ayton begins second ac...</td>\n",
       "      <td>Phoenix Suns big man Deandre Ayton is again in...</td>\n",
       "      <td>US,BASKETBALL,NBA,PHX,AYTON,Bahamas,Americas,S...</td>\n",
       "      <td>US;BASKETBALL;NBA;PHX;AYTON;Bahamas;Americas;S...</td>\n",
       "      <td>Suns' Ayton begins second act after suspension...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>https://www.reuters.com/article/us-usa-biofuel...</td>\n",
       "      <td>news</td>\n",
       "      <td>\\n                White House says it is stick...</td>\n",
       "      <td>The Trump administration plans to stick with i...</td>\n",
       "      <td>US,USA,BIOFUELS,EXCLUSIVE,Biofuels,Advocacy Gr...</td>\n",
       "      <td>US;USA;BIOFUELS;EXCLUSIVE;Biofuels;Advocacy Gr...</td>\n",
       "      <td>White House says it is sticking with 2020 biof...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>https://coed.com/2019/11/26/dallas-cowboys-hea...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>Dallas Cowboys Head Coaching Odds 2019: Who Wi...</td>\n",
       "      <td>Who will be the Dallas Cowboys head coach next...</td>\n",
       "      <td>Sports,Sports,Betting Odds,dallas-cowboys,NFL</td>\n",
       "      <td>Sports,Sports,Betting Odds,dallas-cowboys,NFL</td>\n",
       "      <td>Dallas Cowboys Head Coaching Odds 2019: Who Wi...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163132</th>\n",
       "      <td>https://www.redandblack.com/sports/georgia-foo...</td>\n",
       "      <td>news</td>\n",
       "      <td>Georgia football signs 5-star outside lineback...</td>\n",
       "      <td>Mekhail Sherman signed his letter of intent to...</td>\n",
       "      <td>mekhail sherman, linebacker, georgia, sport, b...</td>\n",
       "      <td>mekhail sherman, linebacker, georgia, sport, b...</td>\n",
       "      <td>Georgia football signs 5-star outside lineback...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163221</th>\n",
       "      <td>https://6abc.com/sports/eagles-ready-to-take-o...</td>\n",
       "      <td>news</td>\n",
       "      <td>Philadelphia Eagles ready to take on Cowboys, ...</td>\n",
       "      <td>\"Embarrassing!\" That's the word Carson Wentz, ...</td>\n",
       "      <td>Philadelphia eagles, eagles, nfl, football, Da...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Philadelphia Eagles ready to take on Cowboys, ...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163319</th>\n",
       "      <td>https://madison.com/opinion/letters/coach-gard...</td>\n",
       "      <td>news</td>\n",
       "      <td>Coach Gard must right sinking ship -- Hal Wiss...</td>\n",
       "      <td>What’s going on with the Wisconsin men’s baske...</td>\n",
       "      <td>, letter to editor, commentary, greg gard, bas...</td>\n",
       "      <td>, letter to editor, commentary, greg gard, bas...</td>\n",
       "      <td>Coach Gard must right sinking ship -- Hal Wiss...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163328</th>\n",
       "      <td>https://qctimes.com/sports/high-school/basketb...</td>\n",
       "      <td>news</td>\n",
       "      <td>Central DeWitt remains No. 1 in Area Fab 5 | H...</td>\n",
       "      <td>AREA FAB 5</td>\n",
       "      <td>, northeast, central dewitt, game, sport, albu...</td>\n",
       "      <td>, northeast, central dewitt, game, sport, albu...</td>\n",
       "      <td>Central DeWitt remains No. 1 in Area Fab 5 | H...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163413</th>\n",
       "      <td>https://www.redandblack.com/sports/georgia-foo...</td>\n",
       "      <td>news</td>\n",
       "      <td>Georgia football signs 4-star quarterback Cars...</td>\n",
       "      <td>The Georgia football team officially signed fo...</td>\n",
       "      <td>carson beck, sport, alabama, toby bullock, foo...</td>\n",
       "      <td>carson beck, sport, alabama, toby bullock, foo...</td>\n",
       "      <td>Georgia football signs 4-star quarterback Cars...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3669 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url        dom_cat  \\\n",
       "64      https://www.nwitimes.com/news/local/govt-and-p...           news   \n",
       "199     https://www.news4jax.com/news/politics/2019/12...           news   \n",
       "302     https://www.reuters.com/article/us-basketball-...           news   \n",
       "309     https://www.reuters.com/article/us-usa-biofuel...           news   \n",
       "342     https://coed.com/2019/11/26/dallas-cowboys-hea...  entertainment   \n",
       "...                                                   ...            ...   \n",
       "163132  https://www.redandblack.com/sports/georgia-foo...           news   \n",
       "163221  https://6abc.com/sports/eagles-ready-to-take-o...           news   \n",
       "163319  https://madison.com/opinion/letters/coach-gard...           news   \n",
       "163328  https://qctimes.com/sports/high-school/basketb...           news   \n",
       "163413  https://www.redandblack.com/sports/georgia-foo...           news   \n",
       "\n",
       "                                                    title  \\\n",
       "64      Visclosky set to vote today for both impeachme...   \n",
       "199     Sports betting's rapid expansion faces more te...   \n",
       "302     \\n                Suns' Ayton begins second ac...   \n",
       "309     \\n                White House says it is stick...   \n",
       "342     Dallas Cowboys Head Coaching Odds 2019: Who Wi...   \n",
       "...                                                   ...   \n",
       "163132  Georgia football signs 5-star outside lineback...   \n",
       "163221  Philadelphia Eagles ready to take on Cowboys, ...   \n",
       "163319  Coach Gard must right sinking ship -- Hal Wiss...   \n",
       "163328  Central DeWitt remains No. 1 in Area Fab 5 | H...   \n",
       "163413  Georgia football signs 4-star quarterback Cars...   \n",
       "\n",
       "                                              description  \\\n",
       "64      U.S. Rep. Pete Visclosky will join the Democra...   \n",
       "199     The line for the ticket windows at a Rhode Isl...   \n",
       "302     Phoenix Suns big man Deandre Ayton is again in...   \n",
       "309     The Trump administration plans to stick with i...   \n",
       "342     Who will be the Dallas Cowboys head coach next...   \n",
       "...                                                   ...   \n",
       "163132  Mekhail Sherman signed his letter of intent to...   \n",
       "163221  \"Embarrassing!\" That's the word Carson Wentz, ...   \n",
       "163319  What’s going on with the Wisconsin men’s baske...   \n",
       "163328                                         AREA FAB 5   \n",
       "163413  The Georgia football team officially signed fo...   \n",
       "\n",
       "                                                 keywords  \\\n",
       "64      , indiana, national-government, local-governme...   \n",
       "199                            Sports,politics,Government   \n",
       "302     US,BASKETBALL,NBA,PHX,AYTON,Bahamas,Americas,S...   \n",
       "309     US,USA,BIOFUELS,EXCLUSIVE,Biofuels,Advocacy Gr...   \n",
       "342         Sports,Sports,Betting Odds,dallas-cowboys,NFL   \n",
       "...                                                   ...   \n",
       "163132  mekhail sherman, linebacker, georgia, sport, b...   \n",
       "163221  Philadelphia eagles, eagles, nfl, football, Da...   \n",
       "163319  , letter to editor, commentary, greg gard, bas...   \n",
       "163328  , northeast, central dewitt, game, sport, albu...   \n",
       "163413  carson beck, sport, alabama, toby bullock, foo...   \n",
       "\n",
       "                                            news_keywords  \\\n",
       "64      , indiana, national-government, local-governme...   \n",
       "199                                                   NaN   \n",
       "302     US;BASKETBALL;NBA;PHX;AYTON;Bahamas;Americas;S...   \n",
       "309     US;USA;BIOFUELS;EXCLUSIVE;Biofuels;Advocacy Gr...   \n",
       "342         Sports,Sports,Betting Odds,dallas-cowboys,NFL   \n",
       "...                                                   ...   \n",
       "163132  mekhail sherman, linebacker, georgia, sport, b...   \n",
       "163221                                                NaN   \n",
       "163319  , letter to editor, commentary, greg gard, bas...   \n",
       "163328  , northeast, central dewitt, game, sport, albu...   \n",
       "163413  carson beck, sport, alabama, toby bullock, foo...   \n",
       "\n",
       "                                                     text  kw_label  \n",
       "64      Visclosky set to vote today for both impeachme...  politics  \n",
       "199     Sports betting's rapid expansion faces more te...  politics  \n",
       "302     Suns' Ayton begins second act after suspension...    sports  \n",
       "309     White House says it is sticking with 2020 biof...  politics  \n",
       "342     Dallas Cowboys Head Coaching Odds 2019: Who Wi...    sports  \n",
       "...                                                   ...       ...  \n",
       "163132  Georgia football signs 5-star outside lineback...    sports  \n",
       "163221  Philadelphia Eagles ready to take on Cowboys, ...    sports  \n",
       "163319  Coach Gard must right sinking ship -- Hal Wiss...    sports  \n",
       "163328  Central DeWitt remains No. 1 in Area Fab 5 | H...    sports  \n",
       "163413  Georgia football signs 4-star quarterback Cars...    sports  \n",
       "\n",
       "[3669 rows x 8 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[~df_final.kw_label.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>dom_cat</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>keywords</th>\n",
       "      <th>news_keywords</th>\n",
       "      <th>text</th>\n",
       "      <th>kw_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>73277</th>\n",
       "      <td>https://trib.com/sports/college/wyoming/footba...</td>\n",
       "      <td>financial_news</td>\n",
       "      <td>2020 Wyoming signee: Braden Siders | Football ...</td>\n",
       "      <td>Wyoming football got a signed National Letter ...</td>\n",
       "      <td>, college-sports, wyoming cowboys football, re...</td>\n",
       "      <td>, college-sports, wyoming cowboys football, re...</td>\n",
       "      <td>2020 Wyoming signee: Braden Siders | Football ...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107446</th>\n",
       "      <td>https://www.reuters.com/article/us-basketball-...</td>\n",
       "      <td>news</td>\n",
       "      <td>\\n                Knicks hire Blatt in advisor...</td>\n",
       "      <td>David Blatt, who led the Cleveland Cavaliers t...</td>\n",
       "      <td>US,BASKETBALL,NBA,NYK,BLATT,Sport,Basketball,N...</td>\n",
       "      <td>US;BASKETBALL;NBA;NYK;BLATT;Sport;Basketball;N...</td>\n",
       "      <td>Knicks hire Blatt in advisory role - Reuters F...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17083</th>\n",
       "      <td>https://www.union-bulletin.com/seattle_times/k...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Koprivica, Olejniczak power Florida State past...</td>\n",
       "      <td>TALLAHASSEE, Fla. (AP) — Balsa Koprivica score...</td>\n",
       "      <td>florida, koprivica, dominik olejniczak, sport,...</td>\n",
       "      <td>florida, koprivica, dominik olejniczak, sport,...</td>\n",
       "      <td>Koprivica, Olejniczak power Florida State past...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73687</th>\n",
       "      <td>https://cumberlink.com/sports/high-school/bask...</td>\n",
       "      <td>news</td>\n",
       "      <td>HS Girls Basketball Highlights: Cedar Cliff dr...</td>\n",
       "      <td>Cedar Cliff dropped Lower Dauphin 44-39 in OT ...</td>\n",
       "      <td>, prep-sports, cliff, dauphin, sport, trey, ot...</td>\n",
       "      <td>, prep-sports, cliff, dauphin, sport, trey, ot...</td>\n",
       "      <td>HS Girls Basketball Highlights: Cedar Cliff dr...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121709</th>\n",
       "      <td>https://madison.com/sports/baseball/profession...</td>\n",
       "      <td>news</td>\n",
       "      <td>Eric Sogard agrees to 1-year deal with Brewers...</td>\n",
       "      <td>Sogard, a utility player who played for Milwau...</td>\n",
       "      <td>, pro-sports, pro-baseball, mlb, pro baseball,...</td>\n",
       "      <td>, pro-sports, pro-baseball, mlb, pro baseball,...</td>\n",
       "      <td>Eric Sogard agrees to 1-year deal with Brewers...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129879</th>\n",
       "      <td>https://www.union-bulletin.com/seattle_times/w...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Washington men face early struggle before pull...</td>\n",
       "      <td>In each of their past two meetings, Seattle Un...</td>\n",
       "      <td>isaiah stewart, seattle university, quade gree...</td>\n",
       "      <td>isaiah stewart, seattle university, quade gree...</td>\n",
       "      <td>Washington men face early struggle before pull...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89946</th>\n",
       "      <td>https://www.sportingnews.com/ca/soccer/news/ba...</td>\n",
       "      <td>sports</td>\n",
       "      <td>'Bang up for another derby' - Rashford excited...</td>\n",
       "      <td>Marcus Rashford says he is 'bang up for anothe...</td>\n",
       "      <td>, Football, Premier League, News, Manchester C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'Bang up for another derby' - Rashford excited...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158393</th>\n",
       "      <td>https://www.skysports.com/football/news/11938/...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Carabao Cup hits and misses: Rashford strikes ...</td>\n",
       "      <td></td>\n",
       "      <td>Sky, Sports, capital, one , cup, carling,footb...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Carabao Cup hits and misses: Rashford strikes ...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72570</th>\n",
       "      <td>https://cumberlink.com/mid-penn-girls-basketba...</td>\n",
       "      <td>news</td>\n",
       "      <td>Mid-Penn Girls Basketball results for Dec. 17 ...</td>\n",
       "      <td>Commonwealth Division</td>\n",
       "      <td>, prep-sports, james buchanan, dauphin, altoon...</td>\n",
       "      <td>, prep-sports, james buchanan, dauphin, altoon...</td>\n",
       "      <td>Mid-Penn Girls Basketball results for Dec. 17 ...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93257</th>\n",
       "      <td>https://www.skysports.com/football/news/11095/...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Mesut Ozil vs China: US Secretary of State bac...</td>\n",
       "      <td>The United States secretary of state Mike Pomp...</td>\n",
       "      <td>News,Football,Sky,Sports,Sky Sports,sport,news</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mesut Ozil vs China: US Secretary of State bac...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url         dom_cat  \\\n",
       "73277   https://trib.com/sports/college/wyoming/footba...  financial_news   \n",
       "107446  https://www.reuters.com/article/us-basketball-...            news   \n",
       "17083   https://www.union-bulletin.com/seattle_times/k...          sports   \n",
       "73687   https://cumberlink.com/sports/high-school/bask...            news   \n",
       "121709  https://madison.com/sports/baseball/profession...            news   \n",
       "129879  https://www.union-bulletin.com/seattle_times/w...          sports   \n",
       "89946   https://www.sportingnews.com/ca/soccer/news/ba...          sports   \n",
       "158393  https://www.skysports.com/football/news/11938/...          sports   \n",
       "72570   https://cumberlink.com/mid-penn-girls-basketba...            news   \n",
       "93257   https://www.skysports.com/football/news/11095/...          sports   \n",
       "\n",
       "                                                    title  \\\n",
       "73277   2020 Wyoming signee: Braden Siders | Football ...   \n",
       "107446  \\n                Knicks hire Blatt in advisor...   \n",
       "17083   Koprivica, Olejniczak power Florida State past...   \n",
       "73687   HS Girls Basketball Highlights: Cedar Cliff dr...   \n",
       "121709  Eric Sogard agrees to 1-year deal with Brewers...   \n",
       "129879  Washington men face early struggle before pull...   \n",
       "89946   'Bang up for another derby' - Rashford excited...   \n",
       "158393  Carabao Cup hits and misses: Rashford strikes ...   \n",
       "72570   Mid-Penn Girls Basketball results for Dec. 17 ...   \n",
       "93257   Mesut Ozil vs China: US Secretary of State bac...   \n",
       "\n",
       "                                              description  \\\n",
       "73277   Wyoming football got a signed National Letter ...   \n",
       "107446  David Blatt, who led the Cleveland Cavaliers t...   \n",
       "17083   TALLAHASSEE, Fla. (AP) — Balsa Koprivica score...   \n",
       "73687   Cedar Cliff dropped Lower Dauphin 44-39 in OT ...   \n",
       "121709  Sogard, a utility player who played for Milwau...   \n",
       "129879  In each of their past two meetings, Seattle Un...   \n",
       "89946   Marcus Rashford says he is 'bang up for anothe...   \n",
       "158393                                                      \n",
       "72570                               Commonwealth Division   \n",
       "93257   The United States secretary of state Mike Pomp...   \n",
       "\n",
       "                                                 keywords  \\\n",
       "73277   , college-sports, wyoming cowboys football, re...   \n",
       "107446  US,BASKETBALL,NBA,NYK,BLATT,Sport,Basketball,N...   \n",
       "17083   florida, koprivica, dominik olejniczak, sport,...   \n",
       "73687   , prep-sports, cliff, dauphin, sport, trey, ot...   \n",
       "121709  , pro-sports, pro-baseball, mlb, pro baseball,...   \n",
       "129879  isaiah stewart, seattle university, quade gree...   \n",
       "89946   , Football, Premier League, News, Manchester C...   \n",
       "158393  Sky, Sports, capital, one , cup, carling,footb...   \n",
       "72570   , prep-sports, james buchanan, dauphin, altoon...   \n",
       "93257      News,Football,Sky,Sports,Sky Sports,sport,news   \n",
       "\n",
       "                                            news_keywords  \\\n",
       "73277   , college-sports, wyoming cowboys football, re...   \n",
       "107446  US;BASKETBALL;NBA;NYK;BLATT;Sport;Basketball;N...   \n",
       "17083   florida, koprivica, dominik olejniczak, sport,...   \n",
       "73687   , prep-sports, cliff, dauphin, sport, trey, ot...   \n",
       "121709  , pro-sports, pro-baseball, mlb, pro baseball,...   \n",
       "129879  isaiah stewart, seattle university, quade gree...   \n",
       "89946                                                 NaN   \n",
       "158393                                                NaN   \n",
       "72570   , prep-sports, james buchanan, dauphin, altoon...   \n",
       "93257                                                 NaN   \n",
       "\n",
       "                                                     text kw_label  \n",
       "73277   2020 Wyoming signee: Braden Siders | Football ...   sports  \n",
       "107446  Knicks hire Blatt in advisory role - Reuters F...   sports  \n",
       "17083   Koprivica, Olejniczak power Florida State past...   sports  \n",
       "73687   HS Girls Basketball Highlights: Cedar Cliff dr...   sports  \n",
       "121709  Eric Sogard agrees to 1-year deal with Brewers...   sports  \n",
       "129879  Washington men face early struggle before pull...   sports  \n",
       "89946   'Bang up for another derby' - Rashford excited...   sports  \n",
       "158393  Carabao Cup hits and misses: Rashford strikes ...   sports  \n",
       "72570   Mid-Penn Girls Basketball results for Dec. 17 ...   sports  \n",
       "93257   Mesut Ozil vs China: US Secretary of State bac...   sports  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sampling for inspection\n",
    "df_final[df_final.kw_label=='sports'].sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Basic Cleaning Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filling NAs\n",
    "df_final['text'] = df_final.text.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_printable(string):\n",
    "    \"\"\"\n",
    "    Clean up Non-ASCII characters.\n",
    "\n",
    "    Replace non-printable characters in a string.\n",
    "    \"\"\"\n",
    "    no_print_trans_table = ''.join([chr(i) for i in range(128)] + [' '] * 128)\n",
    "    \n",
    "    try:\n",
    "        trans = string.translate(no_print_trans_table)\n",
    "    except AttributeError:\n",
    "        trans = \"\"\n",
    "    return trans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Getting rid of non-ascii characters \n",
    "df_final['text'] = df_final[\"text\"].apply(lambda text: make_printable(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>dom_cat</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>keywords</th>\n",
       "      <th>news_keywords</th>\n",
       "      <th>text</th>\n",
       "      <th>kw_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://chicago.cbslocal.com/2019/12/18/man-st...</td>\n",
       "      <td>sports</td>\n",
       "      <td>Man Stabbed After His Granddaughter And Three ...</td>\n",
       "      <td>A man's granddaughter is one of four teens acc...</td>\n",
       "      <td>grandfather, granddaughter, teen, teens, home ...</td>\n",
       "      <td>grandfather, granddaughter, teen, teens, home ...</td>\n",
       "      <td>Man Stabbed After His Granddaughter And Three ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.releasewire.com/press-releases/rele...</td>\n",
       "      <td>news</td>\n",
       "      <td>Commence Industrial CRM Achieves Sales Optimiz...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Commence Industrial CRM Achieves Sales Optimiz...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.kark.com/news/national-news/1-dead...</td>\n",
       "      <td>news</td>\n",
       "      <td>1 dead, 2 injured in Oregon shopping center st...</td>\n",
       "      <td>Multiple people were stabbed at the Murray Hil...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1 dead, 2 injured in Oregon shopping center st...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.ii.co.uk/etfs/sg-wti-x2-daily-long...</td>\n",
       "      <td>financial_news</td>\n",
       "      <td>LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...</td>\n",
       "      <td>Real-time share price updates and latest news ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.peicanada.com/island_deaths/frances...</td>\n",
       "      <td>news</td>\n",
       "      <td>Frances Salsman | Island Deaths | peicanada.com</td>\n",
       "      <td>SALSMAN, Frances (Fran) Townsend At the Prince...</td>\n",
       "      <td>island_deaths, death_notices</td>\n",
       "      <td>island_deaths, death_notices</td>\n",
       "      <td>Frances Salsman | Island Deaths | peicanada.co...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163428</th>\n",
       "      <td>https://www.bakingbusiness.com/articles/49996-...</td>\n",
       "      <td>financial_news</td>\n",
       "      <td>Dairy Farmers of America merges ingredients bu...</td>\n",
       "      <td>KANSAS CITY â Dairy Farmers of America (D.F....</td>\n",
       "      <td>baking business, Baking &amp; Snack, baking and sn...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Dairy Farmers of America merges ingredients bu...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163429</th>\n",
       "      <td>https://www.fark.com/comments/10653287?utm_sou...</td>\n",
       "      <td>news</td>\n",
       "      <td>FARK.com:  (10653287) You can now watch the vi...</td>\n",
       "      <td>Humorous views on interesting, bizarre and amu...</td>\n",
       "      <td>Fark, Fark.com, Drew Curtis, News, Community N...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FARK.com: (10653287) You can now watch the vid...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163430</th>\n",
       "      <td>https://www.mygrandeprairienow.com/65144/local...</td>\n",
       "      <td>news</td>\n",
       "      <td>Local man hoping to make holidays special by h...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Local man hoping to make holidays special by h...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163431</th>\n",
       "      <td>https://www.thehollywoodgossip.com/slideshows/...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>90 Day Fiance Before The 90 Days: Trailer Teas...</td>\n",
       "      <td>90 Day Fiance: Before The 90 Days just release...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>90 Day Fiance Before The 90 Days: Trailer Teas...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163432</th>\n",
       "      <td>https://www.hometownsource.com/county_news_rev...</td>\n",
       "      <td>news</td>\n",
       "      <td>Kenneth \"Ken\" Edwin Holmstrom | Obituaries | h...</td>\n",
       "      <td>Kenneth Holmstrom of Harris passed away Saturd...</td>\n",
       "      <td>obituaries</td>\n",
       "      <td>obituaries</td>\n",
       "      <td>Kenneth \"Ken\" Edwin Holmstrom | Obituaries | h...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>163433 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url         dom_cat  \\\n",
       "0       https://chicago.cbslocal.com/2019/12/18/man-st...          sports   \n",
       "1       http://www.releasewire.com/press-releases/rele...            news   \n",
       "2       https://www.kark.com/news/national-news/1-dead...            news   \n",
       "3       https://www.ii.co.uk/etfs/sg-wti-x2-daily-long...  financial_news   \n",
       "4       http://www.peicanada.com/island_deaths/frances...            news   \n",
       "...                                                   ...             ...   \n",
       "163428  https://www.bakingbusiness.com/articles/49996-...  financial_news   \n",
       "163429  https://www.fark.com/comments/10653287?utm_sou...            news   \n",
       "163430  https://www.mygrandeprairienow.com/65144/local...            news   \n",
       "163431  https://www.thehollywoodgossip.com/slideshows/...   entertainment   \n",
       "163432  https://www.hometownsource.com/county_news_rev...            news   \n",
       "\n",
       "                                                    title  \\\n",
       "0       Man Stabbed After His Granddaughter And Three ...   \n",
       "1       Commence Industrial CRM Achieves Sales Optimiz...   \n",
       "2       1 dead, 2 injured in Oregon shopping center st...   \n",
       "3       LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...   \n",
       "4         Frances Salsman | Island Deaths | peicanada.com   \n",
       "...                                                   ...   \n",
       "163428  Dairy Farmers of America merges ingredients bu...   \n",
       "163429  FARK.com:  (10653287) You can now watch the vi...   \n",
       "163430  Local man hoping to make holidays special by h...   \n",
       "163431  90 Day Fiance Before The 90 Days: Trailer Teas...   \n",
       "163432  Kenneth \"Ken\" Edwin Holmstrom | Obituaries | h...   \n",
       "\n",
       "                                              description  \\\n",
       "0       A man's granddaughter is one of four teens acc...   \n",
       "1                                                     NaN   \n",
       "2       Multiple people were stabbed at the Murray Hil...   \n",
       "3       Real-time share price updates and latest news ...   \n",
       "4       SALSMAN, Frances (Fran) Townsend At the Prince...   \n",
       "...                                                   ...   \n",
       "163428  KANSAS CITY â Dairy Farmers of America (D.F....   \n",
       "163429  Humorous views on interesting, bizarre and amu...   \n",
       "163430                                                NaN   \n",
       "163431  90 Day Fiance: Before The 90 Days just release...   \n",
       "163432  Kenneth Holmstrom of Harris passed away Saturd...   \n",
       "\n",
       "                                                 keywords  \\\n",
       "0       grandfather, granddaughter, teen, teens, home ...   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3                                                     NaN   \n",
       "4                            island_deaths, death_notices   \n",
       "...                                                   ...   \n",
       "163428  baking business, Baking & Snack, baking and sn...   \n",
       "163429  Fark, Fark.com, Drew Curtis, News, Community N...   \n",
       "163430                                                NaN   \n",
       "163431                                                NaN   \n",
       "163432                                         obituaries   \n",
       "\n",
       "                                            news_keywords  \\\n",
       "0       grandfather, granddaughter, teen, teens, home ...   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3                                                     NaN   \n",
       "4                            island_deaths, death_notices   \n",
       "...                                                   ...   \n",
       "163428                                                NaN   \n",
       "163429                                                NaN   \n",
       "163430                                                NaN   \n",
       "163431                                                NaN   \n",
       "163432                                         obituaries   \n",
       "\n",
       "                                                     text kw_label  \n",
       "0       Man Stabbed After His Granddaughter And Three ...     None  \n",
       "1       Commence Industrial CRM Achieves Sales Optimiz...     None  \n",
       "2       1 dead, 2 injured in Oregon shopping center st...     None  \n",
       "3       LSE:SG30 ETF Share Price | SG WTI X2 Daily Lon...     None  \n",
       "4       Frances Salsman | Island Deaths | peicanada.co...     None  \n",
       "...                                                   ...      ...  \n",
       "163428  Dairy Farmers of America merges ingredients bu...     None  \n",
       "163429  FARK.com: (10653287) You can now watch the vid...     None  \n",
       "163430  Local man hoping to make holidays special by h...     None  \n",
       "163431  90 Day Fiance Before The 90 Days: Trailer Teas...     None  \n",
       "163432  Kenneth \"Ken\" Edwin Holmstrom | Obituaries | h...     None  \n",
       "\n",
       "[163433 rows x 8 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Veterans Ace nears completion | News | samessenger.com Thank you for Reading! Please log in, or sign up for a new account and purchase a subscription to continue reading. Sign Up Log In Purchase a Subscription Thank you for Reading! Please log in, or sign up for a new account and purchase a subscription to continue reading. Please purchase a subscription to continue reading. Your current subscription does not provide access to this content. Sign Up Log In Current e-Edition Subscribers Free access for current print subscribers Get Started Current Print Subscribers Free access for current print subscribers Get Started Franklin County's #1 Source for Local News Starting at $5.00 for 30 days Get Started View all rates Rate Price Duration One Month $5.00 for 30 days One Year $50.00 for 365 days You are the owner of this article. Edit Article Add New Article Subscribe | Sign In Home News Upbeat News Business Profiles Local History Sports Classifieds Employment Real Estate Property For Rent Cars and Vehicles Legal Notices Items for Sale Auctions, Garage, and Estate Sales Place an Ad Obituaries Editorial Letters to the Editor Editorials Business Directory Things to Do About Us Contact Us Subscription Services Submission Forms Archive e-Edition NEWSLETTERS Facebook Twitter YouTube Subscribe | Sign In Subscribe | Sign InDashboard Profile Saved items Logout Sign Up Log In Dashboard Logout My Account Dashboard Profile Saved items Logout Facebook Twitter YouTube News Sports Obituaries Editorial e-Edition The soon-to-be Veterans Memorial Ace Hardware amid Tuesday   s snowfall. TOM BENTON/Messenger Staff Facebook Twitter WhatsApp SMS Email Print Save top story Veterans Ace nears completion By TOM BENTON Messenger Staff Writer Dec 18, 2019 3 hrs ago Facebook Twitter WhatsApp SMS Email The soon-to-be Veterans Memorial Ace Hardware amid Tuesday   s snowfall. TOM BENTON/Messenger Staff Facebook Twitter WhatsApp SMS Email Print Save SWANTON VILLAGE     Another thing to look forward to in the new year: the Veterans Ace Memorial Hardware, which is finally nearing completion.Project developer Gordon Winters told the Messenger Tuesday the next step inside the heavily renovated building is painting walls.Winters said he expects shelving and products in place by the end of January.   We are almost there,    Winters said.Construction has been underway throughout the year.Winters told the Messenger this summer the project is taking more work than expected.   This needed way more work\""
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.sample(1).iloc[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "business_economy    1438\n",
       "sports              1199\n",
       "politics             479\n",
       "world                395\n",
       "entertainment        158\n",
       "Name: kw_label, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.kw_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>dom_cat</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>keywords</th>\n",
       "      <th>news_keywords</th>\n",
       "      <th>text</th>\n",
       "      <th>kw_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>82356</th>\n",
       "      <td>https://lmtribune.com/sports/prep-roundup-lust...</td>\n",
       "      <td>news</td>\n",
       "      <td>PREP ROUNDUP: Lustig tallies 44 as Colfax boys...</td>\n",
       "      <td>BOYS BASKETBALL</td>\n",
       "      <td>colfax, john lustig, sport, basketball, team, ...</td>\n",
       "      <td>colfax, john lustig, sport, basketball, team, ...</td>\n",
       "      <td>PREP ROUNDUP: Lustig tallies 44 as Colfax boys...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118479</th>\n",
       "      <td>https://www.onenewspage.com/video/20191206/125...</td>\n",
       "      <td>news</td>\n",
       "      <td>Celtic v Spartak Moscow | Eurofan - One News P...</td>\n",
       "      <td>Celtic v Spartak Moscow | Eurofan: Tom Deacon,...</td>\n",
       "      <td>Spartak,soccer,Ð¡Ð¿Ð°Ñ\\x80Ñ\\x82Ð°Ðº,Champions ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Celtic v Spartak Moscow | Eurofan - One News P...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69366</th>\n",
       "      <td>https://www.fordcountyrecord.com/sports/pbl-fr...</td>\n",
       "      <td>news</td>\n",
       "      <td>PBL freshman boys basketball wins 49-19 over A...</td>\n",
       "      <td>DANVILLE -- The Paxton-Buckley-Loda freshman b...</td>\n",
       "      <td>sports, prep-sports, boys-basketball</td>\n",
       "      <td>sports, prep-sports, boys-basketball</td>\n",
       "      <td>PBL freshman boys basketball wins 49-19 over A...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147669</th>\n",
       "      <td>https://www.onenewspage.com/n/Technology/1zkl5...</td>\n",
       "      <td>news</td>\n",
       "      <td>The Oscars of the video game industry just cel...</td>\n",
       "      <td>Â· *The Game Awards 2019 celebrated the year's...</td>\n",
       "      <td>oscars,video,game,industry,year,best,games,win...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Oscars of the video game industry just cel...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73526</th>\n",
       "      <td>https://www.insidenova.com/sports/wakefield-sw...</td>\n",
       "      <td>news</td>\n",
       "      <td>Wakefield sweeps rivalry games | Sports | insi...</td>\n",
       "      <td>Strong defense, especially in the second half,...</td>\n",
       "      <td>sports, arlington, wakefield, washington_lee, ...</td>\n",
       "      <td>sports, arlington, wakefield, washington_lee, ...</td>\n",
       "      <td>Wakefield sweeps rivalry games | Sports | insi...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90582</th>\n",
       "      <td>https://www.eacourier.com/sports/eagles-apache...</td>\n",
       "      <td>news</td>\n",
       "      <td>Eagles, Apaches set for EAC Tournament champio...</td>\n",
       "      <td>THATCHER — A sold-out gymnasium, close games, ...</td>\n",
       "      <td>game, sport, basketball, morenci, duncan, rebo...</td>\n",
       "      <td>game, sport, basketball, morenci, duncan, rebo...</td>\n",
       "      <td>Eagles, Apaches set for EAC Tournament champio...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69787</th>\n",
       "      <td>https://www.heraldbulletin.com/pacers-rally-la...</td>\n",
       "      <td>news</td>\n",
       "      <td>Pacers rally late to snap Lakers' road winning...</td>\n",
       "      <td>INDIANAPOLIS — Domantas Sabonis scored 26 poin...</td>\n",
       "      <td>indiana pacers, lakers, sport, basketball, mal...</td>\n",
       "      <td>indiana pacers, lakers, sport, basketball, mal...</td>\n",
       "      <td>Pacers rally late to snap Lakers' road winning...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17759</th>\n",
       "      <td>https://lacrossetribune.com/sports/local/preps...</td>\n",
       "      <td>news</td>\n",
       "      <td>High school basketball: La Crosse Aquinas girl...</td>\n",
       "      <td>The Aquinas High School girls basketball team ...</td>\n",
       "      <td>, taylor theusch, aquinas, courtney becker, sp...</td>\n",
       "      <td>, taylor theusch, aquinas, courtney becker, sp...</td>\n",
       "      <td>High school basketball: La Crosse Aquinas girl...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74320</th>\n",
       "      <td>https://www.idahostatejournal.com/sports/isu/t...</td>\n",
       "      <td>news</td>\n",
       "      <td>Tough to swallow: ISU men lose late lead, fall...</td>\n",
       "      <td>POCATELLO — A late scoring drought, missed fre...</td>\n",
       "      <td>tarik cool, free throw, overtime, sport, baske...</td>\n",
       "      <td>tarik cool, free throw, overtime, sport, baske...</td>\n",
       "      <td>Tough to swallow: ISU men lose late lead, fall...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79015</th>\n",
       "      <td>https://bleacherreport.com/articles/2867403-nf...</td>\n",
       "      <td>sports</td>\n",
       "      <td>NFL Playoffs 2019-20: Bracket Predictions, Odd...</td>\n",
       "      <td>Four tickets to the 2019-20  NFL  playoffs rem...</td>\n",
       "      <td>Football, NFL, NFL Playoffs, Breaking News</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NFL Playoffs 2019-20: Bracket Predictions, Odd...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url dom_cat  \\\n",
       "82356   https://lmtribune.com/sports/prep-roundup-lust...    news   \n",
       "118479  https://www.onenewspage.com/video/20191206/125...    news   \n",
       "69366   https://www.fordcountyrecord.com/sports/pbl-fr...    news   \n",
       "147669  https://www.onenewspage.com/n/Technology/1zkl5...    news   \n",
       "73526   https://www.insidenova.com/sports/wakefield-sw...    news   \n",
       "90582   https://www.eacourier.com/sports/eagles-apache...    news   \n",
       "69787   https://www.heraldbulletin.com/pacers-rally-la...    news   \n",
       "17759   https://lacrossetribune.com/sports/local/preps...    news   \n",
       "74320   https://www.idahostatejournal.com/sports/isu/t...    news   \n",
       "79015   https://bleacherreport.com/articles/2867403-nf...  sports   \n",
       "\n",
       "                                                    title  \\\n",
       "82356   PREP ROUNDUP: Lustig tallies 44 as Colfax boys...   \n",
       "118479  Celtic v Spartak Moscow | Eurofan - One News P...   \n",
       "69366   PBL freshman boys basketball wins 49-19 over A...   \n",
       "147669  The Oscars of the video game industry just cel...   \n",
       "73526   Wakefield sweeps rivalry games | Sports | insi...   \n",
       "90582   Eagles, Apaches set for EAC Tournament champio...   \n",
       "69787   Pacers rally late to snap Lakers' road winning...   \n",
       "17759   High school basketball: La Crosse Aquinas girl...   \n",
       "74320   Tough to swallow: ISU men lose late lead, fall...   \n",
       "79015   NFL Playoffs 2019-20: Bracket Predictions, Odd...   \n",
       "\n",
       "                                              description  \\\n",
       "82356                                     BOYS BASKETBALL   \n",
       "118479  Celtic v Spartak Moscow | Eurofan: Tom Deacon,...   \n",
       "69366   DANVILLE -- The Paxton-Buckley-Loda freshman b...   \n",
       "147669  Â· *The Game Awards 2019 celebrated the year's...   \n",
       "73526   Strong defense, especially in the second half,...   \n",
       "90582   THATCHER — A sold-out gymnasium, close games, ...   \n",
       "69787   INDIANAPOLIS — Domantas Sabonis scored 26 poin...   \n",
       "17759   The Aquinas High School girls basketball team ...   \n",
       "74320   POCATELLO — A late scoring drought, missed fre...   \n",
       "79015   Four tickets to the 2019-20  NFL  playoffs rem...   \n",
       "\n",
       "                                                 keywords  \\\n",
       "82356   colfax, john lustig, sport, basketball, team, ...   \n",
       "118479  Spartak,soccer,Ð¡Ð¿Ð°Ñ\\x80Ñ\\x82Ð°Ðº,Champions ...   \n",
       "69366                sports, prep-sports, boys-basketball   \n",
       "147669  oscars,video,game,industry,year,best,games,win...   \n",
       "73526   sports, arlington, wakefield, washington_lee, ...   \n",
       "90582   game, sport, basketball, morenci, duncan, rebo...   \n",
       "69787   indiana pacers, lakers, sport, basketball, mal...   \n",
       "17759   , taylor theusch, aquinas, courtney becker, sp...   \n",
       "74320   tarik cool, free throw, overtime, sport, baske...   \n",
       "79015          Football, NFL, NFL Playoffs, Breaking News   \n",
       "\n",
       "                                            news_keywords  \\\n",
       "82356   colfax, john lustig, sport, basketball, team, ...   \n",
       "118479                                                NaN   \n",
       "69366                sports, prep-sports, boys-basketball   \n",
       "147669                                                NaN   \n",
       "73526   sports, arlington, wakefield, washington_lee, ...   \n",
       "90582   game, sport, basketball, morenci, duncan, rebo...   \n",
       "69787   indiana pacers, lakers, sport, basketball, mal...   \n",
       "17759   , taylor theusch, aquinas, courtney becker, sp...   \n",
       "74320   tarik cool, free throw, overtime, sport, baske...   \n",
       "79015                                                 NaN   \n",
       "\n",
       "                                                     text kw_label  \n",
       "82356   PREP ROUNDUP: Lustig tallies 44 as Colfax boys...   sports  \n",
       "118479  Celtic v Spartak Moscow | Eurofan - One News P...   sports  \n",
       "69366   PBL freshman boys basketball wins 49-19 over A...   sports  \n",
       "147669  The Oscars of the video game industry just cel...   sports  \n",
       "73526   Wakefield sweeps rivalry games | Sports | insi...   sports  \n",
       "90582   Eagles, Apaches set for EAC Tournament champio...   sports  \n",
       "69787   Pacers rally late to snap Lakers' road winning...   sports  \n",
       "17759   High school basketball: La Crosse Aquinas girl...   sports  \n",
       "74320   Tough to swallow: ISU men lose late lead, fall...   sports  \n",
       "79015   NFL Playoffs 2019-20: Bracket Predictions, Odd...   sports  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[df_final.kw_label=='sports'].sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comments\n",
    "\n",
    "Our basic labeling seems to have worked.. \n",
    "\n",
    "## 4. Converting to BERT format and Saving files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = Path.home() / 'Data' / 'cc_news' / 'model_input'\n",
    "if not file_path.is_dir():\n",
    "    Path.mkdir(file_path, parents=True, exist_ok=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving to disk\n",
    "df_final.to_csv(file_path / 'all_data.csv', header=True, index=False, na_rep=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting our data into train and dev\n",
    "from sklearn.model_selection import train_test_split\n",
    "train, dev = train_test_split(df_final[~df_final.kw_label.isna()], test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "business_economy    1300\n",
       "sports              1085\n",
       "politics             424\n",
       "world                349\n",
       "entertainment        144\n",
       "Name: kw_label, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.kw_label.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basic file transformation for BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.DataFrame({'id': range(len(train.index)),\n",
    "                          'label': train[\"kw_label\"],\n",
    "                          'alpha': ['a'] * train.shape[0],\n",
    "                          'text': train[\"text\"]\n",
    "                          })\n",
    "    \n",
    "dev = pd.DataFrame({'id': range(len(dev.index)),\n",
    "                    'label': dev['kw_label'],\n",
    "                    'alpha': ['a'] * dev.shape[0],\n",
    "                    'text': dev[\"text\"]\n",
    "                    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv(file_path / 'train.tsv', sep='\\t', header=False, index=False)\n",
    "dev.to_csv(file_path / 'dev.tsv', sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have a good size corpus for training a BERT model and a very large \"unlabeled\" corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabeled = df_final[df_final.kw_label.isna()]\n",
    "unlabeled = pd.DataFrame({'id': range(len(unlabeled)),\n",
    "                            'label': None,\n",
    "                            'alpha': ['a'] * unlabeled.shape[0],\n",
    "                            'text': unlabeled['text']\n",
    "                          })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save unlabeled documents as test\n",
    "unlabeled.to_csv(file_path / 'test.tsv', sep='\\t', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End of Notebook\n",
    "\n",
    "And we reached the end of another notebook. Here, we\n",
    "\n",
    "1. Using the keywords from the meta data tags of the web pages, created class labels. \n",
    "2. Did some basic preprocessing for modeling \n",
    "3. Train and test split\n",
    "\n",
    "In the next step, \n",
    "1. We will prepare the corpus for BERT (Input features)\n",
    "2. Fine tune a BERT model using transfer learning\n",
    "3. And classify our documents into their final categories. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 cc_news",
   "language": "python",
   "name": "cc_news_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}