2020-04-24_TextProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from wordcloud import STOPWORDS\n",
    "import re\n",
    "import spacy\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Munirah's model for NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "munirahModel = 'D://Users/figohjs/Documents/NLP/StrPrioritization/Notebook/Model/NER_All_Labels_lg_2'\n",
    "nerModel2 = spacy.load(munirahModel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'D://Users/figohjs/Documents/NLP/StrPrioritization/Data/Interim/2020-02-04_ProcessedDF.csv'\n",
    "\n",
    "df = pd.read_csv(file)\n",
    "\n",
    "#create true positive col\n",
    "def truePositive(x):\n",
    "    if len([1 for i in range(1, 8) if x['analystRule' + str(i)] and x['rule' + str(i)]])!=0:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "#create false negative col\n",
    "def falseNegative(x):\n",
    "    if len([1 for i in range(1, 8) if x['analystRule' + str(i)] and not x['rule' + str(i)]])!=0:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "    \n",
    "def falsePositive(x):\n",
    "    if len([1 for i in range(1, 8) if not x['analystRule' + str(i)] and x['rule' + str(i)]])!=0:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "    \n",
    "def noFlag(x):\n",
    "    if not x['FN'] and not x['FP'] and not x['TP']:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "    \n",
    "df['FN'] = df.apply(falseNegative, axis = 1)\n",
    "\n",
    "df['TP'] = df.apply(truePositive, axis = 1)\n",
    "\n",
    "df['FP'] = df.apply(falsePositive, axis = 1)\n",
    "\n",
    "df['noFlag'] = df.apply(noFlag, axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "IndexToReportID_Dict = df['RECORD_ID'].to_dict()\n",
    "\n",
    "IndexToStrDesc_Dict = df['SUSPICION_DESC'].to_dict()\n",
    "\n",
    "ReportIDToIndex_Dict = {j:i for i,j in IndexToReportID_Dict.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processText(textArray):\n",
    "    start = datetime.now()\n",
    "    processedTextList = []\n",
    "    \n",
    "    #stopwords list\n",
    "    otherStopWords = ['also', 'via',  'within', ' even though ', 'on', 'please', 'still'\n",
    "                     'pada', 'dan', 'sahaja', 'pula', 'juga', \n",
    "                     'yang', 'terdapat', 'oleh', 'telah', 'adalah', 'sejak',\n",
    "                     'since', 'might', 'o/b', 'e.g', 'a/l', 'a/p', 'i.e']\n",
    "    stopWordList = list(STOPWORDS) + otherStopWords\n",
    "    \n",
    "    #regex - special characters + digits\n",
    "    regexSpecialChar = '\\/|\\,|\\:|\\(|\\)|\\?|\\*|\\-|\\[|\\]|\\.|\\+|\\&|\\=|\\d|\\%'\n",
    "    \n",
    "    for no, text in enumerate(textArray):\n",
    "        #remove stopwords\n",
    "        processedText = ' '.join([word for word in str(text).split(' ') if\n",
    "                                  word.lower() not in stopWordList])\n",
    "        \n",
    "        #denote ic to ic pattern\n",
    "        icPattern = '\\d{6}[-]\\d{1,2}[-]\\d{4}'\n",
    "#         icRep = 'ic'\n",
    "        icRep = ''\n",
    "        processedText = re.sub(icPattern, icRep, processedText)\n",
    "        \n",
    "        #denote date to date pattern\n",
    "        datePattern = '\\d{1,2}[-\\.\\/]\\d{1,2}[-\\.\\/]\\d{4}'\n",
    "#         dateRep = 'date'\n",
    "        dateRep = ''\n",
    "        processedText = re.sub(datePattern, dateRep, processedText)\n",
    "        \n",
    "        #denote phone to phone pattern \n",
    "        phonePattern = '01\\d{8}'\n",
    "#         phoneRep = 'phone'\n",
    "        phoneRep = ''\n",
    "        processedText = re.sub(phonePattern, phoneRep, processedText)\n",
    "        \n",
    "        #denote amount to amount pattern\n",
    "        amountPattern = 'RM *\\d+[\\,\\.]*\\d*[\\,\\.]*\\d*K*|RM *\\d+\\,\\d+\\.\\d+'\n",
    "#         amountRep = 'amount'\n",
    "        amountRep = ''\n",
    "        processedText = re.sub(amountPattern, amountRep, processedText)\n",
    "        \n",
    "        #remove special characters and digits\n",
    "        processedText = ' '.join([re.sub(regexSpecialChar, ' ', word) for word in processedText.split(' ')])\n",
    "        #remove additional spaces, leading and trailing spaces\n",
    "        processedText = re.sub('\\s+', ' ', processedText).strip()\n",
    "        \n",
    "        #denote name to named entity - need better ner model\n",
    "        doc = nerModel2(processedText)\n",
    "        nePattern = '|'.join([str(i) for i in doc.ents]) \n",
    "#         neRep = 'name'\n",
    "        neRep = ''\n",
    "        try:\n",
    "            if nePattern!='':\n",
    "#                 processedText = re.sub(nePattern, neRep, processedText) + ', %s'%no\n",
    "                processedText = re.sub(nePattern, neRep, processedText) \n",
    "        except:\n",
    "            print(processedText + '\\n')\n",
    "            print(no)\n",
    "            \n",
    "#         #remove lagging number ', \\d'\n",
    "#         laggingPattern = ', \\d+$'\n",
    "#         processedText = re.sub(laggingPattern, '', processedText)\n",
    "        processedTextList.append(processedText)        \n",
    "    \n",
    "    end = datetime.now()\n",
    "    print(\"Time taken in minutes: %s\" %round((end - start).seconds/60, 2))\n",
    "    return processedTextList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken in minutes: 5.27\n"
     ]
    }
   ],
   "source": [
    "#process text for all 6418 records\n",
    "processedStr = processText(df['SUSPICION_DESC'].values)\n",
    "#take True Positive as bad str - 418\n",
    "\n",
    "#store indices for bad str\n",
    "badIndex = list(df[df['TP']].index)\n",
    "\n",
    "#assign cleaned desc to a column\n",
    "df['SUSPICION_DESC_CLEAN'] = processedStr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save to csv\n",
    "filename = 'D://Users/figohjs/Documents/NLP/StrPrioritization/Data/Interim/%s_ProcessedDF.csv'%datetime.now().strftime('%Y-%m-%d')\n",
    "df.to_csv(filename, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TestEnv",
   "language": "python",
   "name": "testenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}