SpaCy Implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np # linear algebra\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import base64\n",
    "import string\n",
    "import re\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "stopwords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('research_paper.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Conference'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train, test = train_test_split(df, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Research title sample:', train['Title'].iloc[0])\n",
    "print('Conference of this paper:', train['Conference'].iloc[0])\n",
    "print('Training Data Shape:', train.shape)\n",
    "print('Testing Data Shape:', test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(8,4))\n",
    "sns.barplot(x = train['Conference'].unique(), y=train['Conference'].value_counts())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "punctuations = string.punctuation\n",
    "\n",
    "# Define function to cleanup text by removing personal pronouns, stopwords, and puncuation\n",
    "def cleanup_text(docs, logging=False):\n",
    "    texts = []\n",
    "    counter = 1\n",
    "    for doc in docs:\n",
    "        if counter % 1000 == 0 and logging:\n",
    "            print(\"Processed %d out of %d documents.\" % (counter, len(docs)))\n",
    "        counter += 1\n",
    "        doc = nlp(doc, disable=['parser', 'ner'])\n",
    "        tokens = [tok.lemma_.lower().strip() for tok in doc if tok.lemma_ != '-PRON-']\n",
    "        tokens = [tok for tok in tokens if tok not in stopwords and tok not in punctuations]\n",
    "        tokens = ' '.join(tokens)\n",
    "        texts.append(tokens)\n",
    "    return pd.Series(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INFO_text = [text for text in train[train['Conference'] == 'INFOCOM']['Title']]\n",
    "\n",
    "IS_text = [text for text in train[train['Conference'] == 'ISCAS']['Title']]\n",
    "\n",
    "INFO_clean = cleanup_text(INFO_text)\n",
    "INFO_clean = ' '.join(INFO_clean).split()\n",
    "\n",
    "IS_clean = cleanup_text(IS_text)\n",
    "IS_clean = ' '.join(IS_clean).split()\n",
    "\n",
    "INFO_counts = Counter(INFO_clean)\n",
    "IS_counts = Counter(IS_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INFO_common_words = [word[0] for word in INFO_counts.most_common(20)]\n",
    "INFO_common_counts = [word[1] for word in INFO_counts.most_common(20)]\n",
    "\n",
    "fig = plt.figure(figsize=(18,6))\n",
    "sns.barplot(x=INFO_common_words, y=INFO_common_counts)\n",
    "plt.title('Most Common Words used in the research papers for conference INFOCOM')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IS_common_words = [word[0] for word in IS_counts.most_common(20)]\n",
    "IS_common_counts = [word[1] for word in IS_counts.most_common(20)]\n",
    "\n",
    "fig = plt.figure(figsize=(18,6))\n",
    "sns.barplot(x=IS_common_words, y=IS_common_counts)\n",
    "plt.title('Most Common Words used in the research papers for conference ISCAS')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS\n",
    "from sklearn.metrics import accuracy_score\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "import re\n",
    "import spacy\n",
    "spacy.load('en')\n",
    "from spacy.lang.en import English\n",
    "parser = English()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "STOPLIST = set(stopwords.words('english') + list(ENGLISH_STOP_WORDS))\n",
    "SYMBOLS = \" \".join(string.punctuation).split(\" \") + [\"-\", \"...\", \"”\", \"”\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CleanTextTransformer(TransformerMixin):\n",
    "    def transform(self, X, **transform_params):\n",
    "        return [cleanText(text) for text in X]\n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        return self\n",
    "    def get_params(self, deep=True):\n",
    "        return {}\n",
    "    \n",
    "def cleanText(text):\n",
    "    text = text.strip().replace(\"\\n\", \" \").replace(\"\\r\", \" \")\n",
    "    text = text.lower()\n",
    "    return text\n",
    "\n",
    "def tokenizeText(sample):\n",
    "    tokens = parser(sample)\n",
    "    lemmas = []\n",
    "    for tok in tokens:\n",
    "        lemmas.append(tok.lemma_.lower().strip() if tok.lemma_ != \"-PRON-\" else tok.lower_)\n",
    "    tokens = lemmas\n",
    "    tokens = [tok for tok in tokens if tok not in STOPLIST]\n",
    "    tokens = [tok for tok in tokens if tok not in SYMBOLS]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def printNMostInformative(vectorizer, clf, N):\n",
    "    feature_names = vectorizer.get_feature_names()\n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    topClass1 = coefs_with_fns[:N]\n",
    "    topClass2 = coefs_with_fns[:-(N + 1):-1]\n",
    "    print(\"Class 1 best: \")\n",
    "    for feat in topClass1:\n",
    "        print(feat)\n",
    "    print(\"Class 2 best: \")\n",
    "    for feat in topClass2:\n",
    "        print(feat)\n",
    "\n",
    "vectorizer = CountVectorizer(tokenizer=tokenizeText, ngram_range=(1,1))\n",
    "clf = LinearSVC()\n",
    "pipe = Pipeline([('cleanText', CleanTextTransformer()), ('vectorizer', vectorizer), ('clf', clf)])\n",
    "\n",
    "# data\n",
    "train1 = train['Title'].tolist()\n",
    "labelsTrain1 = train['Conference'].tolist()\n",
    "\n",
    "test1 = test['Title'].tolist()\n",
    "labelsTest1 = test['Conference'].tolist()\n",
    "# train\n",
    "pipe.fit(train1, labelsTrain1)\n",
    "\n",
    "# test\n",
    "preds = pipe.predict(test1)\n",
    "print(\"accuracy:\", accuracy_score(labelsTest1, preds))\n",
    "print(\"Top 10 features used to predict: \")\n",
    "\n",
    "printNMostInformative(vectorizer, clf, 10)\n",
    "\n",
    "pipe = Pipeline([('cleanText', CleanTextTransformer()), ('vectorizer', vectorizer)])\n",
    "transform = pipe.fit_transform(train1, labelsTrain1)\n",
    "vocab = vectorizer.get_feature_names()\n",
    "\n",
    "for i in range(len(train1)):\n",
    "    s = \"\"\n",
    "    indexIntoVocab = transform.indices[transform.indptr[i]:transform.indptr[i+1]]\n",
    "    numOccurences = transform.data[transform.indptr[i]:transform.indptr[i+1]]\n",
    "    for idx, num in zip(indexIntoVocab, numOccurences):\n",
    "        s += str((vocab[idx], num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "print(metrics.classification_report(labelsTest1, preds, \n",
    "                                    target_names=df['Conference'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}