preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I/O handling and profiling\n",
    "import json\n",
    "import pickle\n",
    "import timeit\n",
    "\n",
    "# feature extraction\n",
    "import re\n",
    "import nltk\n",
    "import nltk.sentiment\n",
    "import snowballstemmer\n",
    "\n",
    "# feature representation\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hand-crafted corpus of stopwords relevant to Yelp dataset\n",
    "f = open(\"stopwords_list.dat\")\n",
    "stopwords = set([word.strip().lower() for word in f.readlines()])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of special symbols to remove during preprocessing\n",
    "f = open(\"spl_symbols.dat\")\n",
    "spl_symb = set([word.strip().lower() for word in f.readlines()])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "devfile = \"data/dev.json\"\n",
    "trainfile = \"data/train.json\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing Text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessing steps relevant to the task:\n",
    "* convert to lowercase: handle sparsity due to different capitalization\n",
    "* stem words: reduce vocabulary size <sup>#</sup>\n",
    "* substitute numbers with a text signifying that there was a number: exact value of numbers often not important <sup>#</sup>\n",
    "* remove stopwords\n",
    "* remove punctuation marks\n",
    "* mark words following negation with a suffic NEG uptil the first punctuation mark: differentiate positive sentences from negation\n",
    "* remove infrequent words: reduce vocabulary size, Zipf's Law (remove words occuring $<3$ times)\n",
    "* use n-grams: add local sequential information to features (make use of bigrams/trigrams)\n",
    "\n",
    "<sup>#</sup> cause decrease in accuracy hence not part of final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeral = re.compile('\\d+\\.*\\d*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = snowballstemmer.stemmer(\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_spl_symbols(word):\n",
    "    word = [char for char in word if char not in spl_symb]\n",
    "    word = \"\".join(word)\n",
    "    return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    # tokenize\n",
    "    text = nltk.word_tokenize(text.lower())\n",
    "    \n",
    "    # stemming\n",
    "    # text = stemmer.stemWords(text)\n",
    "    \n",
    "    # substitute all numbers with a common text\n",
    "    # text = [numeral.sub(\"_numeral_\",word) for word in text]\n",
    "    \n",
    "    # remove stopwords\n",
    "    text = [word for word in text if word not in stopwords]\n",
    "    \n",
    "    # propogate negation\n",
    "    text = nltk.sentiment.util.mark_negation(text)\n",
    "    \n",
    "    # remove unnecessary symbols\n",
    "    text = list(map(remove_spl_symbols, text))\n",
    "    \n",
    "    # remove punctuations ONLY after removing negation\n",
    "    text = [word for word in text if len(word)>0]\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(tokenizer=preprocess, min_df=3, lowercase=False,\n",
    "                             sublinear_tf=True, strip_accents=\"ascii\", ngram_range=(1,2), max_features = 312000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing and Vectorizing train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = []\n",
    "train_labels = []\n",
    "f = open(trainfile,\"r\")\n",
    "while(True):\n",
    "    s = f.readline()\n",
    "    if (s==\"\"):\n",
    "        break\n",
    "    json_dump = json.loads(s)    \n",
    "    train_data.append(json_dump[\"review\"])\n",
    "    train_labels.append(json_dump[\"ratings\"])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train examples, Number of features: (100000, 312000)\n",
      "156.76875230800215\n"
     ]
    }
   ],
   "source": [
    "start = timeit.default_timer()\n",
    "feature_vector = vectorizer.fit_transform(train_data)\n",
    "print(\"Number of train examples, Number of features:\",feature_vector.shape)\n",
    "print(timeit.default_timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(feature_vector,open(\"train_vector.pickle\", \"wb\"))\n",
    "pickle.dump(vectorizer,open(\"vectorizer.pickle\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_data\n",
    "del feature_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing and Vectorizing dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data = []\n",
    "dev_labels = []\n",
    "f = open(devfile,\"r\")\n",
    "while(True):\n",
    "    s = f.readline()\n",
    "    if (s==\"\"):\n",
    "        break\n",
    "    json_dump = json.loads(s)    \n",
    "    dev_data.append(json_dump[\"review\"])\n",
    "    dev_labels.append(json_dump[\"ratings\"])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 312000)\n",
      "7.1499977499988745\n"
     ]
    }
   ],
   "source": [
    "start = timeit.default_timer()\n",
    "dev_feature_vector = vectorizer.transform(dev_data)\n",
    "print(dev_feature_vector.shape)\n",
    "print(timeit.default_timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(dev_feature_vector,open(\"dev_vector.pickle\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"feature_names.txt\",\"w\")\n",
    "for i in vectorizer.get_feature_names():\n",
    "    f.write(i+\"\\n\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}