get_windows.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import spacy\n",
    "from gensim import corpora, models\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from utils import preprocess, get_windows"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "MIN_COUNTS = 20\n",
    "MAX_COUNTS = 1800\n",
    "# words with count < MIN_COUNTS\n",
    "# and count > MAX_COUNTS\n",
    "# will be removed\n",
    "\n",
    "MIN_LENGTH = 15\n",
    "# minimum document length \n",
    "# (number of words)\n",
    "# after preprocessing\n",
    "\n",
    "# half the size of the context around a word\n",
    "HALF_WINDOW_SIZE = 5\n",
    "# it must be that 2*HALF_WINDOW_SIZE < MIN_LENGTH"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Load NLP model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Load dataset"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "rvData = pd.read_csv(\"reviews.csv\")\n",
    "# pre-processing\n",
    "rvData['cleanReviews'] = rvData['reviews'].str.replace(\"[^a-zA-Z#]\", \" \") # remove special characters\n",
    "docs = rvData['reviews']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "1867"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of documents\n",
    "len(docs)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [],
   "source": [
    "# store an index with a document\n",
    "docs = [(i, doc) for i, doc in enumerate(docs)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Preprocess dataset and create windows"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1867/1867 [00:33<00:00, 56.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of removed short documents: 517\n",
      "total number of tokens: 69224\n",
      "number of tokens to be removed: 18242\n",
      "number of additionally removed short documents: 200\n",
      "total number of tokens: 48585\n",
      "\n",
      "minimum word count number: 16\n",
      "this number can be less than MIN_COUNTS because of document removal\n"
     ]
    }
   ],
   "source": [
    "encoded_docs, decoder, word_counts = preprocess(\n",
    "    docs, nlp, MIN_LENGTH, MIN_COUNTS, MAX_COUNTS\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "# new ids will be created for the documents.\n",
    "# create a way of restoring initial ids:\n",
    "doc_decoder = {i: doc_id for i, (doc_id, doc) in enumerate(encoded_docs)}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1150it [00:00, 9826.09it/s]\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "# new ids are created here\n",
    "for index, (_, doc) in tqdm(enumerate(encoded_docs)):\n",
    "    windows = get_windows(doc, HALF_WINDOW_SIZE)\n",
    "    # index represents id of a document, \n",
    "    # windows is a list of (word, window around this word),\n",
    "    # where word is in the document\n",
    "    data += [[index, w[0]] + w[1] for w in windows]\n",
    "\n",
    "data = np.array(data, dtype='int64')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "12"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a row in 'data' contains:\n",
    "# id of a document, id of a word in this document, a window around this word\n",
    "# 1 + 1 + 10\n",
    "data.shape[1]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "data": {
      "text/plain": "48585"
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of windows (equals to the total number of tokens)\n",
    "data.shape[0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Get unigram distribution"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [],
   "source": [
    "word_counts = np.array(word_counts)\n",
    "unigram_distribution = word_counts/sum(word_counts)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Prepare word vectors"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 17.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vocab_size = len(decoder)\n",
    "embedding_dim = 50\n",
    "\n",
    "# train a skip-gram word2vec model\n",
    "texts = [[str(j) for j in doc] for i, doc in encoded_docs]\n",
    "model = models.Word2Vec(texts, size=embedding_dim, window=5, workers=4, sg=1, negative=15, iter=70)\n",
    "model.init_sims(replace=True)\n",
    "\n",
    "word_vectors = np.zeros((vocab_size, embedding_dim)).astype('float32')\n",
    "for i in decoder:\n",
    "    word_vectors[i] = model.wv[str(i)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "522"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of unique words\n",
    "vocab_size"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Prepare initialization for document weights"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [],
   "source": [
    "texts = [[decoder[j] for j in doc] for i, doc in encoded_docs]\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_topics = 25\n",
    "lda = models.LdaModel(corpus, alpha=0.9, id2word=dictionary, num_topics=n_topics)\n",
    "corpus_lda = lda[corpus]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic 0 : play family hacker get share account online item like way\n",
      "topic 1 : play family get hacker item account online cheater share like\n",
      "topic 2 : play family online get hacker item like share way Souls\n",
      "topic 3 : play family hacker online get share account character way like\n",
      "topic 4 : play family get online share hacker good like cause cheater\n",
      "topic 5 : play get family item hacker share cheater like way account\n",
      "topic 6 : play get family cheater way share player online item hacker\n",
      "topic 7 : play family share hacker account get item time cheater player\n",
      "topic 8 : play get family item hacker like account share player cheat\n",
      "topic 9 : play way account item share hacker family get people cheat\n",
      "topic 10 : play share hacker family get account like cheater online cause\n",
      "topic 11 : play hacker share family time account item get online player\n",
      "topic 12 : play get family item hacker like player way account share\n",
      "topic 13 : play get family hacker good player item like share way\n",
      "topic 14 : play family hacker get account share online cheater like way\n",
      "topic 15 : play family get share hacker account online way item like\n",
      "topic 16 : play get family hacker account like item way share player\n",
      "topic 17 : play hacker online share family get account player like item\n",
      "topic 18 : play get hacker share family item account way like online\n",
      "topic 19 : play get hacker family like account item cause player share\n",
      "topic 20 : play family get player hacker way item account time people\n",
      "topic 21 : play get family hacker account online share cheater player like\n",
      "topic 22 : play get like hacker share way item Dark family online\n",
      "topic 23 : play get family hacker online share like item account time\n",
      "topic 24 : play get family account online hacker share item cheater way\n"
     ]
    }
   ],
   "source": [
    "for i, topics in lda.show_topics(n_topics, formatted=False):\n",
    "    print('topic', i, ':', ' '.join([t for t, _ in topics]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1150/1150 [00:00<00:00, 1838.14it/s]\n"
     ]
    }
   ],
   "source": [
    "doc_weights_init = np.zeros((len(corpus_lda), n_topics))\n",
    "for i in tqdm(range(len(corpus_lda))):\n",
    "    topics = corpus_lda[i]\n",
    "    for j, prob in topics:\n",
    "        doc_weights_init[i, j] = prob"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Save data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [],
   "source": [
    "np.save('data.npy', data)\n",
    "np.save('word_vectors.npy', word_vectors)\n",
    "np.save('unigram_distribution.npy', unigram_distribution)\n",
    "np.save('decoder.npy', decoder)\n",
    "np.save('doc_decoder.npy', doc_decoder)\n",
    "np.save('doc_weights_init.npy', doc_weights_init)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}