data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - Build tokenized docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aba27c570a3142f3bbf510eef9620cfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=195027), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "pattern = re.compile(r'[A-Za-z]+[\\w]*|[\\w]*[A-Za-z]+[\\w]*')   # obtains tokens with a least one alphebet\n",
    "stopwords = stopwords.words('English')\n",
    "docs = []\n",
    "token_set = set()\n",
    "\n",
    "with open('algorithm_corpus.txt', 'r') as file:\n",
    "    for line in tqdm_notebook(file.readlines()):\n",
    "        tokens = pattern.findall(line.lower())\n",
    "        tokens = [token for token in tokens]\n",
    "        \n",
    "        if 5 <= len(tokens):\n",
    "            docs.append(tokens)\n",
    "        \n",
    "        token_set.update(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - Map id to word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4723b9d4ab494f11a12c0c58b8f1efa7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "word_to_id = dict()\n",
    "id_to_word = dict()\n",
    "\n",
    "for i, token in tqdm_notebook(enumerate(token_set)):\n",
    "    word_to_id[token] = i\n",
    "    id_to_word[i] = token"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - Transform docs into id representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_in_id = [[word_to_id[token] for token in doc] for doc in docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of documents: 137266\n",
      "dictionary size: 31853\n",
      "word size: 3819131\n"
     ]
    }
   ],
   "source": [
    "print('number of documents: {}'.format(len(docs)))\n",
    "print('dictionary size: {}'.format(len(word_to_id)))\n",
    "print('word size: {}'.format(sum(len(doc) for doc in docs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Training data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CBOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "290da93325274e4e8b703964f582cfeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=137266), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "window_size = 5\n",
    "example_len = 5 * 2 + 1\n",
    "X_cbow = []\n",
    "Y_cbow = []\n",
    "\n",
    "for doc in tqdm_notebook(docs_in_id):\n",
    "    for i in range(len(doc) - example_len + 1):\n",
    "        X_cbow.extend(doc[i:i+5]+doc[i+6:i+11])\n",
    "        Y_cbow.append(doc[i+5])\n",
    "        \n",
    "X_cbow = np.array(X_cbow)\n",
    "X_cbow = X_cbow.reshape(-1, window_size * 2).T\n",
    "Y_cbow = np.array(Y_cbow)\n",
    "Y_cbow = Y_cbow.reshape(1, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Skip Gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7107303953bb486b946e21798e438f23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=137266), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "window_size = 10\n",
    "example_len = window_size * 2 + 1\n",
    "X_skip = []\n",
    "Y_skip = []\n",
    "\n",
    "for doc in tqdm_notebook(docs_in_id):\n",
    "    for i in range(len(doc) - example_len + 1):\n",
    "        for j in list(range(i, i+window_size)) + list(range(i+window_size+1, i+window_size*2+1)):\n",
    "            X_skip.append(doc[j])\n",
    "            Y_skip.append(doc[i+window_size])\n",
    "        \n",
    "X_skip = np.array(X_skip)\n",
    "X_skip = np.expand_dims(X_skip, axis=0)\n",
    "Y_skip = np.array(Y_skip)\n",
    "Y_skip = np.expand_dims(Y_skip, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 35750700)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_skip.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('pickles/X_skip.pkl', 'wb') as file:\n",
    "    pickle.dump(X_skip, file)\n",
    "    \n",
    "with open('pickles/Y_skip.pkl', 'wb') as file:\n",
    "    pickle.dump(Y_skip, file)\n",
    "    \n",
    "# with open('pickles/word_to_id.pkl', 'wb') as file:\n",
    "#     pickle.dump(word_to_id, file)\n",
    "    \n",
    "# with open('pickles/id_to_word.pkl', 'wb') as file:\n",
    "#     pickle.dump(id_to_word, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}