vocabulary-embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate intial word embedding for headlines and description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The embedding is limited to a fixed vocabulary size (`vocab_size`) but\n",
    "a vocabulary of all the words that appeared in the data is built."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 40000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower = False # dont lower case the text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# read tokenized headlines and descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "#  Articles taken from Telam site\n",
    "with open('data-es/tn/sports50k-telam.pkl', 'rb') as fp:\n",
    "    heads, desc, keywords = pickle.load(fp) # keywords are not used in this project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if lower:\n",
    "    heads = [h.lower() for h in heads]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if lower:\n",
    "    desc = [h.lower() for h in desc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Instituto, con un cambio, para visitar el lunes a Independiente Rivadavia\\n'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i=0\n",
    "heads[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'El defensor uruguayo Sergio Rodr\\xc3\\xadguez no se recuper\\xc3\\xb3 de un cuadro de gastroenteritis y no estar\\xc3\\xa1 en el equipo de Instituto de C\\xc3\\xb3rdoba que visitar\\xc3\\xa1 el lunes a las 21.30 a Independiente Rivadavia Mendoza por la quinta fecha del Torneo de Primera B Nacional, dej\\xc3\\xa1ndole su lugar en la zaga central a Facundo Ag\\xc3\\xbcero.'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49998, 47189)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(heads),len(set(heads))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49998, 49057)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(desc),len(set(desc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# build vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from itertools import chain\n",
    "def get_vocab(lst):\n",
    "    vocabcount = Counter(w for txt in lst for w in txt.split())\n",
    "    vocab = map(lambda x: x[0], sorted(vocabcount.items(), key=lambda x: -x[1]))\n",
    "    return vocab, vocabcount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab, vocabcount = get_vocab(heads+desc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "most popular tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['de', 'la', 'en', 'el', 'a', 'del', 'y', 'que', 'por', 'con', 'El', 'los', 'se', 'al', 'un', 'hoy', 'para', 'su', 'fecha', 'ante', 'Primera', 'las', 'argentino', 'una', 'partido']\n",
      "... 75657\n"
     ]
    }
   ],
   "source": [
    "print vocab[:25]\n",
    "print '...',len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEaCAYAAAAL7cBuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4HNX1//H3kSxLlovcq2zLxgWDMcbdmGKDARNwDAQINXQHEiDf0ElCQgmB/JKQhAAhDr3jUEKPaW4YAy5gXHE3tjG49y6d3x8zShahspJ2NVrp83oePdqdmZ09d2Z2z957Z+aauyMiIlJUWtQBiIhI9aQEISIixVKCEBGRYilBiIhIsZQgRESkWEoQIiJSLCWIFGBmt5rZU+VY3s2sS/j4QTO7JUFxdDCz7WaWHj6fYGaXJmLd4freMrMLErW+mPVWeBskuoxlvFfsfnvMzH4bPj7SzL6oihiSwcyGmtmqciz/321uZuea2dsJjudbx3EFXp/wmKqrOlEHIMnl7pfHs5yZLQcudfd3S1nXl0CDRMRlZrcCXdz9vJj1n5iIdRcV7zaortx9MtA96jii4O5PA08neJ1xH8dmlgcsAzLcfX+yYqquVIOoRixQLfeJmenHhKQ8HcflUy2/jFKBmV1kZq/FPF9kZv+Keb7SzHqHjw83s2lmtiX8f3jMchPM7E4zmwLsBDqbWSczm2hm28zsHaB5GbFcb2ZrzOwrM7u4yLzYpormZva6mW02s41mNtnM0szsSaAD8FpY9b7BzPLCJo9LzOxL4P2YabEfsgPM7BMz22pmr5hZ0/C9vtOsYGbLzWy4mY0AfgH8MHy/WTHborBpIc3MfmVmK8xsrZk9YWY54bzCOC4wsy/NbL2Z/bKU7RO7DYaa2SozuzZc7xozu6i07Qt0NLMp4f5428z+uz/MbJCZfRhu01lmNjRm3kVmNj983VIz+3G8+63Ict/aluF2vM7MPg+PqefNLCtm/slm9lkY04dm1itm3o1mtjqM6QszO7aE9zzJzD4N9+vKsMZXOK/U7W9m9cJtvsnM5gH9S9u4ZnacmS0Iy3IfYDHzLjSzD8LHZmZ/DvfbVjObbWY9Y97zT+HxssXMPginlXkch8fdXcUdx8Ck8P/m8FgdHBtT+PqyPt93lHT8VHvurr8K/AGdgc0ESbYtsAJYFTNvUzivafj4fIImvbPD583CZScAXwIHh/MzgKnAPUAmcBSwDXiqhDhGAN8APYH6wDOAEzTfADwG/DZ8fBfwYPgeGcCRgIXzlgPDY9abF67niXC99WKm1YmJfXXMe79YGCcwtHB7xKzzv+8B3Fq0TOH6Lg0fXwwsDrdlA+Al4Mkisf0zjOtQYA/Qo4RtFLsNhgL7gdvDbfA9gsTcpITXTgCWAN3C95oA3B3OawdsCNeRBhwXPm8Rzj8JOIDgC+/o8H36VGC/fWtbhtvxE4LjrikwH7g8nHcYsBYYCKQDF4TLZxI0U60E2sZsxwNKKPdQ4JCwXL3CWE+JZ/sDdwOTw9jaA3MocizEvE9zguP79HB//DzcP4XHwYXAB+HjE4AZQONwm/YA2oTz7g/3Tbuw3IeHZS6MtaLH8beWLSameD7fxR4/qfCnGkQFuftSggO7N8GX+DjgKzM7kODLYLK7FxB8SSxy9yfdfb+7PwssAEbGrO4xd5/rQRtnG4JfXLe4+x53nwS8RsnOBB519znuvoPgi7ck+8L1d3T3fe4+2cOjuBS3uvsOd99VwvwnY977FuBMq2DnXxHnAve4+1J33w7cDJxl36693Obuu9x9FjCL4IsqHvuA28Nt8CawndLb+B9194XhNhhLsM8BzgPedPc33b3A3d8BphMkDNz9DXdf4oGJwNsESRnKt9+Kc6+7f+XuGwmOj8KYRgP/cPeP3T3f3R8n+PIeBOQTfGkeZGYZ7r7c3ZcUt3J3n+Dus8NyfQ48S3Bcxypp+58J3OnuG919JXBvKeX4HjDX3V9w933AX4CvS1h2H9AQOJDgh818d19jQbPsxcDP3H11WO4P3X1PzGuTdRzH8/ku6fip9pQgKmciwS+to8LHEwg+REeHz+F/tYtYKwh+6RRaGfO4LbApPFBjly9J2yKvL23ZPxD8Kn87bPK4qZRli4utrPkrCH4FJqIKXXS7rSD4hdYqZlrsF8lO4u9A3xAm43hfW9L7dATOCJtyNpvZZuAIgiSMmZ1oZh9Z0Jy3meDLsHDblGe/lTema4vE1J6g1rAY+D+CZLTWzJ4zs7bFrdzMBprZeDNbZ2ZbgMv57n4tKYbylO1by4Y/WIo95tz9feA+gtrCWjMbY2aNwriyCH6plyRZx3E8n++KHqeRU4KonMIEcWT4eCLfTRBfEXxoY3UgqNIWiv0VvwZoYmb1iyxfkjUEXwBlLuvu29z9WnfvDHwfuCamDbqkmkRZNYyi770PWA/sALILZ4S/xlqUY71Ft1sHgqaHb8p4XVVaSfDLs3HMX313v9vMMgmaKv4ItHL3xsCb/K99Pe79VoGY7iwSU3b4yxZ3f8bdjyDYtg78voT1PAO8CrR39xyCpkkrYdmiylO2by1rZlbktd/i7ve6e1/gIIJmm+sJjrfdBM15Jb60jJhLOo7Le5wWvn51McumHCWIypkIDAPqufsqgnbXEUAz4NNwmTeBbmZ2jpnVMbMfEhzcrxe3QndfQdBMcZuZ1TWzI/h2dbWoscCFZnaQmWUDvylpwbDzskv4IdxC0ORQEM7+hqC9v7zOi3nv24EX3D0fWAhkhZ2dGcCvCJo3Cn0D5FnJZ209C/zcgg77BsDvgOeL/PKP2lPASDM7wczSzSzLgg7lXKAuQXnXAfvN7ETg+JjXxr3fyumfwOVhDcDMrH64DxqaWXczOyZMXruBXfxv/xfVENjo7rvNbABwTjliGAvcbGZNwm1xVSnLvgEcbGanhc2HVwOti1vQzPqH5cog+AGyGygIm3IfAe4xs7bhvhgcljNeJR3H6wi2UUmfjXJ9vlONEkQluPtCgvbryeHzrcBSYEp4cOHuG4CTgWsJOjBvAE529/WlrPocgk7GjQRfHE+UEsNbBO227xM0H71fynq7Au+GMU8FHnD38eG8u4Bfhc0S15WyjqKeJOhQ/Zqgmn91GNcW4CfAQwS/pnYAsWc1FZ7xtcHMZhaz3kfCdU8iOA99N6V/0VS5sH19FMEZWesIfr1fD6S5+zaCbTGWoNPyHIJf5IWvLc9+K09M04HLCJpiNoXrvjCcnUnQgbyeYH+1JOjbKc5PgNvNbBvw67Ac8bqNoJllGUG/y5OlxLseOCOMawPBMTqlhMUbESTATeH6NxA0mwJcB8wGphF8bn5P+b7fSjqOdwJ3AlPCz8agIvFX5POdMgrPYBERqZXMbALBWUsPRR1LdaMahIiIFEsJQkREiqUmJhERKZZqECIiUiwlCBERKVZK3tnQzEYCIxs2bHhZt27dog5HRCSlzJgxY727tyhruZTug+jXr59Pnz496jBERFKKmc1w935lLacmJhERKZYShIiIFEsJQkREipWSCcLMRprZmC1btkQdiohIjZWSCcLdX3P30Tk5OVGHIiJSY6VkghARkeRTghARkWIpQYiISLGUIEREpFgpmSB0FpOISPKlZILQWUwiIsmXkglCRESSTwlCRESKpQQhIiLFUoIQEZFiKUGIiEixUnJEuUK79uYze1XFTnXNzEijXeN61M9M6U0gIpI01ebb0cyGAncAc4Hn3H1CWa9ZvG47I+/7oFLv2yQ7g/ZNs2nfJJvcpvXIbZJN+yb1aN80m3aN65GVkV6p9YuIpKqkJggzewQ4GVjr7j1jpo8A/gqkAw+5+92AA9uBLGBVPOvv2CybB35U5qh5xdq5L59Vm3aycuMuVm3aybw1W3ln3jfszS/41nItG2aGCSRIGrlN6tG+STbtm2bTJieLOulqpRORmimpY1Kb2VEEX/pPFCYIM0sHFgLHESSCacDZwAJ3LzCzVsA97n5uWetP9JjUBQXO2m17WLlpJys3/i95rAwTyZotuyiI2VzpaUbrRln0aNOQo7q14OhuLejYrH7C4hERSYZ4x6ROag3C3SeZWV6RyQOAxe6+FMDMngNGufu8cP4mILOkdZrZaGA0QIcOHRIab1qa0Toni9Y5WfTPa/qd+fvyC/h6y+4geWzayapNu1i5cSczv9zMu/PXApDXLJuju7Xg6O4tGNS5Gdl1q00rnohIuUTx7dUOWBnzfBUw0MxOA04AGgP3lfRidx8DjIGgBpHEOL8jIz0taG5qmv2decvX72DiwnVMXLiO56ev5PGpK6ibnsaATk3/mzC6tmyAmVVlyCIiFVZtft66+0vAS/Esa2YjgZFdunRJblDlkNe8PnnN63PB4Xns3pfP9OWbmLhwLRMXruPON+dz55vzaZOTxQkHt+aCw/Po1FxNUSJSvSW1DwIgbGJ6PaYPYjBwq7ufED6/GcDd7yrvuhPdB5EsX23exaSF65jwxTreW/AN+wucY7q35OIjOnH4Ac1UqxCRKhVvH0QUCaIOQSf1scBqgk7qc9x9bjnWWViDuGzRokUJjzmZ1m7bzdMffcnTH69g/fa9dG/VkIuG5HHKYe10Sq2IVIlqkSDM7FlgKNAc+Ab4jbs/bGbfA/5CcJrrI+5+Z0XWnyo1iOLs2Z/Pa7PW8MgHy5i3ZitNsjM4Z2AHzh+UR+ucrKjDE5EarFokiGRJ5RpEUe7OJ8s28siUZbw97xvSzfjeIW24+IhO9G7fOOrwRKQGqtEJolAq1yCKs3LjTh7/cDnPT1vJtj37OaxDYy4e0okRPVuToQvyRCRBlCBS2PY9+3lxxioenbKM5Rt20iYni1G923FU1+b0zWtCZh31VYhIxdXoBFGTmphKU1DgjP9iLY99uJypSzawv8DJykhjYKdmHNm1OYMPaMaBrRuRnqazoEQkfjU6QRSqqTWI4mzfs5+Pl25g8qL1TFq0jqXrdgDQMLMOffOa0D+vKScc3IouLRtGHKmIVHdKEDXc6s27mLZsI58s38i0ZRtZtHY7aQZnD+jANcd1o1mDEu9WIiK1XI1OELWliak81m3bw/3jF/PURyuol5HOVcd24eIhnXS3WRH5jhqdIArV5hpESRav3c7v3pzP+wvW8vPh3fjZ8K5RhyQi1Uy8CUI/L2uYLi0b8MiF/TmpVxsemLCYlRt3Rh2SiKSolEwQZjbSzMZs2VKx4UZrg19+rwdpZvz2jXllLywiUoyUTBDu/pq7j87JyYk6lGqrbeN6XHlMF8bN/YaJC9dFHY6IpKCUTBASn0uP7ERes2xue3Uue/cXlP0CEZEYShA1WGaddH7z/YNZun4HN730Odv37I86JBFJIUoQNdyw7i25clgXXv50NSf8eRIfLFofdUgikiJSMkGok7p8rjuhOy9cfjiZGWmc9/DH3Pe+rh0RkbKlZIJQJ3X59e3YhDevPpLTDmvHH99eyB/HfUEqXwMjIslXbcakluTLykjnj2ccSt06adw3fjGrN+/i4LaNaNagLif2bKMR7UTkW5Qgapm0NON3px5C/cw6PDJlGS9/uhqAf01fxcMX9KdeXSUJEQnoVhu1WH6Bs33PfsbN/ZobX/ycwZ2b8ffz+pJTLyPq0EQkieK91YZqELVYepqRUy+DM/u1JyPduGbsLPrc8Q69cnPo0DSbnHoZXHB4Hge0aBB1qCISgZSsQehursnx+arNvD33Gz5auoF12/fwzdbd1E1P4/5z+3BEl+aYaWAikZpAd3OVSlu5cSeXPD6Nhd9sp1n9upzRrz03juiuRCGS4tTEJJXWvmk2L15xOC/NXM3kRet5cOISWjfK5MIhnaIOTUSqQEpeByFVp2FW0A8x5vy+DO/RijvemM+TU5dTUJC6NU8RiY8ShMQlLc348w8P5fADmnHLK3O56tlPdaGdSA2nBCFxa5iVwRMXD+Bnx3bljdlrGDf3m6hDEpEkUoKQcjEzrjqmCwe2bsjtr81lxYYdUYckIkmiBCHlVic9jd+ddghbdu3juD9P4rp/zWLqkg1qchKpYapVgjCz+mY23cxOjjoWKV2fDk14/7qh/KBPO8bN+Zqz//kRpz7wIdOXb4w6NBFJkKQmCDN7xMzWmtmcItNHmNkXZrbYzG6KmXUjMDaZMUnitGqUxV2n9WLar4Zz56k9+XrLbs74x1TufW8Ra7ftjjo8EamkZNcgHgNGxE4ws3TgfuBE4CDgbDM7yMyOA+YBa5MckyRYVkY65w7syPvXHc2oQ9tyzzsLGfi797jv/UVqdhJJYUm9UM7dJ5lZXpHJA4DF7r4UwMyeA0YBDYD6BEljl5m96e7fGUjZzEYDowE6dOiQvOCl3LLr1uHPP+zN+YM78tiHK/jj2wtZvXkXt4/qSUZ6tWrNFJE4RHEldTtgZczzVcBAd78SwMwuBNYXlxwA3H0MMAaCW20kN1QpLzOjb8em9OnQhI5Ns7lv/GJmr97Cn8/sTddWDaMOT0TKodr9rHP3x9z99dKW0ZCj1Z+Zcd0J3XnwvL58tXk3J/3tA+59b5GuwBZJIVEkiNVA+5jnueG0uGnI0dQxomdrxv3fURx3UCvueWchvx+3IOqQRCROUSSIaUBXM+tkZnWBs4BXy7MC1SBSS4uGmdx39mGc1b89YyYtZdl6XVwnkgqSfZrrs8BUoLuZrTKzS9x9P3AlMA6YD4x197nlWa9qEKnHzLjm+G5kpKVx04uf6wpskRRQZoIwszPMrGH4+Fdm9pKZ9Yln5e5+tru3cfcMd89194fD6W+6ezd3P8Dd7yxv0KpBpKaWDbO48cQDmb16C1c8NZN9+cWehyAi1UQ8NYhb3H2bmR0BDAceBv6e3LBKpxpE6rrkiE786YxDmbdmK/1++y5XPDWDddv2RB2WiBQjngSRH/4/CRjj7m8AdZMXktR0Jx7Shocv6MexPVry3oK1HP2H8TwwYTGbduyNOjQRiVHmkKNm9jrBWUbHAX2AXcAn7n5o8sMrMSaNSV1DLFm3nV+9PIepSzfQqXl9/n5eHw5s3SjqsERqtISNSW1m2QS3y5jt7ovMrA1wiLu/nZhQK05jUtcM7s5LM1dz/QuzqJOWxi++dyA/GpxHWprGvhZJhngTRJlNTO6+k+D+SEeEk/YD+tkuCWNm/KBvLu9eczQtGmZy62vzePiDZezZn1/2i0UkaeI5i+k3BHdZvTmclAE8lcygyqKzmGqmzi0aMPmGYQzIa8qdb85nyN3jeXHGKnbvU6IQiUI8TUyfAYcBM939sHDa5+7eqwriK5WamGqm3fvyeWvOGh6csJQvvtlGbpN63DjiQEYe2jbq0ERqhIQ1MQF7PcgiHq64fmWDEylNVkY6px6Wy6tXDeHu0w7BDK569lNG/GUSc1ar1ihSVeJJEGPN7B9AYzO7DHgX+Gdywyqdmphqh8w66Zw1oAPvXzuUG0Z0Z/32vfzokU+48415GmdCpAqU2cQEEA7mczxgwDh3fyfZgcVDTUy1y5zVW7jzjflMXbqBA1s3pFduDlcd05X2TbOjDk0kpSTyNNdOwBp33x0+rwe0cvfliQi0MpQgap/8AufBiUuYvnwjHy3dyO79+fRsm8PgA5ox+qjONG+QGXWIItVeIhPEdOBwd98bPq8LTHH3/gmJtBKUIGq3VZt28tLM1UxauI7pKzbRMKsOPxnahdP75tKioRKFSEkSmSA+c/feRabNivJK6kJKEFJo6pIN/P4/C/hs5WYy66Tx/07vxaje7aIOS6RaSuRZTOvM7PsxKx4FrK9McJWlTmopavABzfj3T4fw758OoVWjLH7+/Gfc+94ivtq8K+rQRFJWPDWIA4CngbYEndQrgR+5++Lkh1c61SCkOOu27eHSx6cxa1XwA+LIrs2585RD6NBMndkikMAmppgVNgBw9+2VjC1hlCCkNHO/2sK4OV/zwIQlODDq0LbceOKBtGqUFXVoIpFKZB9EJvADIA+oUzjd3W+vZIyVpgQh8Vi1aSePTlnO0x+vYO/+Aob3aMVVx3TlkFyNJyK1U7wJok5ZCwCvAFuAGYBGdpGUk9skm1tOPoiz+rfniakrePnT1bw97xsOatOI3512CL3bN446RJFqKZ4axBx371lF8ZSLahBSEVt372PstJU8OHEpW3fv46phXejfqSm92zcmKyM96vBEki6RNYgPzewQd5+dgLgSImbAoKhDkRTUKCuDS4/szImHtOHasZ/xp3cWAtCyYSaPXtSfg9uq6UkE4qtBzAO6AMsImpgMcN3NVWqKddv28OGS9fzy5Tls37OfUb3bcs+ZvUnXgEVSQyWyBnFiAuIRqbZaNMxkVO92HNm1BQ+MX8xDHyzji6+3cdmRnRneoxU52RlRhygSiTIThLuvADCzloDOD5Qaq2n9uvzypB60bJTJ4x+u4Np/zSI9zRjeoyXnD8pj8AHNVKuQWiWeJqbvA38iuFBuLdARmO/uByc/vNKpiUmSJb/A+WzlJl6btYZnPv6SvfkFtGqUychebfnZ8K40zFKtQlJXIpuY7gAGAe+6+2FmNgw4r7IBilRn6WlG345N6duxKTedeCDvzv+Gf3+6moc+WMbcr7Zy7fHd6JfXNOowRZIqnnsx7XP3DUCamaW5+3igzMwjUlNkZaRzcq+2PHRBf+4YdTDz1mzl9Aencs3zn7F0XbW5sYBIwsVTg9gc3mZjEvC0ma0FdiQ3LJHq6fzBeZzetz1/fPsLnpi6nFdmfcXwHi059bB2nHBwa8zURyE1Rzx9EPWBXQS1jXOBHODpsFaRuEDMegA/A5oD77n738t6jfogJErrtu3h7xOW8Mpnq9mwYy8dmmZzWp92/KBPrka5k2otIfdiMrN0gr6HYRUM4hHgZGBt7NXYZjYC+CuQDjzk7nfHzEsDnnD3Mvs5lCCkOsgvcF7//Cue+mgF05ZvAuC0Pu245rhu5DZRopDqJyGd1O6eb2YFZpbj7hUZfOEx4D7giZjA0oH7geOAVcA0M3vV3eeFZ0xdATxZgfcSiUR6mjGqdztG9W7HknXbeX7aSv45eSkvzVzNsQe25GfDu9IrV/d7ktQTTxPTK8BhwDvE9D24+9VxvYFZHvB6YQ3CzAYDt7r7CeHzm8P13RXzmjfc/aQS1jcaGA3QoUOHvitWrIgnDJEqtXLjTp755EuemrqCbXv2M6RLM277fk+6tGwQdWgiCb3d9wXFTXf3x+MMJI9vJ4jTgRHufmn4/HxgIPACcBqQCXzu7veXtW41MUl1t2XXPh6avJTHpixn+979fK9nG342vCvdWjWMOjSpxRJ2HUS8iaCy3H0CMCGeZXWzPkkVOfUyuPb47pw/qCMPTlzKE1OX88bsNQzIa8ofzuhFx2b1ow5RpERlXgdhZl3N7AUzm2dmSwv/KvGeq4H2Mc9zw2lxc/fX3H10To7uuimpoWWjLH498iA+uPEYfj68GzO/3MTJf/uAx6YsY/ue/VGHJ1KseC6UexT4O7AfGEbQ4fxUJd5zGtDVzDqZWV3gLODV8qzAzEaa2ZgtWyrSby4SndY5WfxseFfeueZoDmrTiFtfm8dR/288D01eyt79BVGHJ/It8SSIeu7+HkF/xQp3vxUotgO5KDN7FpgKdDezVWZ2ibvvB64ExgHzgbHuPrc8QasGIamuU/P6PDd6EC9eMZguLRvw2zfmc/lTM9i1Nz/q0ET+K55O6g+BIwg6kd8naA662927Jz+8EmMq7IO4bNGiRVGFIZIwT360gl+/Mocm2XW59vhunNG3PXXrxPP7TaT8EnkWU3+CX/qNCW7c1wj4g7t/lIhAK0NnMUlNMn35Ru56awEzVmyiRcNMfjr0AH40OI803WJcEixhCSJmhdnuvrPSkSWQEoTUNO7OxIXruPe9Rcz8cjOHtm/M1cd04dgeraIOTWqQeBNEPGcxDQ6HHV0QPj/UzB5IQIwVpk5qqanMjKHdW/LiFYdz92mHsGXnXi55fDp/fXcRBQXx/ZgTSZR4mpg+Bk4HXnX3w8Jpc2LvrRQV1SCkptuzP5+bX5zNS5+uZmj3Ftx56iG0a1wv6rAkxSWsBgHg7iuLTIr0VAvVIKS2yKyTzp/OPJQ7TunJh4s3cOyfJnD/+MXsy9cpsZJ88SSIlWZ2OOBmlmFm1xF0WkdGp7lKbWJmnD+oI+9fdzRDu7XkD+O+YPg9E3l/wTfE24coUhHxJIjLgZ8C7YCvgN7hcxGpQrlNsnnw/L48eF5f0sy4+LHpXDt2Fl9v2R11aFJDxX0WU3WkPgiprfbuL+Bv7y/ivvGLyUhP45wBHbhi6AG0apQVdWiSAhJ5FlNnM3vNzNaZ2Voze8XMOicmzIpRH4TUdnXrpHHt8d1575qjOaV3Wx6fupyj/t94nv54hZqdJGHiOYvpI4IBfp4NJ50FXOXuA5McW5lUgxAJLFm3nV+8NJuPl22kd/vGXH9Cd4Z0aR51WFJNJfIspmx3f9Ld94d/TwGqx4pUIwe0aMCzlw3irtMOYc2WXZz70Mfc+upctuzcF3VoksLiSRBvmdlNZpZnZh3N7AbgTTNramZNkx2giMQnLc04e0AHJlw3jB8N7sjjU5dz/F8mMnHhuqhDkxQVTxPTslJmu7tH1h+hJiaRks1YsZGrnvmUNVt3c/aADtx04oE0ysqIOiypBhJ+L6bqRHdzFYnP1t37uOvN+Tw3bSVtGmVxy8kHMaJna8x0A8DaLJF3c80CfkJwy28HJgMPunvkJ1+rBiESnxkrNnHdv2axbP0OjjuoFXeM6knrHHUl1laJ7KR+AjgY+BtwX/j4ycqFJyJVqW/HJvzn/47k6mO6MGnhOk7+22RmrNgUdVhSzcWTIHq6+yXuPj78u4wgSYhICsmsk841x3fnlSuHUDc9jbPHfMTY6UVvsybyP/EkiJlmNqjwiZkNBNSuI5KiDmzdiJd/OoQ+HRtzwwuf84dxC3RxnRQrngTRF/jQzJab2XKCMab7m9lsM/s8qdGJSFK0apTFoxcO4MSerbl//BJGPzmDtVsj71aUaiaeTuqOpc139xUJjSgOOotJJDHyC5y/T1jMX99bRP3MOlx7fHfOH1TqR15qgGQMOdqSmCuo3f3LioeXGDqLSSQxFnzbZ3Y9AAAS30lEQVS9lVv+PYdpyzcxuHMz/n5eHxpn1406LEmSRN6s7/tmtghYBkwElgNvVTpCEak2DmzdiOdGD+b6E7ozdekGhv5xAi/NXKW+iVounj6IO4BBwEJ37wQcC3yU1KhEpMqlpxk/HdaFf/90CJ2a1+easbO44qmZbN65N+rQJCLxJIh97r4BSDOzNHcfD5RZNRGR1NS7fWNeuPxwrj62K+Pmfc2Iv0xmyuL1UYclEYgnQWw2swbAJOBpM/srsCO5YYlIlNLTjGuO68bYHw9mf0EB5z70MTe8MIuCAjU51SbxJIhRwE7g58B/gCXAyGQGJSLVQ/+8prx3zVCO7taCsdNX8cMxU1m5cWfUYUkVScmb9RXSWUwiVcPdefiDZfz2jfkAvHH1ERzcNifiqKSiEnkvpipjZqeY2T/N7HkzOz7qeEQkYGZcemRn/nF+XwBOuX8KnyzbGHFUkmxJTxBm9kg4lvWcItNHmNkXZrbYzG4CcPd/h/d6uhz4YbJjE5HyOeHg1jx96UAy66Rz5j+m8tDkpVGHJElUFTWIx4ARsRPMLJ1gnOsTgYOAs83soJhFfhXOF5FqZkiX5rx/3dH0aNOI374xn58//xn78wuiDkuSoE5JM8xsNsH4D9+ZRTCSXK943sDdJ5lZXpHJA4DF7r40fK/ngFFmNh+4G3jL3WeWENdoYDRAhw4d4glBRBKsZcMsXr/qCK5+9lNe/nQ1S9dt5/GLB+jq6xqmxAQBnJzE920HxN5neBUwELgKGA7kmFkXd3+w6AvdfQwwBoJO6iTGKCKlSE8z7j+3Dz3eX8Qf317IkLvf58Hz+3Jk1xZRhyYJUmKCiOImfO5+L3BvWcvF3Kwv+UGJSKmuPKYrXVo25IqnZ3D+w59wzsAO3HlKTw1rWgPEcy+mQWY2zcy2m9leM8s3s62VfN/VQPuY57nhtLi4+2vuPjonR6fZiVQHI3q2Ztovh3NYh8Y88/GXnHTvB2zcoVt0pLp4OqnvA84GFgH1gEupfAfyNKCrmXUys7rAWcCr8b7YzEaa2ZgtW7ZUMgwRSZTmDTJ56YrDuWhIHvPWbKXPHe/w6qyvog5LKiGus5jcfTGQ7u757v4oRc5KKo2ZPUswyFB3M1tlZpe4+37gSmAcMB8Y6+5z412nahAi1ZOZ8ZuRB/PgeX0AuPrZT7nxhc91V9gUFc+AQZMIOo4fAr4G1gAXuvuhyQ+vxJg0YJBINTfvq618797JALRomMmk64dRr256xFEJJPZK6vPD5a4kuElfe+C0yoVXOapBiFR/B7VtxII7RpDbpB7rtu2hx6//w+xVahZOJfEkiFPcfbe7b3X329z9GpJ7CmyZ1AchkhqyMtKZfMMwzhkYXLM08r4PeHTKsoijknjFkyAuKGbahQmOo1xUgxBJHWbG7049hD+eEbRK3/baPG54YVbEUUk8SruS+mzgHKCTmcWeYdQI0F26RKRcTu+bS/+8Jhz9hwmMnb6Kr7fu4YmLB0QdlpSixE5qM+sIdALuAm6KmbUN+Dw8EykS6qQWSV1bdu7j0NvfBqBeRjof3XwsOdkZEUdVu1S6k9rdV7j7BHcfDCwAGoZ/q6JMDmFsamISSVE52RksuGMEnZrXZ9e+fA69/W0+/XJT1GFJMeK5kvoM4BPgDOBM4GMzOz3ZgYlIzZWVkc7464Zybth5feoDH/LMx19GHJUUVdrN+gr9Cujv7msBzKwF8C7wQjIDE5Ga785TD6FPhyZc+69Z/OLl2azYuIObT+wRdVgSiucsprTC5BDaEOfrkkanuYrUHD/om8urVw4B4B8Tl3L+wx/ryutqIp4v+v+Y2Tgzu9DMLgTeAN5KblilUx+ESM3SK7cxU28+BoDJi9bT77fvsntffsRRSZkJwt2vB/4B9Ar/xrj7DckOTERqlzY59Vhwxwia1q/Lhh17OfCW/7Bu256ow6rV4umk/r27v+Tu14R/L5vZ76siOBGpXbIy0vnkF8fSu31jAPrf+S5ffL0t4qhqr3iamI4rZtqJiQ5ERASgTnoaL//kcE49rB0AJ/xlEtOX69rcKJSYIMzsinBc6u5m9nnM3zLg86oLsdjY1EktUoOZGX/+YW+uPa4bAKc/OJW/vquLYqtaaVdS5wBNKOZKanevFum8X79+Pn369KjDEJEkGjf3a3785AwATurVhnvOPJTMOrpteGXEeyV1meNBVGdKECK1w4oNOzj6DxOAYGyJ1686glaNsqINKoUlcjwIEZFIdWxWn1m/Pp68Ztms27aHgb97jzmr1cScbEoQIpIScrIzmHD9MM7slwvAyX/7gA+XrI84qppNCUJEUsrvf9CLq47pAsA5//yYN2eviTiimislE4TOYhKpvcyMa4/vzp/CAYh+8vRM3lKSSIqUTBC61YaI/KBvLo9e2B+AK56eyb+mr4w4oponJROEiAjAsANb8uhFQZK4/oXPeXTKMt3oL4GUIEQkpQ3r3pKnLhkIBONd3/2fBRQUKEkkghKEiKS8I7o259nLBtG+aT3+MXEpPx/7WdQh1QhKECJSIww+oBn3nd2Hnu0a8eqsrzjl/ilRh5TylCBEpMY4tH1j7jq1F4M6NeOzlZs5558fsXz9jqjDSllKECJSoxySm8Ntow5mSJdmfLhkA3e/tYDJi9ZFHVZKqjYJwsw6m9nDZqaxrkWkUrq1asijFw6gbU4W/5n7NTe9OJstO/dFHVbKSWqCMLNHzGytmc0pMn2EmX1hZovN7CYAd1/q7pckMx4RqT3q1knjgxuPYfRRnVm9eReH3v62bs1RTsmuQTwGjIidYGbpwP0Egw4dBJxtZgclOQ4RqYXS0oxLj+zEL7/XA4AHxi/hnncWsnd/QcSRpYakJgh3nwQUHTtiALA4rDHsBZ4DRsW7TjMbbWbTzWz6unVqVxSR0rVsmMVFQ/Lo0aYRM1Zs4t73FvH5qs1Rh5USouiDaAfEXhO/CmhnZs3M7EHgMDO7uaQXu/sYd+/n7v1atGiR7FhFpAaok57GWz87kudGDwLg8qdmcunj0yKOqvqrE3UAhdx9A3B5PMua2UhgZJcuXZIblIjUKD3aNOKiIXl8smwj785fy778AjLSq825OtVOFFtmNdA+5nluOC1uulmfiFRE3Tpp/GbkwZzZL/gKOvCW/zDgznfZultnOBUnihrENKCrmXUiSAxnAeeUZwWqQYhIZZzUqw2bdu7li6+38dacr1mzeTeNWmdEHVa1k+zTXJ8FpgLdzWyVmV3i7vuBK4FxwHxgrLvPLc96VYMQkcpo3iCT/xvejbMGdADg4semceyfJvDsJ19GHFn1ktQahLufXcL0N4E3K7pe1SBEJBEO69CYM/vlsmNvPpMWrmPCF2s5O0waApbK907v16+fT58+PeowRKQGOPWBKezam89lR3YmPc0Y1r0lOdk1s9nJzGa4e7+ylkvJ7nsNOSoiidaucT0WfL2Na/81i/97/jOe/Gh51CFFLiUThPogRCTR7jmzN5OuH8ak64eRlZHG1t37ow4pctXmOggRkSjVrZNGh2bZANTLSGfmik08NHkpALlNshnRs3WU4UUiJROEOqlFJJk6t2jA9BWbmL5iEwBmMO+2EdSrmx5xZFUrJROEu78GvNavX7/Loo5FRGqesT8ezM69QRPTMx9/yV1vLWD3vnwlCBGR2i49zWiYFZzBVPh/Ty28A6wShIhIKTLrBOfynPbAFOqE922qk2b89pSeHN6leZShJV1KJgj1QYhIVTmya3PO6t/+vzWIAnde+ewrZn65SQmiOlIfhIhUlZaNsrj7B73++9zDBLE3P3UvMo5XSl4HISISFTMjI93Yl1/z+yRSsgYhIhKljPQ0vtywk6lLNnxrenbddHrl5mBmEUWWWCmZINQHISJRapJdlzdmr+GN2Wu+M+/50YMY2LlZBFElXkomCPVBiEiUnv/xIFZu3PWtacvW7+AXL8+uUbfoSMkEISISpdwm2eQ2yf7WtMbhnV/zC2pO34Q6qUVEEqBOWtDvsK8Gnd2kBCEikgCFF9HlFyhBiIhIjMIaxP4alCDUByEikgB10oME8cD4xfxr+spSl+2f15TrTuheFWFVSkrWIDSinIhUNy0aZPL9Q9vSomFmqcstW7+D56Z9WUVRVY7GpBYRqUK3/HsOr3/+FZ/++vjIYqjRY1KLiKSq9DQjVboplCBERKqQGRSkSIZQghARqULpZhSkSNO+EoSISBVKSzPylSBERKSoNFMfhIiIFCMthfogqs2FcmZWH3gA2AtMcPenIw5JRCThgrOYUiNBJLUGYWaPmNlaM5tTZPoIM/vCzBab2U3h5NOAF9z9MuD7yYxLRCQqFjYxpcI1aMmuQTwG3Ac8UTjBzNKB+4HjgFXANDN7FcgFZoeL5Sc5LhGRSKSHo81dM3YWlRl37of92yd9YKKkJgh3n2RmeUUmDwAWu/tSADN7DhhFkCxygc8opWZjZqOB0QAdOnRIfNAiIkl0WIfG5DXLZvqKjZVaz7E9WiUoopJF0QfRDoi9k9UqYCBwL3CfmZ0EvFbSi919DDAGglttJDFOEZGEO6pbCyZcPyzqMOJSbTqp3X0HcFE8y2pMahGR5IviNNfVQPuY57nhtLi5+2vuPjonJyehgYmIyP9EkSCmAV3NrJOZ1QXOAl4tzwp0u28RkeRL9mmuzwJTge5mtsrMLnH3/cCVwDhgPjDW3eeWZ72qQYiIJF+yz2I6u4TpbwJvVnS96oMQEUm+lLzVhmoQIiLJl5IJQn0QIiLJl5IJQjUIEZHkS+kxqc1sHbAZKFqVyImZVtLj5sD6BIUSu97KLlvS/OKmF51W2vNklL22lrukuCq6bKLKXtK8ml7uos+r+z6vTLmLTqtouTu6e4tS5gfcPaX/gDGlTSvl8fRkxlDRZUuaX1Y5y3qejLLX1nJX17KXNK+mlzvV9nllyl1GWRNe7pRsYiqiuNtyvBbH42THUNFlS5pfVjnLep6MstfWcpd3vVVV9rK2SyJUx3IXfV7d93llyl10WlLLndJNTJVhZtPdvV/UcUShtpZd5a59amvZE1XumlCDqKgxUQcQodpadpW79qmtZU9IuWttDUJEREpXm2sQIiJSCiUIEREplhKEiIgUSwkiZGb1zexxM/unmZ0bdTxVxcw6m9nDZvZC1LFUNTM7Jdzfz5vZ8VHHU1XMrIeZPWhmL5jZFVHHU5XCz/l0Mzs56liqkpkNNbPJ4X4fGu/ranSCMLNHzGytmc0pMn2EmX1hZovN7KZw8mnAC+5+GfD9Kg82gcpTbndf6u6XRBNp4pWz7P8O9/flwA+jiDdRylnu+e5+OXAmMCSKeBOlnJ9xgBuBsVUbZXKUs+wObAeyCIZ5jk+irjKsjn/AUUAfYE7MtHRgCdAZqAvMAg4CbgZ6h8s8E3XsVVXumPkvRB13hGX/E9An6tirstwEP4LeAs6JOvaqKjdwHMEAZRcCJ0cdexWXPS2c3wp4Ot73qNE1CHefBGwsMnkAsNiDX857geeAUQRZNTdcJqW3SznLXaOUp+wW+D3wlrvPrOpYE6m8+9zdX3X3E4GUbk4tZ7mHAoOAc4DLzKzWfM7dvSCcvwnIjPc9kjpgUDXVDlgZ83wVMBC4F7jPzE4ieZfqR6nYcptZM+BO4DAzu9nd74okuuQqaZ9fBQwHcsysi7s/GEVwSVTSPh9K0KSaSSUG7qrGii23u18JYGYXAutjvjRrkpL2+WnACUBj4L54V1YbE0Sx3H0HcFHUcVQ1d99A0AZf67j7vQQ/DGoVd58ATIg4jMi4+2NRx1DV3P0l4KXyvi6lq1gVtBpoH/M8N5xW09XWckPtLbvKHagt5YYEl702JohpQFcz62RmdQk6rV6NOKaqUFvLDbW37Cp37So3JLjsNTpBmNmzwFSgu5mtMrNL3H0/cCUwDpgPjHX3uVHGmWi1tdxQe8uucteuckPVlF036xMRkWLV6BqEiIhUnBKEiIgUSwlCRESKpQQhIiLFUoIQEZFiKUGIiEixlCBEqpCZ3Wpm10Udh0g8lCBEKii8G6w+Q1Jj6eAWKQczywsHY3kCmAM8HI5QNtfMbotZbrmZ3WZmM81stpkdWMy6LjOzt8ysXlWWQSReupurSPl1BS5w94/MrKm7bzSzdOA9M+vl7p+Hy6139z5m9hPgOuDSwhWY2ZUEA9ic4u57qrwEInFQDUKk/Fa4+0fh4zPNbCbwKXAwwehdhQpvrzwDyIuZ/iPgROB0JQepzpQgRMpvB4CZdSKoGRzr7r2ANwjG/C1U+OWfz7dr67MJEkYuItWYEoRIxTUiSBZbzKwVQa0gHp8CPwZeNbO2yQpOpLKUIEQqyN1nEXzZLwCeAaaU47UfENQ+3jCz5smJUKRydLtvEREplmoQIiJSLCUIEREplhKEiIgUSwlCRESKpQQhIiLFUoIQEZFiKUGIiEixlCBERKRY/x+k8er1yupuAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.plot([vocabcount[w] for w in vocab]);\n",
    "plt.gca().set_xscale(\"log\", nonposx='clip')\n",
    "plt.gca().set_yscale(\"log\", nonposy='clip')\n",
    "plt.title('word distribution in headlines and discription')\n",
    "plt.xlabel('rank')\n",
    "plt.ylabel('total appearances');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zipf's law states that given some corpus of natural language utterances, the frequency of any word is inversely proportional to its rank in the frequency table. Thus the most frequent word will occur approximately twice as often as the second most frequent word, three times as often as the third most frequent word, and so on."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Index words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty = 0 # RNN mask of no data\n",
    "eos = 1  # end of sentence\n",
    "start_idx = eos+1 # first real word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_idx(vocab, vocabcount):\n",
    "    word2idx = dict((word, idx+start_idx) for idx,word in enumerate(vocab))\n",
    "    word2idx['<empty>'] = empty\n",
    "    word2idx['<eos>'] = eos\n",
    "    \n",
    "    idx2word = dict((idx,word) for word,idx in word2idx.iteritems())\n",
    "\n",
    "    return word2idx, idx2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx, idx2word = get_idx(vocab, vocabcount)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000653"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get glove number of symbols\n",
    "glove_name = \"data-es/glove/SBW-vectors-100-1MM.txt\"\n",
    "import commands\n",
    "cmd_result =commands.getstatusoutput('wc -l '+glove_name)\n",
    "glove_n_symbols = int(cmd_result[1].split()[0])\n",
    "glove_n_symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   get glove glove into an array\n",
    "import numpy as np\n",
    "glove_index_dict = {}\n",
    "glove_embedding_weights = np.empty((glove_n_symbols, embedding_dim))\n",
    "globale_scale=.1\n",
    "with open(glove_name, 'r') as fp:\n",
    "    i = 0\n",
    "    for l in fp:\n",
    "        l = l.strip().split()\n",
    "        w = l[0]\n",
    "        glove_index_dict[w] = i\n",
    "        glove_embedding_weights[i,:] = map(float,l[1:])\n",
    "        i += 1\n",
    "glove_embedding_weights *= globale_scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.040815727600190289"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove_embedding_weights.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for w,i in glove_index_dict.iteritems():\n",
    "    w = w.lower()\n",
    "    if w not in glove_index_dict:\n",
    "        glove_index_dict[w] = i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## embedding matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use GloVe to initialize embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random-embedding/glove scale 0.00956354285866652 std 0.005521256656101686\n",
      "number of tokens, in small vocab, found in glove and copied to embedding 21788 0.5447\n"
     ]
    }
   ],
   "source": [
    "# generate random embedding with same scale as glove\n",
    "np.random.seed(seed)\n",
    "shape = (vocab_size, embedding_dim)\n",
    "scale = glove_embedding_weights.std()*np.sqrt(12)/2 # uniform and not normal\n",
    "embedding = np.random.uniform(low=-scale, high=scale, size=shape)\n",
    "print 'random-embedding/glove scale', scale, 'std', embedding.std()\n",
    "\n",
    "# copy from glove weights of words that appear in our short vocabulary (idx2word)\n",
    "c = 0\n",
    "for i in range(vocab_size):\n",
    "    w = idx2word[i]\n",
    "    g = glove_index_dict.get(w, glove_index_dict.get(w.lower()))\n",
    "    if g is None and w.startswith('#'): # glove has no hastags (I think...)\n",
    "        w = w[1:]\n",
    "        g = glove_index_dict.get(w, glove_index_dict.get(w.lower()))\n",
    "    if g is not None:\n",
    "        embedding[i,:] = glove_embedding_weights[g,:]\n",
    "        c+=1\n",
    "print 'number of tokens, in small vocab, found in glove and copied to embedding', c,c/float(vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "lots of word in the full vocabulary (word2idx) are outside `vocab_size`.\n",
    "Build an alterantive which will map them to their closest match in glove but only if the match\n",
    "is good enough (cos distance above `glove_thr`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_thr = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2glove = {}\n",
    "for w in word2idx:\n",
    "    if w in glove_index_dict:\n",
    "        g = w\n",
    "    elif w.lower() in glove_index_dict:\n",
    "        g = w.lower()\n",
    "    elif w.startswith('#') and w[1:] in glove_index_dict:\n",
    "        g = w[1:]\n",
    "    elif w.startswith('#') and w[1:].lower() in glove_index_dict:\n",
    "        g = w[1:].lower()\n",
    "    else:\n",
    "        continue\n",
    "    word2glove[w] = g"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for every word outside the embedding matrix find the closest word inside the mebedding matrix.\n",
    "Use cos distance of GloVe vectors.\n",
    "\n",
    "Allow for the last `nb_unknown_words` words inside the embedding matrix to be considered to be outside.\n",
    "Dont accept distances below `glove_thr`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of glove substitutes found 9442\n"
     ]
    }
   ],
   "source": [
    "normed_embedding = embedding/np.array([np.sqrt(np.dot(gweight,gweight)) for gweight in embedding])[:,None]\n",
    "\n",
    "nb_unknown_words = 100\n",
    "\n",
    "glove_match = []\n",
    "for w,idx in word2idx.iteritems():\n",
    "    if idx >= vocab_size-nb_unknown_words and w.isalpha() and w in word2glove:\n",
    "        gidx = glove_index_dict[word2glove[w]]\n",
    "        gweight = glove_embedding_weights[gidx,:].copy()\n",
    "        # find row in embedding that has the highest cos score with gweight\n",
    "        gweight /= np.sqrt(np.dot(gweight,gweight))\n",
    "        score = np.dot(normed_embedding[:vocab_size-nb_unknown_words], gweight)\n",
    "        while True:\n",
    "            embedding_idx = score.argmax()\n",
    "            s = score[embedding_idx]\n",
    "            if s < glove_thr:\n",
    "                break\n",
    "            if idx2word[embedding_idx] in word2glove :\n",
    "                glove_match.append((w, embedding_idx, s)) \n",
    "                break\n",
    "            score[embedding_idx] = -1\n",
    "glove_match.sort(key = lambda x: -x[2])\n",
    "print '# of glove substitutes found', len(glove_match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "manually check that the worst substitutions we are going to do are good enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5009126470244796 cuervos => CALANDRIA\n",
      "0.5008413593481128 recurrencia => tumor\n",
      "0.5008288014698516 Carbonero => Kiko\n",
      "0.5007494593684143 entubado => dique\n",
      "0.5006904559456788 Aries => aparente\n",
      "0.5003991718893522 protocolo => segura\n",
      "0.5000843245354017 babel => fantástica\n",
      "0.500050047502019 Antiguos => antiguos\n",
      "0.5000436395201061 almanaques => Editorial\n",
      "0.5000297363619396 anotada => lanzador\n"
     ]
    }
   ],
   "source": [
    "for orig, sub, score in glove_match[-10:]:\n",
    "    print score, orig,'=>', idx2word[sub]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "build a lookup table of index of outside words to index of inside words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_idx2idx = dict((word2idx[w],embedding_idx) for  w, embedding_idx, _ in glove_match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49998"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = [[word2idx[token] for token in headline.split()] for headline in heads]\n",
    "len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEgVJREFUeJzt3X+s3XV9x/Hna4CbU2OL3DWsrSuL3RZMJrobwGgWhKxUaix/OMayzc6Q9B9mNJvRYpaQoST1HxWTjaWBbmVRsUMdjbKxDjXbkoHcDvwBaLhiCW2AdrawOTNM3Xt/nE/dWb1399z23nO59/N8JDfn+31/P+d7P59w6Ot8P9/POTdVhSSpPz+11B2QJC0NA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6NFABJViW5K8m3kjyW5I1Jzk2yP8nj7XF1a5skn0gyneTrSd4wdJ5trf3jSbYt1qAkSXPLKJ8ETrIH+Kequi3JS4CfBT4IHKuqnUl2AKur6gNJrgLeDVwFXALcUlWXJDkXmAImgQIOAL9WVcdn+73nnXdebdiw4cxGKEmdOXDgwL9V1cRc7c6eq0GSVwK/Dvw+QFX9EPhhkq3AZa3ZHuArwAeArcAdNUiW+9vVw/mt7f6qOtbOux/YDHx6tt+9YcMGpqam5uqiJGlIkidHaTfKFNAFwFHgL5I8lOS2JC8D1lTV063NM8Catr0WeGro+Ydabbb6qR3fnmQqydTRo0dHGYMk6TSMEgBnA28Abq2q1wP/CewYbtDe7S/It8pV1a6qmqyqyYmJOa9gJEmnaZQAOAQcqqoH2v5dDALh2Ta1Q3s80o4fBtYPPX9dq81WlyQtgTkDoKqeAZ5K8sutdAXwKLAPOLmSZxtwd9veB7yzrQa6FHi+TRXdC2xKsrqtGNrUapKkJTDnTeDm3cAn2wqgJ4B3MQiPvUmuA54Ermlt72GwAmga+EFrS1UdS/Ih4MHW7qaTN4QlSeM30jLQpTI5OVmuApKk+UlyoKom52rnJ4ElqVMGgCR1ygCQpE6NehNYndmw44sz1g/u3DLmnkhaLAZA52b7h17SyucUkCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnfJzAJ1wvb+kU3kFIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KmRAiDJwSTfSPJwkqlWOzfJ/iSPt8fVrZ4kn0gyneTrSd4wdJ5trf3jSbYtzpAkSaOYzxXAW6rqoqqabPs7gPuqaiNwX9sHeCuwsf1sB26FQWAANwKXABcDN54MDUnS+J3JFNBWYE/b3gNcPVS/owbuB1YlOR+4EthfVceq6jiwH9h8Br9fknQGRg2AAv4+yYEk21ttTVU93bafAda07bXAU0PPPdRqs9UlSUtg1L8I9uaqOpzk54D9Sb41fLCqKkktRIdawGwHePWrX70Qp9QCmu0vix3cuWXMPZF0pka6Aqiqw+3xCPB5BnP4z7apHdrjkdb8MLB+6OnrWm22+qm/a1dVTVbV5MTExPxGI0ka2ZwBkORlSV5xchvYBHwT2AecXMmzDbi7be8D3tlWA10KPN+miu4FNiVZ3W7+bmo1SdISGGUKaA3w+SQn23+qqv4uyYPA3iTXAU8C17T29wBXAdPAD4B3AVTVsSQfAh5s7W6qqmMLNhJJ0rzMGQBV9QTwuhnq3wOumKFewPWznGs3sHv+3ZQkLTQ/CSxJnTIAJKlTBoAkdcoAkKROGQCS1KlRPwmsZWC2T+lK0ky8ApCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq1MgBkOSsJA8l+ULbvyDJA0mmk3wmyUta/afb/nQ7vmHoHDe0+reTXLnQg5EkjW4+VwDvAR4b2v8I8LGqeg1wHLiu1a8Djrf6x1o7klwIXAu8FtgM/FmSs86s+5Kk0zVSACRZB2wBbmv7AS4H7mpN9gBXt+2tbZ92/IrWfitwZ1W9UFXfBaaBixdiEJKk+Tt7xHYfB94PvKLtvwp4rqpOtP1DwNq2vRZ4CqCqTiR5vrVfC9w/dM7h52iZ27DjizPWD+7cMuaeSBrVnFcASd4GHKmqA2PoD0m2J5lKMnX06NFx/EpJ6tIoU0BvAt6e5CBwJ4Opn1uAVUlOXkGsAw637cPAeoB2/JXA94brMzznx6pqV1VNVtXkxMTEvAckSRrNnFNAVXUDcANAksuA91XV7yT5a+AdDEJhG3B3e8q+tv8v7fiXqqqS7AM+leSjwM8DG4GvLuxw+jDbdIskzceo9wBm8gHgziQfBh4Cbm/124G/SjINHGOw8oeqeiTJXuBR4ARwfVX96Ax+vyTpDMwrAKrqK8BX2vYTzLCKp6r+C/jNWZ5/M3DzfDspSVp4fhJYkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASerUnAGQ5GeSfDXJ15I8kuRPWv2CJA8kmU7ymSQvafWfbvvT7fiGoXPd0OrfTnLlYg1KkjS3s0do8wJweVV9P8k5wD8n+VvgD4GPVdWdSf4cuA64tT0er6rXJLkW+AjwW0kuBK4FXgv8PPAPSX6pqn60COPSi8SGHV+csX5w55Yx90TSqea8AqiB77fdc9pPAZcDd7X6HuDqtr217dOOX5EkrX5nVb1QVd8FpoGLF2QUkqR5G+keQJKzkjwMHAH2A98BnquqE63JIWBt214LPAXQjj8PvGq4PsNzJEljNlIAVNWPquoiYB2Dd+2/slgdSrI9yVSSqaNHjy7Wr5Gk7s1rFVBVPQd8GXgjsCrJyXsI64DDbfswsB6gHX8l8L3h+gzPGf4du6pqsqomJyYm5tM9SdI8jLIKaCLJqrb9UuA3gMcYBME7WrNtwN1te1/bpx3/UlVVq1/bVgldAGwEvrpQA5Ekzc8oq4DOB/YkOYtBYOytqi8keRS4M8mHgYeA21v724G/SjINHGOw8oeqeiTJXuBR4ARwvSuAJGnpzBkAVfV14PUz1J9ghlU8VfVfwG/Ocq6bgZvn301J0kLzk8CS1CkDQJI6Nco9AC2R2T5FK0kLwSsASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSp86eq0GS9cAdwBqggF1VdUuSc4HPABuAg8A1VXU8SYBbgKuAHwC/X1X/2s61DfjjduoPV9WehR2OlosNO744Y/3gzi1j7onUr1GuAE4Af1RVFwKXAtcnuRDYAdxXVRuB+9o+wFuBje1nO3ArQAuMG4FLgIuBG5OsXsCxSJLmYc4AqKqnT76Dr6r/AB4D1gJbgZPv4PcAV7ftrcAdNXA/sCrJ+cCVwP6qOlZVx4H9wOYFHY0kaWTzugeQZAPweuABYE1VPd0OPcNgiggG4fDU0NMOtdpsdUnSEhg5AJK8HPgs8N6q+vfhY1VVDO4PnLEk25NMJZk6evToQpxSkjSDkQIgyTkM/vH/ZFV9rpWfbVM7tMcjrX4YWD/09HWtNlv9/6iqXVU1WVWTExMT8xmLJGke5gyAtqrnduCxqvro0KF9wLa2vQ24e6j+zgxcCjzfporuBTYlWd1u/m5qNUnSEphzGSjwJuD3gG8kebjVPgjsBPYmuQ54ErimHbuHwRLQaQbLQN8FUFXHknwIeLC1u6mqji3IKCRJ8zZnAFTVPwOZ5fAVM7Qv4PpZzrUb2D2fDvZgtjXxkrSY/CSwJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KlR/ii8NDaz/X3kgzu3jLkn0srnFYAkdcoAkKROGQCS1CkDQJI65U3gMZrtBqckLQWvACSpUwaAJHVqzgBIsjvJkSTfHKqdm2R/ksfb4+pWT5JPJJlO8vUkbxh6zrbW/vEk2xZnOJKkUY1yBfCXwOZTajuA+6pqI3Bf2wd4K7Cx/WwHboVBYAA3ApcAFwM3ngwNSdLSmDMAquofgWOnlLcCe9r2HuDqofodNXA/sCrJ+cCVwP6qOlZVx4H9/GSoSJLG6HTvAaypqqfb9jPAmra9FnhqqN2hVputLklaImd8E7iqCqgF6AsASbYnmUoydfTo0YU6rSTpFKcbAM+2qR3a45FWPwysH2q3rtVmq/+EqtpVVZNVNTkxMXGa3ZMkzeV0A2AfcHIlzzbg7qH6O9tqoEuB59tU0b3ApiSr283fTa0mSVoic34SOMmngcuA85IcYrCaZyewN8l1wJPANa35PcBVwDTwA+BdAFV1LMmHgAdbu5uq6tQby5KkMZozAKrqt2c5dMUMbQu4fpbz7AZ2z6t3kqRF4yeBJalTBoAkdcoAkKROGQCS1CkDQJI65R+E0bIw2x/TObhzy5h7Iq0cXgFIUqe8AlgE/ulHScuBVwCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTvlBMC1rfkWEdPq8ApCkThkAktQpA0CSOuU9gDPgl75JWs68ApCkThkAktQpp4C0Irk8VJqbATAC5/olrUROAUlSpwwASerU2KeAkmwGbgHOAm6rqp3j7sNsnOpZ+f6//8beH1BvxnoFkOQs4E+BtwIXAr+d5MJx9kGSNDDuK4CLgemqegIgyZ3AVuDRMfdD+gmuHFJvxh0Aa4GnhvYPAZeMuQ9O9WheFur1YpDoxeZFtww0yXZge9v9fpJvz/GU84B/W9xevSj1Om5YpmPPR874FMty3AvAcc/fL4zSaNwBcBhYP7S/rtV+rKp2AbtGPWGSqaqaXJjuLR+9jhv6Hbvj7ss4xj3uZaAPAhuTXJDkJcC1wL4x90GSxJivAKrqRJI/AO5lsAx0d1U9Ms4+SJIGxn4PoKruAe5ZwFOOPF20wvQ6buh37I67L4s+7lTVYv8OSdKLkF8FIUmdWtYBkGRzkm8nmU6yY6n7s1iS7E5yJMk3h2rnJtmf5PH2uHop+7gYkqxP8uUkjyZ5JMl7Wn1Fjz3JzyT5apKvtXH/SatfkOSB9nr/TFtIseIkOSvJQ0m+0PZX/LiTHEzyjSQPJ5lqtUV/nS/bAOjsayX+Eth8Sm0HcF9VbQTua/srzQngj6rqQuBS4Pr233ilj/0F4PKqeh1wEbA5yaXAR4CPVdVrgOPAdUvYx8X0HuCxof1exv2WqrpoaOnnor/Ol20AMPS1ElX1Q+Dk10qsOFX1j8CxU8pbgT1tew9w9Vg7NQZV9XRV/Wvb/g8G/yisZYWPvQa+33bPaT8FXA7c1eorbtwASdYBW4Db2n7oYNyzWPTX+XIOgJm+VmLtEvVlKaypqqfb9jPAmqXszGJLsgF4PfAAHYy9TYM8DBwB9gPfAZ6rqhOtyUp9vX8ceD/w323/VfQx7gL+PsmB9m0IMIbX+YvuqyA0f1VVSVbscq4kLwc+C7y3qv598KZwYKWOvap+BFyUZBXweeBXlrhLiy7J24AjVXUgyWVL3Z8xe3NVHU7yc8D+JN8aPrhYr/PlfAUw59dKrHDPJjkfoD0eWeL+LIok5zD4x/+TVfW5Vu5i7ABV9RzwZeCNwKokJ9+0rcTX+5uAtyc5yGBK93IGfztkpY+bqjrcHo8wCPyLGcPrfDkHQO9fK7EP2Na2twF3L2FfFkWb/70deKyqPjp0aEWPPclEe+dPkpcCv8Hg/seXgXe0Zitu3FV1Q1Wtq6oNDP5//lJV/Q4rfNxJXpbkFSe3gU3ANxnD63xZfxAsyVUM5gxPfq3EzUvcpUWR5NPAZQy+HfBZ4Ebgb4C9wKuBJ4FrqurUG8XLWpI3A/8EfIP/nRP+IIP7ACt27El+lcFNv7MYvEnbW1U3JflFBu+MzwUeAn63ql5Yup4unjYF9L6qettKH3cb3+fb7tnAp6rq5iSvYpFf58s6ACRJp285TwFJks6AASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqf+B7ftRW0LVPDSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(map(len,Y),bins=50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49998"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [[word2idx[token] for token in d.split()] for d in desc]\n",
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEmFJREFUeJzt3X/M3fVd9/Hn624HTjbXMi6bSru7TBtNNdphw7psMbs3byjMWEyWBWKkLmiNg2RTEy2aiG4uYUbnLcnEG+/1XjEThvshDeusFUmMJjAuNgYtDLlknbQptK4M1CUq8+0f53NtZ/1c7XVx/eg57Ho+km/O9/v+/nqfntO+zvfHOU1VIUnSsP8x6gYkSePHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJn5agbmK8LLrigNmzYMOo2JOkl5cEHH/znqpqYbbmXbDhs2LCBycnJUbchSS8pSb48l+U8rSRJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6rxkvyF9Nm3Y9ekZ64dvettZ7kSSzg6PHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnVnDIcn6JPcmeTTJoSTvbvXfSnI0yUNtuGJonRuSTCV5PMllQ/VtrTaVZNdQ/aIk97f6x5Kcs9hPVJI0d3M5cngB+JWq2gRsBa5LsqnN+4Oq2tyGfQBt3lXADwLbgD9KsiLJCuBDwOXAJuDqoe18oG3r+4BngWsX6flJkuZh1nCoqmNV9bk2/i/AY8CFZ1hlO3BHVf17VX0JmAIuacNUVT1ZVf8B3AFsTxLgLcDH2/p7gCvn+4QkSQv3oq45JNkAvA64v5WuT/Jwkt1JVrfahcBTQ6sdabXT1V8NfLWqXjilPtP+dyaZTDJ54sSJF9O6JOlFmHM4JHkF8AngPVX1PHAL8L3AZuAY8PtL0uGQqrq1qrZU1ZaJiYml3p0kLVtz+s9+kryMQTB8tKo+CVBVzwzN/xPg7jZ5FFg/tPq6VuM09a8Aq5KsbEcPw8tLkkZgLncrBfgw8FhVfXCovnZosZ8CDrbxvcBVSc5NchGwEfgs8ACwsd2ZdA6Di9Z7q6qAe4G3t/V3AHct7GlJkhZiLkcObwR+BngkyUOt9usM7jbaDBRwGPgFgKo6lORO4FEGdzpdV1VfB0hyPbAfWAHsrqpDbXu/BtyR5HeAzzMII0nSiMwaDlX1d0BmmLXvDOu8H3j/DPV9M61XVU8yuJtJkjQG/Ia0JKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkzazgkWZ/k3iSPJjmU5N2tfn6SA0meaI+rWz1Jbk4yleThJBcPbWtHW/6JJDuG6j+a5JG2zs1JshRPVpI0N3M5cngB+JWq2gRsBa5LsgnYBdxTVRuBe9o0wOXAxjbsBG6BQZgANwKvBy4BbpwOlLbMzw+tt23hT02SNF+zhkNVHauqz7XxfwEeAy4EtgN72mJ7gCvb+Hbgthq4D1iVZC1wGXCgqk5W1bPAAWBbm/ddVXVfVRVw29C2JEkj8KKuOSTZALwOuB9YU1XH2qyngTVt/ELgqaHVjrTamepHZqhLkkZkzuGQ5BXAJ4D3VNXzw/PaJ/5a5N5m6mFnkskkkydOnFjq3UnSsjWncEjyMgbB8NGq+mQrP9NOCdEej7f6UWD90OrrWu1M9XUz1DtVdWtVbamqLRMTE3NpXZI0D3O5WynAh4HHquqDQ7P2AtN3HO0A7hqqX9PuWtoKPNdOP+0HLk2yul2IvhTY3+Y9n2Rr29c1Q9uSJI3Ayjks80bgZ4BHkjzUar8O3ATcmeRa4MvAO9q8fcAVwBTwNeCdAFV1Msn7gAfacu+tqpNt/F3AR4CXA59pgyRpRGYNh6r6O+B03zt46wzLF3Ddaba1G9g9Q30S+KHZepEknR1+Q1qS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1Jk1HJLsTnI8ycGh2m8lOZrkoTZcMTTvhiRTSR5PctlQfVurTSXZNVS/KMn9rf6xJOcs5hOUJL14czly+AiwbYb6H1TV5jbsA0iyCbgK+MG2zh8lWZFkBfAh4HJgE3B1WxbgA21b3wc8C1y7kCckSVq4WcOhqv4WODnH7W0H7qiqf6+qLwFTwCVtmKqqJ6vqP4A7gO1JArwF+Hhbfw9w5Yt8DpKkRbaQaw7XJ3m4nXZa3WoXAk8NLXOk1U5XfzXw1ap64ZS6JGmE5hsOtwDfC2wGjgG/v2gdnUGSnUkmk0yeOHHibOxSkpaleYVDVT1TVV+vqv8C/oTBaSOAo8D6oUXXtdrp6l8BViVZeUr9dPu9taq2VNWWiYmJ+bQuSZqDeYVDkrVDkz8FTN/JtBe4Ksm5SS4CNgKfBR4ANrY7k85hcNF6b1UVcC/w9rb+DuCu+fQkSVo8K2dbIMntwJuBC5IcAW4E3pxkM1DAYeAXAKrqUJI7gUeBF4DrqurrbTvXA/uBFcDuqjrUdvFrwB1Jfgf4PPDhRXt2kqR5mTUcqurqGcqn/Qe8qt4PvH+G+j5g3wz1J/nmaSlJ0hjwG9KSpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM6sv62k09uw69Mz1g/f9Laz3IkkLS6PHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktSZNRyS7E5yPMnBodr5SQ4keaI9rm71JLk5yVSSh5NcPLTOjrb8E0l2DNV/NMkjbZ2bk2Sxn6Qk6cWZy5HDR4Btp9R2AfdU1UbgnjYNcDmwsQ07gVtgECbAjcDrgUuAG6cDpS3z80PrnbovSdJZNms4VNXfAidPKW8H9rTxPcCVQ/XbauA+YFWStcBlwIGqOllVzwIHgG1t3ndV1X1VVcBtQ9uSJI3IfK85rKmqY238aWBNG78QeGpouSOtdqb6kRnqM0qyM8lkkskTJ07Ms3VJ0mwWfEG6feKvRehlLvu6taq2VNWWiYmJs7FLSVqW5hsOz7RTQrTH461+FFg/tNy6VjtTfd0MdUnSCM03HPYC03cc7QDuGqpf0+5a2go8104/7QcuTbK6XYi+FNjf5j2fZGu7S+maoW1JkkZk5WwLJLkdeDNwQZIjDO46ugm4M8m1wJeBd7TF9wFXAFPA14B3AlTVySTvAx5oy723qqYvcr+LwR1RLwc+0wZJ0gjNGg5VdfVpZr11hmULuO4029kN7J6hPgn80Gx9SJLOHr8hLUnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM6sP9m9nGzY9elRtyBJY8EjB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUWFA5JDid5JMlDSSZb7fwkB5I80R5Xt3qS3JxkKsnDSS4e2s6OtvwTSXYs7ClJkhZqMY4c/ldVba6qLW16F3BPVW0E7mnTAJcDG9uwE7gFBmEC3Ai8HrgEuHE6UCRJo7EUp5W2A3va+B7gyqH6bTVwH7AqyVrgMuBAVZ2sqmeBA8C2JehLkjRHCw2HAv4qyYNJdrbamqo61safBta08QuBp4bWPdJqp6tLkkZkof9N6Juq6miS7wYOJPni8MyqqiS1wH18QwugnQCvec1rFmuzkqRTLOjIoaqOtsfjwKcYXDN4pp0uoj0eb4sfBdYPrb6u1U5Xn2l/t1bVlqraMjExsZDWJUlnMO9wSHJekldOjwOXAgeBvcD0HUc7gLva+F7gmnbX0lbguXb6aT9waZLV7UL0pa0mSRqRhZxWWgN8Ksn0dv6sqv4yyQPAnUmuBb4MvKMtvw+4ApgCvga8E6CqTiZ5H/BAW+69VXVyAX1JkhZo3uFQVU8CPzJD/SvAW2eoF3Ddaba1G9g9314kSYvLb0hLkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySps3LUDXw72rDr0zPWD9/0trPciSTNj0cOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6izLW1lPd6upJGnAIwdJUmdswiHJtiSPJ5lKsmvU/UjScjYW4ZBkBfAh4HJgE3B1kk2j7UqSlq9xueZwCTBVVU8CJLkD2A48OtKuFpk/qyHppWIsjhyAC4GnhqaPtJokaQTG5chhTpLsBHa2yX9N8vg8N3UB8M+L09XC5QPfMjlWvZ3C3ubH3uZvnPt7qfb2P+eygXEJh6PA+qHpda32LarqVuDWhe4syWRVbVnodpaCvc2Pvc3POPcG493ft3tv43Ja6QFgY5KLkpwDXAXsHXFPkrRsjcWRQ1W9kOR6YD+wAthdVYdG3JYkLVtjEQ4AVbUP2HeWdrfgU1NLyN7mx97mZ5x7g/Hu79u6t1TVYjQiSfo2Mi7XHCRJY2RZhcOof6Ijye4kx5McHKqdn+RAkifa4+pWT5KbW68PJ7l4iXtbn+TeJI8mOZTk3ePSX5LvSPLZJF9ovf12q1+U5P7Ww8fazQwkObdNT7X5G5aqt6EeVyT5fJK7x7C3w0keSfJQkslWG/nr2va3KsnHk3wxyWNJ3jAOvSX5/vbnNT08n+Q949Bb298vtb8LB5Pc3v6OLO57rqqWxcDgQvc/Aq8FzgG+AGw6yz38GHAxcHCo9rvArja+C/hAG78C+AwQYCtw/xL3tha4uI2/EvgHBj9lMvL+2j5e0cZfBtzf9nkncFWr/zHwi238XcAft/GrgI+dhdf2l4E/A+5u0+PU22HgglNqI39d2/72AD/Xxs8BVo1Lb0M9rgCeZvD9gJH3xuALwl8CXj70XvvZxX7PLfkf7LgMwBuA/UPTNwA3jKCPDXxrODwOrG3ja4HH2/j/Ba6eabmz1OddwP8et/6A7wQ+B7yewZd8Vp76+jK46+0NbXxlWy5L2NM64B7gLcDd7R+Iseit7ecwfTiM/HUFXtX+kcu49XZKP5cCfz8uvfHNX5Q4v72H7gYuW+z33HI6rTSuP9GxpqqOtfGngTVtfGT9tsPO1zH4hD4W/bXTNg8Bx4EDDI4Cv1pVL8yw/2/01uY/B7x6qXoD/g/wq8B/telXj1FvAAX8VZIHM/iVARiP1/Ui4ATw/9spuf+X5Lwx6W3YVcDtbXzkvVXVUeD3gH8CjjF4Dz3IIr/nllM4jL0aRPtIbx9L8grgE8B7qur54Xmj7K+qvl5Vmxl8Sr8E+IFR9HGqJD8BHK+qB0fdyxm8qaouZvCrx9cl+bHhmSN8XVcyOM16S1W9Dvg3BqdqxqE3ANp5+58E/vzUeaPqrV3n2M4gXL8HOA/Yttj7WU7hMKef6BiBZ5KsBWiPx1v9rPeb5GUMguGjVfXJcesPoKq+CtzL4LB5VZLp7+oM7/8bvbX5rwK+skQtvRH4ySSHgTsYnFr6wzHpDfjGJ02q6jjwKQbhOg6v6xHgSFXd36Y/ziAsxqG3aZcDn6uqZ9r0OPT248CXqupEVf0n8EkG78NFfc8tp3AY15/o2AvsaOM7GJzrn65f0+6C2Ao8N3Q4u+iSBPgw8FhVfXCc+ksykWRVG385g2shjzEIibefprfpnt8O/E37lLfoquqGqlpXVRsYvKf+pqp+ehx6A0hyXpJXTo8zOH9+kDF4XavqaeCpJN/fSm9l8DP9I+9tyNV885TSdA+j7u2fgK1JvrP9vZ3+c1vc99xSX8wZp4HBHQX/wOB89W+MYP+3MzhH+J8MPjVdy+Dc3z3AE8BfA+e3ZcPgP0D6R+ARYMsS9/YmBofIDwMPteGKcegP+GHg8623g8Bvtvprgc8CUwwO+89t9e9o01Nt/mvP0uv7Zr55t9JY9Nb6+EIbDk2/78fhdW372wxMttf2L4DVY9TbeQw+Yb9qqDYuvf028MX29+FPgXMX+z3nN6QlSZ3ldFpJkjRHhoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqfPffD3BERj9wdgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(map(len,X),bins=50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "with open('data/%s.pkl'%FN,'wb') as fp:\n",
    "    pickle.dump((embedding, idx2word, word2idx, glove_idx2idx),fp,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "with open('data/%s.data.pkl'%FN,'wb') as fp:\n",
    "    pickle.dump((X,Y),fp,-1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}