word2vec_skipgram_medium_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    # obtains tokens with a least 1 alphabet\n",
    "    pattern = re.compile(r'[A-Za-z]+[\\w^\\']*|[\\w^\\']*[A-Za-z]+[\\w^\\']*')\n",
    "    return pattern.findall(text.lower())\n",
    "\n",
    "def mapping(tokens):\n",
    "    word_to_id = dict()\n",
    "    id_to_word = dict()\n",
    "\n",
    "    for i, token in enumerate(set(tokens)):\n",
    "        word_to_id[token] = i\n",
    "        id_to_word[i] = token\n",
    "\n",
    "    return word_to_id, id_to_word\n",
    "\n",
    "def generate_training_data(tokens, word_to_id, window_size):\n",
    "    N = len(tokens)\n",
    "    X, Y = [], []\n",
    "\n",
    "    for i in range(N):\n",
    "        nbr_inds = list(range(max(0, i - window_size), i)) + \\\n",
    "                   list(range(i + 1, min(N, i + window_size + 1)))\n",
    "        for j in nbr_inds:\n",
    "            X.append(word_to_id[tokens[i]])\n",
    "            Y.append(word_to_id[tokens[j]])\n",
    "            \n",
    "    X = np.array(X)\n",
    "    X = np.expand_dims(X, axis=0)\n",
    "    Y = np.array(Y)\n",
    "    Y = np.expand_dims(Y, axis=0)\n",
    "            \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = \"After the deduction of the costs of investing, \" \\\n",
    "      \"beating the stock market is a loser's game.\"\n",
    "tokens = tokenize(doc)\n",
    "word_to_id, id_to_word = mapping(tokens)\n",
    "X, Y = generate_training_data(tokens, word_to_id, 3)\n",
    "vocab_size = len(id_to_word)\n",
    "m = Y.shape[1]\n",
    "# turn Y into one hot encoding\n",
    "Y_one_hot = np.zeros((vocab_size, m))\n",
    "Y_one_hot[Y.flatten(), np.arange(m)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10,  6,  1,  7,  6,  1, 10,  7, 10,  1, 10,  2,  7, 10,  6, 10,\n",
       "         2,  1, 10,  6,  1,  2,  1,  3,  6,  1, 10,  1,  3, 12,  1, 10,\n",
       "         2,  3, 12, 10, 10,  2,  1, 12, 10,  9,  2,  1,  3, 10,  9,  0,\n",
       "         1,  3, 12,  9,  0, 11,  3, 12, 10,  0, 11,  4, 12, 10,  9, 11,\n",
       "         4,  5, 10,  9,  0,  4,  5,  8,  9,  0, 11,  5,  8,  0, 11,  4,\n",
       "         8, 11,  4,  5]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_wrd_emb(vocab_size, emb_size):\n",
    "    \"\"\"\n",
    "    vocab_size: int. vocabulary size of your corpus or training data\n",
    "    emb_size: int. word embedding size. How many dimensions to represent each vocabulary\n",
    "    \"\"\"\n",
    "    WRD_EMB = np.random.randn(vocab_size, emb_size) * 0.01\n",
    "    \n",
    "    assert(WRD_EMB.shape == (vocab_size, emb_size))\n",
    "    return WRD_EMB\n",
    "\n",
    "def initialize_dense(input_size, output_size):\n",
    "    \"\"\"\n",
    "    input_size: int. size of the input to the dense layer\n",
    "    output_szie: int. size of the output out of the dense layer\n",
    "    \"\"\"\n",
    "    W = np.random.randn(output_size, input_size) * 0.01\n",
    "    \n",
    "    assert(W.shape == (output_size, input_size))\n",
    "    return W\n",
    "\n",
    "def initialize_parameters(vocab_size, emb_size):\n",
    "    WRD_EMB = initialize_wrd_emb(vocab_size, emb_size)\n",
    "    W = initialize_dense(emb_size, vocab_size)\n",
    "    \n",
    "    parameters = {}\n",
    "    parameters['WRD_EMB'] = WRD_EMB\n",
    "    parameters['W'] = W\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ind_to_word_vecs(inds, parameters):\n",
    "    \"\"\"\n",
    "    inds: numpy array. shape: (1, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = inds.shape[1]\n",
    "    WRD_EMB = parameters['WRD_EMB']\n",
    "    word_vec = WRD_EMB[inds.flatten(), :].T\n",
    "    \n",
    "    assert(word_vec.shape == (WRD_EMB.shape[1], m))\n",
    "    \n",
    "    return word_vec\n",
    "\n",
    "def linear_dense(word_vec, parameters):\n",
    "    \"\"\"\n",
    "    word_vec: numpy array. shape: (emb_size, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = word_vec.shape[1]\n",
    "    W = parameters['W']\n",
    "    Z = np.dot(W, word_vec)\n",
    "    \n",
    "    assert(Z.shape == (W.shape[0], m))\n",
    "    \n",
    "    return W, Z\n",
    "\n",
    "def softmax(Z):\n",
    "    \"\"\"\n",
    "    Z: output out of the dense layer. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    softmax_out = np.divide(np.exp(Z), np.sum(np.exp(Z), axis=0, keepdims=True) + 0.001)\n",
    "    assert(softmax_out.shape == Z.shape)\n",
    "\n",
    "    return softmax_out\n",
    "\n",
    "def forward_propagation(inds, parameters):\n",
    "    word_vec = ind_to_word_vecs(inds, parameters)\n",
    "    W, Z = linear_dense(word_vec, parameters)\n",
    "    softmax_out = softmax(Z)\n",
    "    \n",
    "    caches = {}\n",
    "    caches['inds'] = inds\n",
    "    caches['word_vec'] = word_vec\n",
    "    caches['W'] = W\n",
    "    caches['Z'] = Z\n",
    "    return softmax_out, caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(softmax_out, Y):\n",
    "    \"\"\"\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    m = softmax_out.shape[1]\n",
    "    cost = -(1 / m) * np.sum(np.log(softmax_out[Y.flatten(), np.arange(Y.shape[1])] + 0.001))\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_backward(Y, softmax_out):\n",
    "    \"\"\"\n",
    "    Y: labels of training data. shape: (vocab_size, m)\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    m = Y.shape[1]\n",
    "    softmax_out[Y.flatten(), np.arange(m)] -= 1.0\n",
    "    dL_dZ = softmax_out\n",
    "    \n",
    "    assert(dL_dZ.shape == softmax_out.shape)\n",
    "    return dL_dZ\n",
    "\n",
    "def dense_backward(dL_dZ, caches):\n",
    "    \"\"\"\n",
    "    dL_dZ: shape: (vocab_size, m)\n",
    "    caches: dict. results from each steps of forward propagation\n",
    "    \"\"\"\n",
    "    W = caches['W']\n",
    "    word_vec = caches['word_vec']\n",
    "    m = word_vec.shape[1]\n",
    "    \n",
    "    dL_dW = (1 / m) * np.dot(dL_dZ, word_vec.T)\n",
    "    dL_dword_vec = np.dot(W.T, dL_dZ)\n",
    "\n",
    "    assert(W.shape == dL_dW.shape)\n",
    "    assert(word_vec.shape == dL_dword_vec.shape)\n",
    "    \n",
    "    return dL_dW, dL_dword_vec\n",
    "\n",
    "def backward_propagation(Y, softmax_out, caches):\n",
    "    dL_dZ = softmax_backward(Y, softmax_out)\n",
    "    dL_dW, dL_dword_vec = dense_backward(dL_dZ, caches)\n",
    "    \n",
    "    gradients = dict()\n",
    "    gradients['dL_dZ'] = dL_dZ\n",
    "    gradients['dL_dW'] = dL_dW\n",
    "    gradients['dL_dword_vec'] = dL_dword_vec\n",
    "    \n",
    "    return gradients\n",
    "\n",
    "def update_parameters(parameters, caches, gradients, learning_rate):\n",
    "    vocab_size, emb_size = parameters['WRD_EMB'].shape\n",
    "    inds = caches['inds']\n",
    "    dL_dword_vec = gradients['dL_dword_vec']\n",
    "    m = inds.shape[-1]\n",
    "    \n",
    "    parameters['WRD_EMB'][inds.flatten(), :] -= dL_dword_vec.T * learning_rate\n",
    "\n",
    "    parameters['W'] -= learning_rate * gradients['dL_dW']\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def skipgram_model_training(X, Y, vocab_size, emb_size, learning_rate, epochs, batch_size=256, parameters=None, print_cost=False, plot_cost=True):\n",
    "    costs = []\n",
    "    m = X.shape[1]\n",
    "    \n",
    "    if parameters is None:\n",
    "        parameters = initialize_parameters(vocab_size, emb_size)\n",
    "    \n",
    "    begin_time = datetime.now()\n",
    "    for epoch in range(epochs):\n",
    "        epoch_cost = 0\n",
    "        batch_inds = list(range(0, m, batch_size))\n",
    "        np.random.shuffle(batch_inds)\n",
    "        for i in batch_inds:\n",
    "            X_batch = X[:, i:i+batch_size]\n",
    "            Y_batch = Y[:, i:i+batch_size]\n",
    "\n",
    "            softmax_out, caches = forward_propagation(X_batch, parameters)\n",
    "            cost = cross_entropy(softmax_out, Y_batch)\n",
    "            gradients = backward_propagation(Y_batch, softmax_out, caches)\n",
    "            update_parameters(parameters, caches, gradients, learning_rate)\n",
    "            epoch_cost += np.squeeze(cost)\n",
    "            \n",
    "        costs.append(epoch_cost)\n",
    "        if print_cost and epoch % (epochs // 500) == 0:\n",
    "            print(\"Cost after epoch {}: {}\".format(epoch, epoch_cost))\n",
    "        if epoch % (epochs // 100) == 0:\n",
    "            learning_rate *= 0.98\n",
    "    end_time = datetime.now()\n",
    "    print('training time: {}'.format(end_time - begin_time))\n",
    "            \n",
    "    if plot_cost:\n",
    "        plt.plot(np.arange(epochs), costs)\n",
    "        plt.xlabel('# of epochs')\n",
    "        plt.ylabel('cost')\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 0: 2.552120233185112\n",
      "Cost after epoch 10: 2.551851450230315\n",
      "Cost after epoch 20: 2.551566801535802\n",
      "Cost after epoch 30: 2.5512460708580944\n",
      "Cost after epoch 40: 2.5508685448159887\n",
      "Cost after epoch 50: 2.5504118555757636\n",
      "Cost after epoch 60: 2.549862465443097\n",
      "Cost after epoch 70: 2.549188335250079\n",
      "Cost after epoch 80: 2.5483591097966576\n",
      "Cost after epoch 90: 2.547341391533995\n",
      "Cost after epoch 100: 2.5460970416292925\n",
      "Cost after epoch 110: 2.544612309450527\n",
      "Cost after epoch 120: 2.5428240003220437\n",
      "Cost after epoch 130: 2.5406765644676037\n",
      "Cost after epoch 140: 2.538110745072787\n",
      "Cost after epoch 150: 2.5350608123497653\n",
      "Cost after epoch 160: 2.531525017385724\n",
      "Cost after epoch 170: 2.5273895323091566\n",
      "Cost after epoch 180: 2.5225727289217215\n",
      "Cost after epoch 190: 2.516999650924172\n",
      "Cost after epoch 200: 2.510599558433586\n",
      "Cost after epoch 210: 2.5034505403524117\n",
      "Cost after epoch 220: 2.495420734652279\n",
      "Cost after epoch 230: 2.486478401762302\n",
      "Cost after epoch 240: 2.476637072489923\n",
      "Cost after epoch 250: 2.465949844682832\n",
      "Cost after epoch 260: 2.454726988015481\n",
      "Cost after epoch 270: 2.442947375773889\n",
      "Cost after epoch 280: 2.4307700497650697\n",
      "Cost after epoch 290: 2.4184112602722494\n",
      "Cost after epoch 300: 2.406108839415478\n",
      "Cost after epoch 310: 2.3943161573140412\n",
      "Cost after epoch 320: 2.383044513564169\n",
      "Cost after epoch 330: 2.372444554200727\n",
      "Cost after epoch 340: 2.3626401976511815\n",
      "Cost after epoch 350: 2.3536937955189154\n",
      "Cost after epoch 360: 2.3457411870064706\n",
      "Cost after epoch 370: 2.338566558584951\n",
      "Cost after epoch 380: 2.332044542303296\n",
      "Cost after epoch 390: 2.326032038264739\n",
      "Cost after epoch 400: 2.3203664924951726\n",
      "Cost after epoch 410: 2.314979372689516\n",
      "Cost after epoch 420: 2.30962767160556\n",
      "Cost after epoch 430: 2.304169566272228\n",
      "Cost after epoch 440: 2.298497654663674\n",
      "Cost after epoch 450: 2.292530357381084\n",
      "Cost after epoch 460: 2.286328979864872\n",
      "Cost after epoch 470: 2.27977244320655\n",
      "Cost after epoch 480: 2.27283821816053\n",
      "Cost after epoch 490: 2.265533048836315\n",
      "Cost after epoch 500: 2.2578779850737556\n",
      "Cost after epoch 510: 2.2500521713255655\n",
      "Cost after epoch 520: 2.2419776765528256\n",
      "Cost after epoch 530: 2.233686020807143\n",
      "Cost after epoch 540: 2.2252298450567807\n",
      "Cost after epoch 550: 2.2166655340829022\n",
      "Cost after epoch 560: 2.2082068989536854\n",
      "Cost after epoch 570: 2.199772715883771\n",
      "Cost after epoch 580: 2.1913989862927505\n",
      "Cost after epoch 590: 2.1831356558547537\n",
      "Cost after epoch 600: 2.1750280265846613\n",
      "Cost after epoch 610: 2.1672566713772494\n",
      "Cost after epoch 620: 2.1597205799232455\n",
      "Cost after epoch 630: 2.15242919545553\n",
      "Cost after epoch 640: 2.145402195977122\n",
      "Cost after epoch 650: 2.1386537624100215\n",
      "Cost after epoch 660: 2.1323072176445854\n",
      "Cost after epoch 670: 2.1262548348963355\n",
      "Cost after epoch 680: 2.1204844166426984\n",
      "Cost after epoch 690: 2.11499410711334\n",
      "Cost after epoch 700: 2.1097796682145034\n",
      "Cost after epoch 710: 2.104921994810863\n",
      "Cost after epoch 720: 2.1003264217940214\n",
      "Cost after epoch 730: 2.095974642442443\n",
      "Cost after epoch 740: 2.091857868850646\n",
      "Cost after epoch 750: 2.0879668947375367\n",
      "Cost after epoch 760: 2.084356900782105\n",
      "Cost after epoch 770: 2.080953417823675\n",
      "Cost after epoch 780: 2.0777400745361816\n",
      "Cost after epoch 790: 2.074708295518386\n",
      "Cost after epoch 800: 2.0718497730855674\n",
      "Cost after epoch 810: 2.0692038314270973\n",
      "Cost after epoch 820: 2.066714951955193\n",
      "Cost after epoch 830: 2.0643706595036466\n",
      "Cost after epoch 840: 2.06216437122276\n",
      "Cost after epoch 850: 2.0600898079892334\n",
      "Cost after epoch 860: 2.058175202297495\n",
      "Cost after epoch 870: 2.0563799990246228\n",
      "Cost after epoch 880: 2.0546949917247206\n",
      "Cost after epoch 890: 2.0531152359309335\n",
      "Cost after epoch 900: 2.051635988571616\n",
      "Cost after epoch 910: 2.0502769692451355\n",
      "Cost after epoch 920: 2.049008861547827\n",
      "Cost after epoch 930: 2.047824814014021\n",
      "Cost after epoch 940: 2.0467210100816775\n",
      "Cost after epoch 950: 2.0456937809009106\n",
      "Cost after epoch 960: 2.044756329333795\n",
      "Cost after epoch 970: 2.0438878470376025\n",
      "Cost after epoch 980: 2.043083269798517\n",
      "Cost after epoch 990: 2.0423396594829835\n",
      "Cost after epoch 1000: 2.041654201143464\n",
      "Cost after epoch 1010: 2.0410352285894775\n",
      "Cost after epoch 1020: 2.04046841979255\n",
      "Cost after epoch 1030: 2.0399501080281195\n",
      "Cost after epoch 1040: 2.0394780573819125\n",
      "Cost after epoch 1050: 2.0390501242896466\n",
      "Cost after epoch 1060: 2.0386709904457003\n",
      "Cost after epoch 1070: 2.038331219570606\n",
      "Cost after epoch 1080: 2.038028179468691\n",
      "Cost after epoch 1090: 2.0377601304399464\n",
      "Cost after epoch 1100: 2.0375253888122375\n",
      "Cost after epoch 1110: 2.037325850516146\n",
      "Cost after epoch 1120: 2.037155689243168\n",
      "Cost after epoch 1130: 2.037012984423696\n",
      "Cost after epoch 1140: 2.036896302306863\n",
      "Cost after epoch 1150: 2.036804238602316\n",
      "Cost after epoch 1160: 2.036736594036464\n",
      "Cost after epoch 1170: 2.036690170161569\n",
      "Cost after epoch 1180: 2.0366635294726056\n",
      "Cost after epoch 1190: 2.036655430777826\n",
      "Cost after epoch 1200: 2.0366646523740575\n",
      "Cost after epoch 1210: 2.0366895303039736\n",
      "Cost after epoch 1220: 2.0367287524699997\n",
      "Cost after epoch 1230: 2.036781237056614\n",
      "Cost after epoch 1240: 2.036845902681418\n",
      "Cost after epoch 1250: 2.0369216893038615\n",
      "Cost after epoch 1260: 2.0370060537127017\n",
      "Cost after epoch 1270: 2.0370989976691023\n",
      "Cost after epoch 1280: 2.037199731390578\n",
      "Cost after epoch 1290: 2.037307342948173\n",
      "Cost after epoch 1300: 2.037420947080273\n",
      "Cost after epoch 1310: 2.037537625095876\n",
      "Cost after epoch 1320: 2.0376582328188357\n",
      "Cost after epoch 1330: 2.037782230953056\n",
      "Cost after epoch 1340: 2.0379088907828393\n",
      "Cost after epoch 1350: 2.0380375137471876\n",
      "Cost after epoch 1360: 2.0381651962537832\n",
      "Cost after epoch 1370: 2.038293274304522\n",
      "Cost after epoch 1380: 2.03842141802597\n",
      "Cost after epoch 1390: 2.0385490823565307\n",
      "Cost after epoch 1400: 2.0386757528101502\n",
      "Cost after epoch 1410: 2.0387988126722187\n",
      "Cost after epoch 1420: 2.0389197834460684\n",
      "Cost after epoch 1430: 2.039038501114362\n",
      "Cost after epoch 1440: 2.0391545912169677\n",
      "Cost after epoch 1450: 2.0392677080833543\n",
      "Cost after epoch 1460: 2.0393756882918375\n",
      "Cost after epoch 1470: 2.039480021585898\n",
      "Cost after epoch 1480: 2.039580668086692\n",
      "Cost after epoch 1490: 2.0396774031782683\n",
      "Cost after epoch 1500: 2.0397700280812723\n",
      "Cost after epoch 1510: 2.0398569132092392\n",
      "Cost after epoch 1520: 2.0399393712725726\n",
      "Cost after epoch 1530: 2.0400174501591497\n",
      "Cost after epoch 1540: 2.0400910511011427\n",
      "Cost after epoch 1550: 2.04016009754901\n",
      "Cost after epoch 1560: 2.0402235066439856\n",
      "Cost after epoch 1570: 2.0402823306491875\n",
      "Cost after epoch 1580: 2.0403366747125093\n",
      "Cost after epoch 1590: 2.040386540383908\n",
      "Cost after epoch 1600: 2.0404319472642674\n",
      "Cost after epoch 1610: 2.0404723203944424\n",
      "Cost after epoch 1620: 2.0405084175710373\n",
      "Cost after epoch 1630: 2.040540375486203\n",
      "Cost after epoch 1640: 2.0405682694525953\n",
      "Cost after epoch 1650: 2.040592188231909\n",
      "Cost after epoch 1660: 2.040611989394712\n",
      "Cost after epoch 1670: 2.0406281392692214\n",
      "Cost after epoch 1680: 2.0406407843418735\n",
      "Cost after epoch 1690: 2.0406500470944575\n",
      "Cost after epoch 1700: 2.0406560587447053\n",
      "Cost after epoch 1710: 2.040659012835521\n",
      "Cost after epoch 1720: 2.040659113167578\n",
      "Cost after epoch 1730: 2.0406564984177473\n",
      "Cost after epoch 1740: 2.0406513135304616\n",
      "Cost after epoch 1750: 2.0406437078444606\n",
      "Cost after epoch 1760: 2.0406341079066412\n",
      "Cost after epoch 1770: 2.040622500302315\n",
      "Cost after epoch 1780: 2.0406090032336905\n",
      "Cost after epoch 1790: 2.0405937635187956\n",
      "Cost after epoch 1800: 2.0405769288509137\n",
      "Cost after epoch 1810: 2.0405590618248297\n",
      "Cost after epoch 1820: 2.0405399852740325\n",
      "Cost after epoch 1830: 2.0405197897647374\n",
      "Cost after epoch 1840: 2.040498609110447\n",
      "Cost after epoch 1850: 2.04047657555893\n",
      "Cost after epoch 1860: 2.040454305341662\n",
      "Cost after epoch 1870: 2.0404315122860797\n",
      "Cost after epoch 1880: 2.040408257445478\n",
      "Cost after epoch 1890: 2.0403846528887972\n",
      "Cost after epoch 1900: 2.0403608077738937\n",
      "Cost after epoch 1910: 2.040337328287254\n",
      "Cost after epoch 1920: 2.040313870004612\n",
      "Cost after epoch 1930: 2.0402904668820225\n",
      "Cost after epoch 1940: 2.0402672059966847\n",
      "Cost after epoch 1950: 2.0402441711075583\n",
      "Cost after epoch 1960: 2.0402219138637667\n",
      "Cost after epoch 1970: 2.040200074929614\n",
      "Cost after epoch 1980: 2.0401786664874813\n",
      "Cost after epoch 1990: 2.040157751599734\n",
      "Cost after epoch 2000: 2.0401373902988387\n",
      "Cost after epoch 2010: 2.040118052268852\n",
      "Cost after epoch 2020: 2.0400993976051356\n",
      "Cost after epoch 2030: 2.0400814235457707\n",
      "Cost after epoch 2040: 2.0400641728663063\n",
      "Cost after epoch 2050: 2.0400476860273624\n",
      "Cost after epoch 2060: 2.040032336788119\n",
      "Cost after epoch 2070: 2.0400178299247576\n",
      "Cost after epoch 2080: 2.0400041548236327\n",
      "Cost after epoch 2090: 2.039991339001205\n",
      "Cost after epoch 2100: 2.0399794085336995\n",
      "Cost after epoch 2110: 2.039968636595198\n",
      "Cost after epoch 2120: 2.039958789666803\n",
      "Cost after epoch 2130: 2.039949855600722\n",
      "Cost after epoch 2140: 2.039941851672354\n",
      "Cost after epoch 2150: 2.039934794513437\n",
      "Cost after epoch 2160: 2.039928857375965\n",
      "Cost after epoch 2170: 2.039923878589548\n",
      "Cost after epoch 2180: 2.0399198492423882\n",
      "Cost after epoch 2190: 2.0399167803561076\n",
      "Cost after epoch 2200: 2.0399146828571966\n",
      "Cost after epoch 2210: 2.039913632939702\n",
      "Cost after epoch 2220: 2.039913545530538\n",
      "Cost after epoch 2230: 2.0399144177808735\n",
      "Cost after epoch 2240: 2.0399162568506397\n",
      "Cost after epoch 2250: 2.0399190700355323\n",
      "Cost after epoch 2260: 2.039922839702439\n",
      "Cost after epoch 2270: 2.0399275581350578\n",
      "Cost after epoch 2280: 2.0399332294776253\n",
      "Cost after epoch 2290: 2.0399398577924845\n",
      "Cost after epoch 2300: 2.039947447222965\n",
      "Cost after epoch 2310: 2.0399558894081253\n",
      "Cost after epoch 2320: 2.039965251920714\n",
      "Cost after epoch 2330: 2.039975545348821\n",
      "Cost after epoch 2340: 2.039986770195688\n",
      "Cost after epoch 2350: 2.0399989268053\n",
      "Cost after epoch 2360: 2.0400118195603145\n",
      "Cost after epoch 2370: 2.040025587236171\n",
      "Cost after epoch 2380: 2.040040245434797\n",
      "Cost after epoch 2390: 2.0400557900258787\n",
      "Cost after epoch 2400: 2.0400722164143206\n",
      "Cost after epoch 2410: 2.040089246539863\n",
      "Cost after epoch 2420: 2.040107084708035\n",
      "Cost after epoch 2430: 2.040125749793622\n",
      "Cost after epoch 2440: 2.0401452319331947\n",
      "Cost after epoch 2450: 2.040165520528342\n",
      "Cost after epoch 2460: 2.0401862621658493\n",
      "Cost after epoch 2470: 2.0402077195568675\n",
      "Cost after epoch 2480: 2.040229913201766\n",
      "Cost after epoch 2490: 2.0402528267923135\n",
      "Cost after epoch 2500: 2.0402764431125893\n",
      "Cost after epoch 2510: 2.040300343342171\n",
      "Cost after epoch 2520: 2.040324840001539\n",
      "Cost after epoch 2530: 2.0403499538672794\n",
      "Cost after epoch 2540: 2.040375662059375\n",
      "Cost after epoch 2550: 2.040401940738403\n",
      "Cost after epoch 2560: 2.040428318590076\n",
      "Cost after epoch 2570: 2.040455147850564\n",
      "Cost after epoch 2580: 2.040482448562608\n",
      "Cost after epoch 2590: 2.0405101917480417\n",
      "Cost after epoch 2600: 2.040538347533328\n",
      "Cost after epoch 2610: 2.040566407674372\n",
      "Cost after epoch 2620: 2.0405947526065296\n",
      "Cost after epoch 2630: 2.0406234009117394\n",
      "Cost after epoch 2640: 2.040652318488071\n",
      "Cost after epoch 2650: 2.0406814704960543\n",
      "Cost after epoch 2660: 2.0407103293454885\n",
      "Cost after epoch 2670: 2.040739290951004\n",
      "Cost after epoch 2680: 2.0407683718951954\n",
      "Cost after epoch 2690: 2.0407975342734015\n",
      "Cost after epoch 2700: 2.040826739666214\n",
      "Cost after epoch 2710: 2.040855459927748\n",
      "Cost after epoch 2720: 2.0408840928857352\n",
      "Cost after epoch 2730: 2.040912652679499\n",
      "Cost after epoch 2740: 2.0409410991050696\n",
      "Cost after epoch 2750: 2.040969391702238\n",
      "Cost after epoch 2760: 2.040997020890427\n",
      "Cost after epoch 2770: 2.0410243724534616\n",
      "Cost after epoch 2780: 2.0410514576649903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 2790: 2.04107823555909\n",
      "Cost after epoch 2800: 2.041104665181756\n",
      "Cost after epoch 2810: 2.041130274123047\n",
      "Cost after epoch 2820: 2.041155422276731\n",
      "Cost after epoch 2830: 2.041180117603853\n",
      "Cost after epoch 2840: 2.041204319827632\n",
      "Cost after epoch 2850: 2.0412279889399034\n",
      "Cost after epoch 2860: 2.0412507070408084\n",
      "Cost after epoch 2870: 2.041272794896291\n",
      "Cost after epoch 2880: 2.041294256662931\n",
      "Cost after epoch 2890: 2.041315054030221\n",
      "Cost after epoch 2900: 2.0413351491858522\n",
      "Cost after epoch 2910: 2.04135419420133\n",
      "Cost after epoch 2920: 2.0413724585371686\n",
      "Cost after epoch 2930: 2.0413899420093236\n",
      "Cost after epoch 2940: 2.0414066093207985\n",
      "Cost after epoch 2950: 2.0414224258650138\n",
      "Cost after epoch 2960: 2.0414371268148455\n",
      "Cost after epoch 2970: 2.0414509195845114\n",
      "Cost after epoch 2980: 2.041463799084694\n",
      "Cost after epoch 2990: 2.0414757338218985\n",
      "Cost after epoch 3000: 2.0414866931424656\n",
      "Cost after epoch 3010: 2.04149650583009\n",
      "Cost after epoch 3020: 2.041505308232312\n",
      "Cost after epoch 3030: 2.0415130897867724\n",
      "Cost after epoch 3040: 2.041519823332646\n",
      "Cost after epoch 3050: 2.041525482654254\n",
      "Cost after epoch 3060: 2.0415299979011183\n",
      "Cost after epoch 3070: 2.0415334272383405\n",
      "Cost after epoch 3080: 2.0415357540893644\n",
      "Cost after epoch 3090: 2.041536955909265\n",
      "Cost after epoch 3100: 2.041537011161341\n",
      "Cost after epoch 3110: 2.0415359563654896\n",
      "Cost after epoch 3120: 2.041533766379564\n",
      "Cost after epoch 3130: 2.0415304181281253\n",
      "Cost after epoch 3140: 2.0415258937472474\n",
      "Cost after epoch 3150: 2.0415201764062387\n",
      "Cost after epoch 3160: 2.0415134113679603\n",
      "Cost after epoch 3170: 2.041505487046088\n",
      "Cost after epoch 3180: 2.041496373464713\n",
      "Cost after epoch 3190: 2.0414860573235236\n",
      "Cost after epoch 3200: 2.0414745263479377\n",
      "Cost after epoch 3210: 2.0414620344914614\n",
      "Cost after epoch 3220: 2.0414483824948415\n",
      "Cost after epoch 3230: 2.0414335331844136\n",
      "Cost after epoch 3240: 2.04141747756259\n",
      "Cost after epoch 3250: 2.0414002076237594\n",
      "Cost after epoch 3260: 2.0413820838043315\n",
      "Cost after epoch 3270: 2.0413628197105824\n",
      "Cost after epoch 3280: 2.041342370792519\n",
      "Cost after epoch 3290: 2.0413207319897335\n",
      "Cost after epoch 3300: 2.0412978991800936\n",
      "Cost after epoch 3310: 2.041274335227503\n",
      "Cost after epoch 3320: 2.0412496687232347\n",
      "Cost after epoch 3330: 2.041223847688436\n",
      "Cost after epoch 3340: 2.0411968705657446\n",
      "Cost after epoch 3350: 2.0411687366683884\n",
      "Cost after epoch 3360: 2.041140005709453\n",
      "Cost after epoch 3370: 2.041110224750004\n",
      "Cost after epoch 3380: 2.04107933445635\n",
      "Cost after epoch 3390: 2.0410473363031123\n",
      "Cost after epoch 3400: 2.0410142325591196\n",
      "Cost after epoch 3410: 2.0409806730098516\n",
      "Cost after epoch 3420: 2.0409461278053125\n",
      "Cost after epoch 3430: 2.0409105304521002\n",
      "Cost after epoch 3440: 2.040873884975192\n",
      "Cost after epoch 3450: 2.04083619611318\n",
      "Cost after epoch 3460: 2.040798196059191\n",
      "Cost after epoch 3470: 2.040759283571656\n",
      "Cost after epoch 3480: 2.04071938530346\n",
      "Cost after epoch 3490: 2.040678507358554\n",
      "Cost after epoch 3500: 2.0406366564733385\n",
      "Cost after epoch 3510: 2.0405946389828333\n",
      "Cost after epoch 3520: 2.0405517884465443\n",
      "Cost after epoch 3530: 2.0405080250667256\n",
      "Cost after epoch 3540: 2.0404633565806907\n",
      "Cost after epoch 3550: 2.040417791279238\n",
      "Cost after epoch 3560: 2.0403722010294882\n",
      "Cost after epoch 3570: 2.0403258608450403\n",
      "Cost after epoch 3580: 2.0402786849616623\n",
      "Cost after epoch 3590: 2.0402306823408725\n",
      "Cost after epoch 3600: 2.040181862422956\n",
      "Cost after epoch 3610: 2.0401331538768264\n",
      "Cost after epoch 3620: 2.040083780090405\n",
      "Cost after epoch 3630: 2.040033649881044\n",
      "Cost after epoch 3640: 2.039982773068357\n",
      "Cost after epoch 3650: 2.039931159881605\n",
      "Cost after epoch 3660: 2.039879787129576\n",
      "Cost after epoch 3670: 2.0398278335916804\n",
      "Cost after epoch 3680: 2.0397752032616108\n",
      "Cost after epoch 3690: 2.039721906497214\n",
      "Cost after epoch 3700: 2.039667954003276\n",
      "Cost after epoch 3710: 2.0396143622897354\n",
      "Cost after epoch 3720: 2.039560272495801\n",
      "Cost after epoch 3730: 2.039505584416371\n",
      "Cost after epoch 3740: 2.0394503086744655\n",
      "Cost after epoch 3750: 2.039394456184107\n",
      "Cost after epoch 3760: 2.0393390750634683\n",
      "Cost after epoch 3770: 2.0392832756098405\n",
      "Cost after epoch 3780: 2.0392269540589854\n",
      "Cost after epoch 3790: 2.0391701210714133\n",
      "Cost after epoch 3800: 2.0391127875494948\n",
      "Cost after epoch 3810: 2.039056025565584\n",
      "Cost after epoch 3820: 2.03899892110597\n",
      "Cost after epoch 3830: 2.0389413674880488\n",
      "Cost after epoch 3840: 2.038883375224698\n",
      "Cost after epoch 3850: 2.0388249550280837\n",
      "Cost after epoch 3860: 2.038767195776272\n",
      "Cost after epoch 3870: 2.0387091653329765\n",
      "Cost after epoch 3880: 2.0386507547272843\n",
      "Cost after epoch 3890: 2.038591974178177\n",
      "Cost after epoch 3900: 2.0385328340675035\n",
      "Cost after epoch 3910: 2.0384744334799465\n",
      "Cost after epoch 3920: 2.038415827947611\n",
      "Cost after epoch 3930: 2.0383569068207725\n",
      "Cost after epoch 3940: 2.0382976799128705\n",
      "Cost after epoch 3950: 2.038238157169447\n",
      "Cost after epoch 3960: 2.038179441856402\n",
      "Cost after epoch 3970: 2.0381205825305595\n",
      "Cost after epoch 3980: 2.0380614674443933\n",
      "Cost after epoch 3990: 2.0380021059243663\n",
      "Cost after epoch 4000: 2.03794250740337\n",
      "Cost after epoch 4010: 2.0378837738849573\n",
      "Cost after epoch 4020: 2.0378249518505633\n",
      "Cost after epoch 4030: 2.037765929001279\n",
      "Cost after epoch 4040: 2.037706714120998\n",
      "Cost after epoch 4050: 2.0376473160788553\n",
      "Cost after epoch 4060: 2.037588830749404\n",
      "Cost after epoch 4070: 2.037530306973252\n",
      "Cost after epoch 4080: 2.0374716324075464\n",
      "Cost after epoch 4090: 2.03741281525967\n",
      "Cost after epoch 4100: 2.037353863804958\n",
      "Cost after epoch 4110: 2.037295863484023\n",
      "Cost after epoch 4120: 2.0372378694676\n",
      "Cost after epoch 4130: 2.0371797698344567\n",
      "Cost after epoch 4140: 2.0371215721992955\n",
      "Cost after epoch 4150: 2.037063284230827\n",
      "Cost after epoch 4160: 2.0370059771694886\n",
      "Cost after epoch 4170: 2.036948716033683\n",
      "Cost after epoch 4180: 2.0368913897459153\n",
      "Cost after epoch 4190: 2.0368340053262988\n",
      "Cost after epoch 4200: 2.0367765698378464\n",
      "Cost after epoch 4210: 2.036720137064649\n",
      "Cost after epoch 4220: 2.036663784953649\n",
      "Cost after epoch 4230: 2.0366074036492656\n",
      "Cost after epoch 4240: 2.036550999586295\n",
      "Cost after epoch 4250: 2.036494579233673\n",
      "Cost after epoch 4260: 2.03643917614055\n",
      "Cost after epoch 4270: 2.036383883848217\n",
      "Cost after epoch 4280: 2.0363285940576983\n",
      "Cost after epoch 4290: 2.0362733126362085\n",
      "Cost after epoch 4300: 2.036218045478287\n",
      "Cost after epoch 4310: 2.0361638035623573\n",
      "Cost after epoch 4320: 2.036109698299792\n",
      "Cost after epoch 4330: 2.0360556232405855\n",
      "Cost after epoch 4340: 2.036001583708298\n",
      "Cost after epoch 4350: 2.035947585048568\n",
      "Cost after epoch 4360: 2.0358946137402505\n",
      "Cost after epoch 4370: 2.0358418009776202\n",
      "Cost after epoch 4380: 2.0357890424114244\n",
      "Cost after epoch 4390: 2.0357363428497552\n",
      "Cost after epoch 4400: 2.035683707118807\n",
      "Cost after epoch 4410: 2.03563209564002\n",
      "Cost after epoch 4420: 2.035580660968796\n",
      "Cost after epoch 4430: 2.0355293010699764\n",
      "Cost after epoch 4440: 2.0354780202670777\n",
      "Cost after epoch 4450: 2.035426822898725\n",
      "Cost after epoch 4460: 2.0353766421068302\n",
      "Cost after epoch 4470: 2.035326653080181\n",
      "Cost after epoch 4480: 2.0352767562750733\n",
      "Cost after epoch 4490: 2.0352269555627633\n",
      "Cost after epoch 4500: 2.0351772548274014\n",
      "Cost after epoch 4510: 2.035128559010933\n",
      "Cost after epoch 4520: 2.0350800669301665\n",
      "Cost after epoch 4530: 2.0350316816768443\n",
      "Cost after epoch 4540: 2.0349834067027235\n",
      "Cost after epoch 4550: 2.0349352454708227\n",
      "Cost after epoch 4560: 2.0348880740719792\n",
      "Cost after epoch 4570: 2.034841115695539\n",
      "Cost after epoch 4580: 2.0347942761820508\n",
      "Cost after epoch 4590: 2.0347475585962034\n",
      "Cost after epoch 4600: 2.0347009660127644\n",
      "Cost after epoch 4610: 2.034655345259131\n",
      "Cost after epoch 4620: 2.0346099444179404\n",
      "Cost after epoch 4630: 2.0345646721640063\n",
      "Cost after epoch 4640: 2.0345195312065\n",
      "Cost after epoch 4650: 2.0344745242638127\n",
      "Cost after epoch 4660: 2.034430468697989\n",
      "Cost after epoch 4670: 2.034386637806869\n",
      "Cost after epoch 4680: 2.0343429431597984\n",
      "Cost after epoch 4690: 2.0342993871406962\n",
      "Cost after epoch 4700: 2.034255972142079\n",
      "Cost after epoch 4710: 2.0342134860428946\n",
      "Cost after epoch 4720: 2.0341712275026036\n",
      "Cost after epoch 4730: 2.0341291110227804\n",
      "Cost after epoch 4740: 2.034087138690807\n",
      "Cost after epoch 4750: 2.03404531260221\n",
      "Cost after epoch 4760: 2.0340043912951815\n",
      "Cost after epoch 4770: 2.033963698783396\n",
      "Cost after epoch 4780: 2.03392315251829\n",
      "Cost after epoch 4790: 2.033882754317723\n",
      "Cost after epoch 4800: 2.0338425060073484\n",
      "Cost after epoch 4810: 2.0338031370650698\n",
      "Cost after epoch 4820: 2.033763996717511\n",
      "Cost after epoch 4830: 2.0337250053659237\n",
      "Cost after epoch 4840: 2.033686164583863\n",
      "Cost after epoch 4850: 2.0336474759524137\n",
      "Cost after epoch 4860: 2.033609640287842\n",
      "Cost after epoch 4870: 2.033572031772899\n",
      "Cost after epoch 4880: 2.033534573743121\n",
      "Cost after epoch 4890: 2.03349726755119\n",
      "Cost after epoch 4900: 2.033460114557103\n",
      "Cost after epoch 4910: 2.0334237874143297\n",
      "Cost after epoch 4920: 2.0333876849060033\n",
      "Cost after epoch 4930: 2.0333517332730073\n",
      "Cost after epoch 4940: 2.0333159336688187\n",
      "Cost after epoch 4950: 2.0332802872540396\n",
      "Cost after epoch 4960: 2.0332454391021897\n",
      "Cost after epoch 4970: 2.0332108121572015\n",
      "Cost after epoch 4980: 2.0331763355248484\n",
      "Cost after epoch 4990: 2.03314201017937\n",
      "training time: 0:00:01.137840\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAH1RJREFUeJzt3Xt4XHd95/H3d0Z3yZIsa2xHimTZudixnYuDYidNACdhSwlswyUUCoRA4MnSZUvylO4W0n3SpWyfLrRNuS5pIJBCQ6EhgQ23QkocIIQ4yI4T3+LYseNLYseybEe+6zLf/eMcjceKLqPL6MwcfV7PM8+c+c05Z74/WZ6Pzvmd+Y25OyIiIgCJqAsQEZHCoVAQEZEMhYKIiGQoFEREJEOhICIiGQoFERHJUCiIiEiGQkFERDIUCiIiklESdQFj1djY6G1tbVGXISJSVNasWXPA3VOjrVd0odDW1kZHR0fUZYiIFBUz25nLejp9JCIiGQoFERHJUCiIiEiGQkFERDIUCiIikqFQEBGRDIWCiIhkFN3nFMZre+dRfvDUi1x0dj0XtdQxe0ZF1CWJiBScaRMKG17q5kurtpEOv5K6taGKty5r5n2XtyogRERC5u5R1zAm7e3tPt5PNB/v6WPjS908vfswv3yuk8e2HaC6rITbr7uA96xoneRKRUQKh5mtcff20dabNkcKAFVlJVzW1sBlbQ18+LUL2N55lL96aCO3f389O7uO8Yk3LcLMoi5TRCQy03qgeUGqhns/uJz3Xd7KP/1qO/c+/kLUJYmIRGpahwJAMmH89R8u5Q0XzOFvf/Isz718JOqSREQiM+1DASCRMP7POy6kujzJp364MepyREQio1AINdaU89+uOY/fbOvi8ecPRF2OiEgkFApZ3ruildSMcu759Y6oSxERiYRCIUtFaZJ3X9bCI1v2s/vg8ajLERGZcgqFQf54efB5hQfW7om4EhGRqadQGKSpvpL2eTP59w37oi5FRGTKKRSG8KalZ/HsviM833k06lJERKaUQmEIb1w6F4BVz+6PuBIRkamlUBhCc30l8xurefz5rqhLERGZUgqFYVx57ixWb++itz8ddSkiIlNGoTCMK89p5FhPP0/vPhx1KSIiUyZvoWBmLWa2ysw2mdlGM7t1iHVWmtkrZrYuvN2Rr3rG6rL5DQCs3XUo4kpERKZOPqfO7gM+7u5rzWwGsMbMHnb3TYPW+7W7vyWPdYxLY005Z8+sZJ2OFERkGsnbkYK773X3teHyEWAz0Jyv18uHZa0zWbdLoSAi08eUjCmYWRuwDFg9xNNXmNnTZvZTM1syFfXk6pKWel565SQvd5+MuhQRkSmR91AwsxrgAeA2d+8e9PRaYJ67Xwx8EfjBMPu4xcw6zKyjs7MzvwVnuaSlHoCndLQgItNEXkPBzEoJAuE+d39w8PPu3u3uR8PlnwClZtY4xHp3u3u7u7enUql8lnyGJU21mMHmvYOzTEQknvJ59ZEB9wCb3f3OYdaZG66HmS0P6ymYT4xVlCaZP6uaLfv0bWwiMj3k8+qjK4EbgfVmti5sux1oBXD3u4AbgD8xsz7gBPBud/c81jRmC+fO0JGCiEwbeQsFd38MsFHW+RLwpXzVMBkWzp3Bv2/cx/GePqrK8pmhIiLR0yeaR7Fo7gzcYevLmjFVROJPoTCKhXNrATSuICLTgkJhFK0NVZSXJNi6X6EgIvGnUBhFMmG0zapmx4FjUZciIpJ3CoUcLEhVs12hICLTgEIhB/Mbq9nVdVzfrSAisadQyMH8xmr60s6eQyeiLkVEJK8UCjlYkKoBYMcBXZYqIvGmUMjBgsZqALZ3alxBROJNoZCDmdVl1FeVarBZRGJPoZCjBY3VbO/U6SMRiTeFQo7awiuQRETiTKGQo9aGKvZ2n+RUX3/UpYiI5I1CIUetDVW4w4u6LFVEYkyhkKPWhioAdh3UKSQRiS+FQo4GQmG3QkFEYkyhkKPUjHLKSxI6UhCRWFMo5MjMaG2oUiiISKwpFMYgCAUNNItIfCkUxqCloYrdB4/j7lGXIiKSFwqFMWhtqOLoqT4OHe+NuhQRkbxQKIyBLksVkbhTKIxB6yyFgojEm0JhDFpm6rMKIhJvCoUxqCxLkppRronxRCS2FApj1NpQxc6D+l4FEYknhcIYzWuo0pGCiMSWQmGMWmdpCm0RiS+FwhjNmxVMob1HU2iLSAwpFMaotaEaQKeQRCSW8hYKZtZiZqvMbJOZbTSzW0dY9zIz6zOzG/JVz2QZ+ADbzi4NNotI/JTkcd99wMfdfa2ZzQDWmNnD7r4peyUzSwKfAX6ex1omTWNNGVVlSXbqswoiEkN5O1Jw973uvjZcPgJsBpqHWPVPgQeA/fmqZTJlptDW6SMRiaEpGVMwszZgGbB6UHsz8DbgK6Nsf4uZdZhZR2dnZ77KzNm8WVU6UhCRWMp7KJhZDcGRwG3u3j3o6c8Bf+Hu6ZH24e53u3u7u7enUql8lZqzebOq2XXwOOm0ptAWkXjJ55gCZlZKEAj3ufuDQ6zSDnzHzAAagevMrM/df5DPuiaqtaGKnr40+4+cYm5dRdTliIhMmryFggXv9PcAm939zqHWcff5WevfC/yo0AMBgtNHEFyBpFAQkTjJ5+mjK4EbgWvMbF14u87MPmJmH8nj6+Zd5rJUjSuISMzk7UjB3R8DbAzrfyBftUy2pvpKkgnTFUgiEjv6RPM4lCYTNNdX6khBRGJHoTBO82ZVsUufahaRmFEojNP8xmq2dx7DXZelikh8KBTG6bzZNRw51cf+I6eiLkVEZNIoFMbpnNk1AGx9+WjElYiITB6FwjidG4bCtv1HIq5ERGTyKBTGKVVTTl1lKVv360hBROJDoTBOZsa5s2vYplAQkRhRKEzAuSmFgojEi0JhAs6bU0PXsR4OHuuJuhQRkUmhUJiA8+fMAODZfYNnBBcRKU4KhQlY2lwHwIYXX4m4EhGRyaFQmICG6jKa6ytZ/6KOFEQkHhQKE7S0uVZHCiISGwqFCbqwuY4dB47RfbI36lJERCZMoTBBGlcQkThRKEzQxWfXA7B256GIKxERmTiFwgTNrC5j4ZwZrN5xMOpSREQmTKEwCVYsaGDNzkP09qejLkVEZEIUCpNg+fwGjvf0s/ElXZoqIsVNoTAJlrc1APDkjq6IKxERmRiFwiSYXVvBglQ1j21TKIhIcVMoTJLXn5/iie1dnOjpj7oUEZFxUyhMkqsXzqanL80T23W0ICLFS6EwSZbPb6CyNMmjW/ZHXYqIyLgpFCZJRWmSK86Zxaotnbh71OWIiIyLQmESXb0wxa6Dx9lx4FjUpYiIjEtOoWBm78ylbbpbuXA2AI9u6Yy4EhGR8cn1SOGTObZNay0NVSxIVfPocwoFESlOJSM9aWZvAq4Dms3sC1lP1QJ9o2zbAnwTmAM4cLe7f37QOtcDnwbS4f5uc/fHxtqJQrLy/Nn8y+qdnOjpp7IsGXU5IiJjMtqRwktAB3ASWJN1ewh44yjb9gEfd/fFwOXAR81s8aB1fgFc7O6XADcDXxtb+YVn5cKULk0VkaI14pGCuz8NPG1m33b3XgAzmwm0uPuIc0W7+15gb7h8xMw2A83Apqx1jmZtUk1wRFHUls9voKI0waNb9nP1otlRlyMiMia5jik8bGa1ZtYArAW+amb/mOuLmFkbsAxYPcRzbzOzZ4EfExwtDLX9LWbWYWYdnZ2Ffb6+ojTJFQtm8UuNK4hIEco1FOrcvRt4O/BNd18BXJvLhmZWAzxAMF7wqmlE3f377r4IeCvB+MKruPvd7t7u7u2pVCrHkqOzcuFsXug6zgu6NFVEikyuoVBiZmcBfwT8KNedm1kpQSDc5+4PjrSuu/8KWGBmjbnuv1CtXBgElz7dLCLFJtdQ+GvgZ8Dz7v47M1sAbB1pAzMz4B5gs7vfOcw654brYWaXAuVA0Y/QzptVTUtDJb/VYLOIFJkRB5oHuPv9wP1Zj7cD7xhlsyuBG4H1ZrYubLsdaA33cVe4j/ebWS9wAniXx2SOiBXzZ/GLzS+TTjuJhEVdjohITnIKBTM7G/giwRs9wK+BW919z3DbhJ83GPHd0N0/A3wmt1KLy4r5DXxvzR627j/Kwrkzoi5HRCQnuZ4++gbBZxOawtsPwzYZxor5swBYrW9jE5EikmsopNz9G+7eF97uBQr/MqAItTRUclZdBau3H4y6FBGRnOUaCl1m9j4zS4a39xGDAeF8MjNWzG9g9Y4uTaUtIkUj11C4meBy1H0En1K+AfhAnmqKjRULZnHgaA/Pd+rzCiJSHMZySepN7p5y99kEIfGp/JUVD5e1zQRg7a4RZwQRESkYuYbCRdlzHbn7QYJpK2QECxprmFFewtO7D0ddiohITnINhUQ4ER4A4RxIOV3OOp0lEsZFLXWsUyiISJHINRT+AfitmX3azD4NPA58Nn9lxcclLfU8u+8IJ3v7oy5FRGRUOYWCu3+TYDK8l8Pb2939W/ksLC4uPrue/rSz4cVXoi5FRGRUOZ8CcvdNZH0XguTmkpZ6ANbtPkx7W0PE1YiIjCzX00cyTrNrK2iqq+DpPTpSEJHCp1CYApe01rNuty5LFZHCp1CYAhedXc/ugyc4dKwn6lJEREakUJgCS5vqANi091VfPCciUlAUClNgSVMtgK5AEpGCp1CYAjOry2iur2TjSzpSEJHCplCYIkuaatnwko4URKSwKRSmyJKmOnYcOMaxU31RlyIiMiyFwhRZ2lyLO2zWYLOIFDCFwhRZEl6BpMFmESlkCoUpMqe2nMaaMg02i0hBUyhMETNjcVMdGxQKIlLAFApTaGlTLVtfPsKpPk2jLSKFSaEwhZY01dGXdp7bdzTqUkREhqRQmEJLm4NPNm/U5xVEpEApFKZQy8wqZpSX6ENsIlKwFApTKJEwLmiq1RVIIlKwFApTbGlTHZv3dtOf9qhLERF5FYXCFFvSVMvJ3jTbOzXYLCKFJ2+hYGYtZrbKzDaZ2UYzu3WIdd5rZs+Y2Xoze9zMLs5XPYViSWawWaeQRKTw5PNIoQ/4uLsvBi4HPmpmiwetswN4vbtfCHwauDuP9RSEc1M1lJckdAWSiBSkknzt2N33AnvD5SNmthloBjZlrfN41iZPAGfnq55CUZJMsGjuDB0piEhBmpIxBTNrA5YBq0dY7UPAT4fZ/hYz6zCzjs7OzskvcIotbqpjw4uv4K7BZhEpLHkPBTOrAR4AbnP3If88NrOrCULhL4Z63t3vdvd2d29PpVL5K3aKLG2upftkH3sOnYi6FBGRM+Q1FMyslCAQ7nP3B4dZ5yLga8D17t6Vz3oKxcA02jqFJCKFJp9XHxlwD7DZ3e8cZp1W4EHgRnd/Ll+1FJpFc2eQTJgGm0Wk4ORtoBm4ErgRWG9m68K224FWAHe/C7gDmAX83yBD6HP39jzWVBAqSpOck6rWkYKIFJx8Xn30GGCjrPNh4MP5qqGQLW2q4zfPH4i6DBGRM+gTzRFZ3FTLy92n6DxyKupSREQyFAoROT3YrHEFESkcCoWILG7SdBciUngUChGpqyyltaFKRwoiUlAUChFaou9WEJECo1CI0NLmOnZ2Haf7ZG/UpYiIAAqFSF3YHAw2r9+jU0giUhgUChG6uKUegKd2HYq4EhGRgEIhQnWVpZw7u4andh2OuhQREUChELllLfU8tfuwptEWkYKgUIjYpfNmcvBYDzu7jkddioiIQiFqy1rDcYXdGlcQkegpFCJ23uwZVJclNa4gIgVBoRCxZMK4uKVeoSAiBUGhUACWtdazeW83J3r6oy5FRKY5hUIBWNYyk76088weHS2ISLQUCgWgvW0mZrB6x8GoSxGRaU6hUADqq8q4YG4tT2zviroUEZnmFAoF4vIFs1iz8xCn+jSuICLRUSgUiMsXNHCqL806XYUkIhFSKBSIFfNnYQZPbNe4gohER6FQIOqqSll8lsYVRCRaCoUC8nvnBOMKx3v6oi5FRKYphUIBWblwNj39aR7fpqMFEYmGQqGAtLfNpLosyaot+6MuRUSmKYVCASkvSXLluY08uqVT368gIpFQKBSYlQtn8+LhE2zbfzTqUkRkGlIoFJiVC1MAPPKsTiGJyNRTKBSYpvpKLjirlp9vejnqUkRkGspbKJhZi5mtMrNNZrbRzG4dYp1FZvZbMztlZn+er1qKzZsvnMuanYd46fCJqEsRkWkmn0cKfcDH3X0xcDnwUTNbPGidg8DHgL/PYx1F57oLzwLgJ+v3RlyJiEw3eQsFd9/r7mvD5SPAZqB50Dr73f13QG++6ihGC1I1LD6rlh8rFERkik3JmIKZtQHLgNXj3P4WM+sws47Ozs7JLK1gvfmis3hq12F2dh2LuhQRmUbyHgpmVgM8ANzm7t3j2Ye73+3u7e7enkqlJrfAAvW2Zc2Ywf0de6IuRUSmkbyGgpmVEgTCfe7+YD5fK26a6it5/fkpvrdmD/1pfZBNRKZGPq8+MuAeYLO735mv14mzd7W3sK/7JL96bnqcMhOR6JXkcd9XAjcC681sXdh2O9AK4O53mdlcoAOoBdJmdhuweLynmeLm2gvmMKu6jG8/uYurF82OuhwRmQbyFgru/hhgo6yzDzg7XzUUu7KSBH+8vJUvP7qNHQeOMb+xOuqSRCTm9InmAvf+35tHaSLBPY9tj7oUEZkGFAoFbvaMCt62rJn7O/Zw4OipqMsRkZhTKBSBW16/gN7+NF9etS3qUkQk5hQKReCcVA3vfE0L9z2xiz2HjkddjojEmEKhSNz6hvPA4B8f3hp1KSISYwqFItFUX8kHr2zjgbV7+N0LB6MuR0RiSqFQRG699jya6yu5/cH19PSloy5HRGJIoVBEqspK+PRbl7B1/1H+4eEtUZcjIjGkUCgy1yyaw3tWtPJPv9zOLzX9hYhMMoVCEbrjLYs5f04Nf/bddezq0tVIIjJ5FApFqKI0yVfe9xr63bnpG0/SpQ+1icgkUSgUqXNSNdxzUzsvHT7Bjfc8qU87i8ikUCgUsdfMa+Cr729n+4Gj/NFdv9WpJBGZMIVCkXvd+Sn+5UMrOHD0FG/+4q/52cZ9UZckIkVMoRAD7W0N/Phjr2V+YzX/5VtruPU7T7G/+2TUZYlIEVIoxERLQxX3f+QKPnbtefx0/T5W/v2j/M2PN7HvFYWDBNydvv40PX1p+vrTuOtrXuXVrNh+Mdrb272joyPqMgrajgPH+Px/PMcPn9kLwFXnNvKWi87iynMbaaqvzNvrnuztp/tkL90n+sL7Xl450Uv3yT66T/RmnjvZ209/2kl7cAuWIR22JcxIJIyEQTJhJMxIJoykGWZGMnFme8IMM0iG25kR7CNzb6cfD/O8GaQ9eON0J6wtuPdw2Qc9TrvjhOumnf40mf70h+v159iefZ9OQ/+r2oOfT//Aa4Xt7mT2O1r7UP/Vk4nTP9uShJFMhvdhW/A4kXlcWmKUlyQpSyYoL01QlkxQVpII2koSlIe3spLEoHWSmfbykgTlpUkqwvvyrLbykgQV4X1JIvj3lslhZmvcvX3U9RQK8bX74HG+/eQuHlr3Ei8ePgFAS0Mli+bWck6qhpaGSmZWlVFfWUpFWRIDzIy0Oyd6+jl6qo/jPX0cPdXPkUFv9oPf6LtP9o469UZZMkFtZSmVZYnMG3jCLGs5eJPOvCEPelPLvGEO8WaXzoTMwBt3+IY7zJvhRAyEihHcEwZSMivIBsIqE1yJ06GVHAipRBhw2e3DhOFY2hNZtQzsd3B7eiA00k5f2ulPp+lPQ386HT4O2tOZ552+dJrefudUXz89fcERx6ms+2C5P7jvT0/4554wKC9JUl6aOCMsysOACdqTVJRmtQ0KmjO2KU0MWh4Ioex9nt42boGkUJAMd2fT3m5Wbz/I7144yLb9R3mh6xi9/WP7tw/e1EuorSiltjK8VZSE96VnPjdEe0VpMk89HNnQf/2TCRoP/2If6ihicACYEbs3i3xwD8JkIDSC4OjPCpF+TvUOhEk/J3tPP38qa/lk76vbzmgfYT8TfWs7feQzEDyvDpczAyU7nLK2GRRmFaWDAirctiIrrJKJyf8dyzUU8vYdzVI4zIwlTXUsaarj5qvmA9DXn6brWA+Hjvdw6FgvJ/v6wcFxzIzqshKqypLUlJdQVZ6ktqK0aP96soE395G/MlwmkZlRmjRKkwkon/rXd/fMUc1I4XKqt5+T4X2mLQyak4MC54zte9McPt4z7H760hNLpJKEnREopckEJUnjPctb+fBrF0zST2mY187r3qVglSQTzKmtYE5tRdSliEw6M6OsxCgrSTAjgtfv6w9OoQ2EyhlHNlnhcuYR0vBHQr39aXrTTmNN/hNWoSAiMslKkglKkgmqyqKuZOx0SaqIiGQoFEREJEOhICIiGQoFERHJUCiIiEiGQkFERDIUCiIikqFQEBGRjKKb+8jMOoGd49y8ETgwieUUA/V5elCfp4eJ9Hmeu6dGW6noQmEizKwjlwmh4kR9nh7U5+lhKvqs00ciIpKhUBARkYzpFgp3R11ABNTn6UF9nh7y3udpNaYgIiIjm25HCiIiMoJpEwpm9gdmtsXMtpnZJ6KuZyLM7Otmtt/MNmS1NZjZw2a2NbyfGbabmX0h7PczZnZp1jY3hetvNbObouhLLsysxcxWmdkmM9toZreG7XHuc4WZPWlmT4d9/lTYPt/MVod9+66ZlYXt5eHjbeHzbVn7+mTYvsXM3hhNj3JnZkkze8rMfhQ+jnWfzewFM1tvZuvMrCNsi+53O/j+2njfgCTwPLAAKAOeBhZHXdcE+vM64FJgQ1bbZ4FPhMufAD4TLl8H/BQw4HJgddjeAGwP72eGyzOj7tsw/T0LuDRcngE8ByyOeZ8NqAmXS4HVYV/+DXh32H4X8Cfh8n8F7gqX3w18N1xeHP6+lwPzw/8Hyaj7N0rf/wz4NvCj8HGs+wy8ADQOaovsd3u6HCksB7a5+3Z37wG+A1wfcU3j5u6/Ag4Oar4e+Odw+Z+Bt2a1f9MDTwD1ZnYW8EbgYXc/6O6HgIeBP8h/9WPn7nvdfW24fATYDDQT7z67ux8NH5aGNweuAb4Xtg/u88DP4nvAtRZ8ofb1wHfc/ZS77wC2Efx/KEhmdjbwZuBr4WMj5n0eRmS/29MlFJqB3VmP94RtcTLH3feGy/uAOeHycH0vyp9JeIpgGcFfzrHuc3gaZR2wn+A/+fPAYXfvC1fJrj/Tt/D5V4BZFFmfgc8B/wNIh49nEf8+O/BzM1tjZreEbZH9bus7mmPI3d3MYndZmZnVAA8At7l7d/BHYSCOfXb3fuASM6sHvg8sirikvDKztwD73X2Nma2Mup4pdJW7v2hms4GHzezZ7Cen+nd7uhwpvAi0ZD0+O2yLk5fDw0jC+/1h+3B9L6qfiZmVEgTCfe7+YNgc6z4PcPfDwCrgCoLTBQN/zGXXn+lb+Hwd0EVx9flK4A/N7AWCU7zXAJ8n3n3G3V8M7/cThP9yIvzdni6h8DvgvPAqhjKCQamHIq5psj0EDFxxcBPw/7La3x9etXA58Ep4WPoz4PfNbGZ4ZcPvh20FJzxPfA+w2d3vzHoqzn1OhUcImFkl8J8IxlJWATeEqw3u88DP4gbgEQ9GIB8C3h1eqTMfOA94cmp6MTbu/kl3P9vd2wj+jz7i7u8lxn02s2ozmzGwTPA7uYEof7ejHnmfqhvBqP1zBOdl/zLqeibYl38F9gK9BOcOP0RwLvUXwFbgP4CGcF0Dvhz2ez3QnrWfmwkG4bYBH4y6XyP09yqC867PAOvC23Ux7/NFwFNhnzcAd4TtCwje4LYB9wPlYXtF+Hhb+PyCrH39Zfiz2AK8Keq+5dj/lZy++ii2fQ779nR42zjw3hTl77Y+0SwiIhnT5fSRiIjkQKEgIiIZCgUREclQKIiISIZCQUREMhQKEntm9rdmdrWZvdXMPjnGbVPhDJxPmdlr81XjMK99dPS1RCaXQkGmgxXAE8DrgV+NcdtrgfXuvszdfz3plYkUGIWCxJaZ/Z2ZPQNcBvwW+DDwFTO7Y4h128zskXCO+l+YWauZXUIwhfH14Vz3lYO2eY2Z/TKcyOxnWdMSPGpmnw+32WBmy8P2BjP7QfgaT5jZRWF7jZl9w4I59Z8xs3dkvcbfWPCdCk+Y2Zyw7Z3hfp82s7GGnMjIov5En2665fNGEAhfJJh6+jcjrPdD4KZw+WbgB+HyB4AvDbF+KfA4kAofvwv4erj8KPDVcPl1hN97EdbxV+HyNcC6cPkzwOey9j0zvHfgP4fLnwX+Z7i8HmgOl+uj/hnrFq+bZkmVuLuUYAqBRQRzBw3nCuDt4fK3CN6ER7IQWEowqyUEX+S0N+v5f4Xguy/MrDacx+gq4B1h+yNmNsvMaoE3EMz1Q/jcoXCxB/hRuLyGYP4jgN8A95rZvwEDkwOKTAqFgsRSeOrnXoLZIg8AVUGzrQOucPcTE30JYKO7XzHM84PnjxnPfDK97j6wXT/h/1d3/4iZrSD4Mpo1ZvYad+8ax/5FXkVjChJL7r7O3S/h9Fd3PgK80d0vGSYQHuf0X+vvBUYbVN4CpMzsCgim9jazJVnPvytsv4pgJstXwn2+N2xfCRxw926CL9D56MCG4SyXwzKzc9x9tbvfAXRy5pTJIhOiIwWJLTNLAYfcPW1mi9x90wir/ynwDTP77wRvtB8cad/u3mNmNwBfMLM6gv9LnyOY6RLgpJk9RTD2cHPY9r+Ar4eD38c5PTXy/wa+bGYbCI4IPsXIp4X+zszOIzha+QXB6TGRSaFZUkUmmZk9Cvy5u3dEXYvIWOn0kYiIZOhIQUREMnSkICIiGQoFERHJUCiIiEiGQkFERDIUCiIikqFQEBGRjP8Pa2LN6LHXK3EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108091710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "paras = skipgram_model_training(X, Y, vocab_size, 50, 0.05, 5000, batch_size=128, parameters=None, print_cost=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = np.arange(vocab_size)\n",
    "X_test = np.expand_dims(X_test, axis=0)\n",
    "softmax_test, _ = forward_propagation(X_test, paras)\n",
    "top_sorted_inds = np.argsort(softmax_test, axis=0)[-4:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "market's neighbor words: ['is', \"loser's\", 'a', 'beating']\n",
      "of's neighbor words: ['the', 'of', 'costs', 'beating']\n",
      "costs's neighbor words: ['beating', 'of', 'deduction', 'the']\n",
      "investing's neighbor words: ['the', 'stock', 'beating', \"loser's\"]\n",
      "a's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "loser's's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "deduction's neighbor words: ['costs', 'after', 'the', 'beating']\n",
      "after's neighbor words: ['of', 'the', 'deduction', 'costs']\n",
      "game's neighbor words: ['is', \"loser's\", 'a', 'beating']\n",
      "stock's neighbor words: ['investing', 'a', 'is', 'beating']\n",
      "the's neighbor words: ['is', 'of', 'beating', 'stock']\n",
      "is's neighbor words: ['game', 'market', \"loser's\", 'stock']\n",
      "beating's neighbor words: ['market', 'stock', 'investing', 'costs']\n"
     ]
    }
   ],
   "source": [
    "for input_ind in range(vocab_size):\n",
    "    input_word = id_to_word[input_ind]\n",
    "    output_words = [id_to_word[output_ind] for output_ind in top_sorted_inds[::-1, input_ind]]\n",
    "    print(\"{}'s neighbor words: {}\".format(input_word, output_words))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}