Predict_Next_Word.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FInal Predict Next Word.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AnupKumarGupta/Machine-Learning-Lab-Assignments/blob/master/Predict_Next_Word.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLPrbhrZ5ha3",
        "colab_type": "code",
        "outputId": "b23db623-5022-4b2a-e8b1-e3f432995e24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "from __future__ import print_function\n",
        "import numpy as np\n",
        "import gensim\n",
        "import string\n",
        "import tensorflow.compat.v1 as tf\n",
        "tf.disable_v2_behavior()\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "\n",
        "import logging\n",
        "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:68: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "non-resource variables are not supported in the long term\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s9n45Rdz52Cw",
        "colab_type": "code",
        "outputId": "59aa4e9b-8a8c-4ae6-bc74-8696911a6def",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Path to the dataset\n",
        "path = \"/content/sherlock_book.txt\"\n",
        "\n",
        "# Data Cleaning and Preprocessing \n",
        "\n",
        "# Restriction on the maximum length of sentence in terms of number of words\n",
        "max_sentence_len = 40\n",
        "\n",
        "# Open data file and read the data as lines\n",
        "with open(path) as file_:\n",
        "  docs = file_.readlines()\n",
        "\n",
        "# Preprocessing the data\n",
        "# 1. Converting the words into lower cases\n",
        "# 2. Removing punctuation marks\n",
        "# 3. Spliting the sentence into two if the max length exceeds\n",
        "# 4. Remove the sentences which are empty or too short \n",
        "sentences = [[word for word in doc.lower().translate(str.maketrans('','',string.punctuation)).split()[:max_sentence_len]] for doc in docs]\n",
        "sentences = [item for item in sentences if len(item) > 2]\n",
        "print('Num of sentences:', len(sentences))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Num of sentences: 57351\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdyFtEeS70-d",
        "colab_type": "code",
        "outputId": "68a6a381-1ee0-4e90-9d78-02312bce5503",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Training the Word2Vec model\n",
        "word_model = gensim.models.Word2Vec(sentences, size=100, min_count=1, window=5, iter=100)\n",
        "pretrained_weights = word_model.wv.vectors\n",
        "vocab_size, emdedding_size = pretrained_weights.shape\n",
        "\n",
        "# Embeddings shape will be : word count X size of vector\n",
        "print('Result embedding shape:', pretrained_weights.shape)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-11-16 12:51:17,140 : INFO : collecting all words and their counts\n",
            "2019-11-16 12:51:17,141 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
            "2019-11-16 12:51:17,171 : INFO : PROGRESS: at sentence #10000, processed 114642 words, keeping 10118 word types\n",
            "2019-11-16 12:51:17,200 : INFO : PROGRESS: at sentence #20000, processed 230211 words, keeping 13760 word types\n",
            "2019-11-16 12:51:17,227 : INFO : PROGRESS: at sentence #30000, processed 344100 words, keeping 16512 word types\n",
            "2019-11-16 12:51:17,254 : INFO : PROGRESS: at sentence #40000, processed 456890 words, keeping 18735 word types\n",
            "2019-11-16 12:51:17,281 : INFO : PROGRESS: at sentence #50000, processed 570396 words, keeping 20963 word types\n",
            "2019-11-16 12:51:17,304 : INFO : collected 22707 word types from a corpus of 653029 raw words and 57351 sentences\n",
            "2019-11-16 12:51:17,305 : INFO : Loading a fresh vocabulary\n",
            "2019-11-16 12:51:17,489 : INFO : effective_min_count=1 retains 22707 unique words (100% of original 22707, drops 0)\n",
            "2019-11-16 12:51:17,490 : INFO : effective_min_count=1 leaves 653029 word corpus (100% of original 653029, drops 0)\n",
            "2019-11-16 12:51:17,571 : INFO : deleting the raw counts dictionary of 22707 items\n",
            "2019-11-16 12:51:17,573 : INFO : sample=0.001 downsamples 59 most-common words\n",
            "2019-11-16 12:51:17,574 : INFO : downsampling leaves estimated 465875 word corpus (71.3% of prior 653029)\n",
            "2019-11-16 12:51:17,655 : INFO : estimated required memory for 22707 words and 100 dimensions: 29519100 bytes\n",
            "2019-11-16 12:51:17,656 : INFO : resetting layer weights\n",
            "2019-11-16 12:51:22,318 : INFO : training model with 3 workers on 22707 vocabulary and 100 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
            "2019-11-16 12:51:23,188 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:23,190 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:23,206 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:23,207 : INFO : EPOCH - 1 : training on 653029 raw words (465771 effective words) took 0.9s, 530863 effective words/s\n",
            "2019-11-16 12:51:24,074 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:24,086 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:24,102 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:24,103 : INFO : EPOCH - 2 : training on 653029 raw words (465988 effective words) took 0.9s, 527073 effective words/s\n",
            "2019-11-16 12:51:25,048 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:25,058 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:25,077 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:25,078 : INFO : EPOCH - 3 : training on 653029 raw words (465758 effective words) took 1.0s, 481760 effective words/s\n",
            "2019-11-16 12:51:26,049 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:26,052 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:26,062 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:26,063 : INFO : EPOCH - 4 : training on 653029 raw words (465726 effective words) took 1.0s, 476219 effective words/s\n",
            "2019-11-16 12:51:26,934 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:26,947 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:26,955 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:26,956 : INFO : EPOCH - 5 : training on 653029 raw words (466058 effective words) took 0.9s, 529623 effective words/s\n",
            "2019-11-16 12:51:27,886 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:27,891 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:27,903 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:27,904 : INFO : EPOCH - 6 : training on 653029 raw words (465532 effective words) took 0.9s, 499327 effective words/s\n",
            "2019-11-16 12:51:28,880 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:28,882 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:28,900 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:28,901 : INFO : EPOCH - 7 : training on 653029 raw words (465819 effective words) took 1.0s, 475684 effective words/s\n",
            "2019-11-16 12:51:29,796 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:29,803 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:29,806 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:29,810 : INFO : EPOCH - 8 : training on 653029 raw words (465891 effective words) took 0.9s, 531105 effective words/s\n",
            "2019-11-16 12:51:30,711 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:30,717 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:30,730 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:30,731 : INFO : EPOCH - 9 : training on 653029 raw words (465860 effective words) took 0.9s, 512510 effective words/s\n",
            "2019-11-16 12:51:31,666 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:31,670 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:31,687 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:31,688 : INFO : EPOCH - 10 : training on 653029 raw words (466142 effective words) took 0.9s, 492818 effective words/s\n",
            "2019-11-16 12:51:32,649 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:32,651 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:32,663 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:32,665 : INFO : EPOCH - 11 : training on 653029 raw words (465863 effective words) took 1.0s, 482514 effective words/s\n",
            "2019-11-16 12:51:33,614 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:33,622 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:33,641 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:33,642 : INFO : EPOCH - 12 : training on 653029 raw words (465669 effective words) took 1.0s, 482518 effective words/s\n",
            "2019-11-16 12:51:34,595 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:34,607 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:34,623 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:34,624 : INFO : EPOCH - 13 : training on 653029 raw words (465663 effective words) took 1.0s, 482950 effective words/s\n",
            "2019-11-16 12:51:35,577 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:35,584 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:35,595 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:35,596 : INFO : EPOCH - 14 : training on 653029 raw words (465429 effective words) took 1.0s, 483350 effective words/s\n",
            "2019-11-16 12:51:36,558 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:36,560 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:36,570 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:36,572 : INFO : EPOCH - 15 : training on 653029 raw words (465857 effective words) took 1.0s, 481470 effective words/s\n",
            "2019-11-16 12:51:37,550 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:37,561 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:37,567 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:37,569 : INFO : EPOCH - 16 : training on 653029 raw words (465749 effective words) took 1.0s, 475498 effective words/s\n",
            "2019-11-16 12:51:38,509 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:38,511 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:38,527 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:38,527 : INFO : EPOCH - 17 : training on 653029 raw words (466195 effective words) took 0.9s, 491443 effective words/s\n",
            "2019-11-16 12:51:39,388 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:39,406 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:39,407 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:39,408 : INFO : EPOCH - 18 : training on 653029 raw words (466373 effective words) took 0.9s, 533874 effective words/s\n",
            "2019-11-16 12:51:40,352 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:40,374 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:40,377 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:40,379 : INFO : EPOCH - 19 : training on 653029 raw words (466002 effective words) took 1.0s, 489528 effective words/s\n",
            "2019-11-16 12:51:41,322 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:41,332 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:41,347 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:41,349 : INFO : EPOCH - 20 : training on 653029 raw words (465765 effective words) took 1.0s, 487218 effective words/s\n",
            "2019-11-16 12:51:42,287 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:42,298 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:42,309 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:42,310 : INFO : EPOCH - 21 : training on 653029 raw words (466043 effective words) took 0.9s, 493268 effective words/s\n",
            "2019-11-16 12:51:43,251 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:43,266 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:43,279 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:43,281 : INFO : EPOCH - 22 : training on 653029 raw words (465798 effective words) took 1.0s, 486540 effective words/s\n",
            "2019-11-16 12:51:44,253 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:44,275 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:44,281 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:44,282 : INFO : EPOCH - 23 : training on 653029 raw words (465930 effective words) took 1.0s, 468476 effective words/s\n",
            "2019-11-16 12:51:45,240 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:45,246 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:45,256 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:45,257 : INFO : EPOCH - 24 : training on 653029 raw words (466247 effective words) took 1.0s, 484982 effective words/s\n",
            "2019-11-16 12:51:46,118 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:46,129 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:46,135 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:46,136 : INFO : EPOCH - 25 : training on 653029 raw words (466003 effective words) took 0.9s, 533893 effective words/s\n",
            "2019-11-16 12:51:47,066 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:47,086 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:47,092 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:47,093 : INFO : EPOCH - 26 : training on 653029 raw words (465524 effective words) took 0.9s, 493619 effective words/s\n",
            "2019-11-16 12:51:47,957 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:47,970 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:47,973 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:47,977 : INFO : EPOCH - 27 : training on 653029 raw words (466057 effective words) took 0.9s, 533899 effective words/s\n",
            "2019-11-16 12:51:48,887 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:48,896 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:48,911 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:48,912 : INFO : EPOCH - 28 : training on 653029 raw words (466029 effective words) took 0.9s, 506781 effective words/s\n",
            "2019-11-16 12:51:49,854 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:49,866 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:49,868 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:49,869 : INFO : EPOCH - 29 : training on 653029 raw words (465975 effective words) took 0.9s, 495137 effective words/s\n",
            "2019-11-16 12:51:50,771 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:50,775 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:50,782 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:50,785 : INFO : EPOCH - 30 : training on 653029 raw words (466011 effective words) took 0.9s, 516613 effective words/s\n",
            "2019-11-16 12:51:51,710 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:51,713 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:51,715 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:51,717 : INFO : EPOCH - 31 : training on 653029 raw words (465880 effective words) took 0.9s, 507439 effective words/s\n",
            "2019-11-16 12:51:52,648 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:52,659 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:52,661 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:52,662 : INFO : EPOCH - 32 : training on 653029 raw words (465907 effective words) took 0.9s, 500764 effective words/s\n",
            "2019-11-16 12:51:53,588 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:53,599 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:53,615 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:53,616 : INFO : EPOCH - 33 : training on 653029 raw words (466015 effective words) took 0.9s, 495611 effective words/s\n",
            "2019-11-16 12:51:54,541 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:54,553 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:54,568 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:54,569 : INFO : EPOCH - 34 : training on 653029 raw words (465686 effective words) took 0.9s, 496782 effective words/s\n",
            "2019-11-16 12:51:55,501 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:55,509 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:55,519 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:55,520 : INFO : EPOCH - 35 : training on 653029 raw words (465786 effective words) took 0.9s, 496992 effective words/s\n",
            "2019-11-16 12:51:56,494 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:56,504 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:56,508 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:56,509 : INFO : EPOCH - 36 : training on 653029 raw words (465926 effective words) took 1.0s, 476942 effective words/s\n",
            "2019-11-16 12:51:57,464 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:57,465 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:57,481 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:57,482 : INFO : EPOCH - 37 : training on 653029 raw words (465429 effective words) took 1.0s, 485187 effective words/s\n",
            "2019-11-16 12:51:58,431 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:58,441 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:58,459 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:58,460 : INFO : EPOCH - 38 : training on 653029 raw words (466534 effective words) took 1.0s, 485830 effective words/s\n",
            "2019-11-16 12:51:59,394 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:51:59,405 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:51:59,409 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:51:59,411 : INFO : EPOCH - 39 : training on 653029 raw words (465574 effective words) took 0.9s, 494163 effective words/s\n",
            "2019-11-16 12:52:00,342 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:00,353 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:00,370 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:00,372 : INFO : EPOCH - 40 : training on 653029 raw words (465952 effective words) took 0.9s, 493657 effective words/s\n",
            "2019-11-16 12:52:01,308 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:01,313 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:01,324 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:01,326 : INFO : EPOCH - 41 : training on 653029 raw words (465714 effective words) took 0.9s, 496880 effective words/s\n",
            "2019-11-16 12:52:02,258 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:02,264 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:02,276 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:02,277 : INFO : EPOCH - 42 : training on 653029 raw words (465960 effective words) took 0.9s, 496036 effective words/s\n",
            "2019-11-16 12:52:03,213 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:03,220 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:03,229 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:03,231 : INFO : EPOCH - 43 : training on 653029 raw words (466203 effective words) took 0.9s, 498190 effective words/s\n",
            "2019-11-16 12:52:04,175 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:04,190 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:04,209 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:04,210 : INFO : EPOCH - 44 : training on 653029 raw words (465547 effective words) took 1.0s, 481521 effective words/s\n",
            "2019-11-16 12:52:05,170 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:05,183 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:05,189 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:05,191 : INFO : EPOCH - 45 : training on 653029 raw words (465956 effective words) took 1.0s, 483795 effective words/s\n",
            "2019-11-16 12:52:06,162 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:06,174 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:06,178 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:06,179 : INFO : EPOCH - 46 : training on 653029 raw words (465949 effective words) took 1.0s, 480011 effective words/s\n",
            "2019-11-16 12:52:07,124 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:07,131 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:07,141 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:07,143 : INFO : EPOCH - 47 : training on 653029 raw words (466351 effective words) took 0.9s, 491228 effective words/s\n",
            "2019-11-16 12:52:08,097 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:08,100 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:08,119 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:08,120 : INFO : EPOCH - 48 : training on 653029 raw words (465964 effective words) took 1.0s, 482221 effective words/s\n",
            "2019-11-16 12:52:09,061 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:09,076 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:09,078 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:09,080 : INFO : EPOCH - 49 : training on 653029 raw words (466397 effective words) took 0.9s, 494698 effective words/s\n",
            "2019-11-16 12:52:10,031 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:10,036 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:10,053 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:10,054 : INFO : EPOCH - 50 : training on 653029 raw words (465785 effective words) took 1.0s, 486840 effective words/s\n",
            "2019-11-16 12:52:11,044 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:11,058 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:11,061 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:11,062 : INFO : EPOCH - 51 : training on 653029 raw words (466197 effective words) took 1.0s, 474387 effective words/s\n",
            "2019-11-16 12:52:12,026 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:12,031 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:12,048 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:12,049 : INFO : EPOCH - 52 : training on 653029 raw words (466129 effective words) took 1.0s, 479109 effective words/s\n",
            "2019-11-16 12:52:13,013 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:13,024 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:13,029 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:13,031 : INFO : EPOCH - 53 : training on 653029 raw words (466274 effective words) took 1.0s, 480175 effective words/s\n",
            "2019-11-16 12:52:14,020 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:14,022 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:14,034 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:14,037 : INFO : EPOCH - 54 : training on 653029 raw words (466155 effective words) took 1.0s, 469282 effective words/s\n",
            "2019-11-16 12:52:14,977 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:14,986 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:14,992 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:14,993 : INFO : EPOCH - 55 : training on 653029 raw words (466003 effective words) took 0.9s, 491309 effective words/s\n",
            "2019-11-16 12:52:15,948 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:15,960 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:15,966 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:15,968 : INFO : EPOCH - 56 : training on 653029 raw words (465849 effective words) took 1.0s, 483896 effective words/s\n",
            "2019-11-16 12:52:16,932 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:16,936 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:16,938 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:16,942 : INFO : EPOCH - 57 : training on 653029 raw words (466069 effective words) took 1.0s, 484679 effective words/s\n",
            "2019-11-16 12:52:17,888 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:17,891 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:17,905 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:17,906 : INFO : EPOCH - 58 : training on 653029 raw words (466522 effective words) took 1.0s, 490967 effective words/s\n",
            "2019-11-16 12:52:18,839 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:18,851 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:18,856 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:18,857 : INFO : EPOCH - 59 : training on 653029 raw words (466189 effective words) took 0.9s, 496296 effective words/s\n",
            "2019-11-16 12:52:19,807 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:19,811 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:19,826 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:19,828 : INFO : EPOCH - 60 : training on 653029 raw words (466036 effective words) took 1.0s, 489598 effective words/s\n",
            "2019-11-16 12:52:20,780 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:20,794 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:20,803 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:20,804 : INFO : EPOCH - 61 : training on 653029 raw words (466049 effective words) took 1.0s, 484527 effective words/s\n",
            "2019-11-16 12:52:21,755 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:21,763 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:21,782 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:21,784 : INFO : EPOCH - 62 : training on 653029 raw words (465754 effective words) took 1.0s, 479692 effective words/s\n",
            "2019-11-16 12:52:22,713 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:22,719 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:22,724 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:22,728 : INFO : EPOCH - 63 : training on 653029 raw words (466311 effective words) took 0.9s, 501509 effective words/s\n",
            "2019-11-16 12:52:23,625 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:23,627 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:23,631 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:23,633 : INFO : EPOCH - 64 : training on 653029 raw words (465689 effective words) took 0.9s, 521122 effective words/s\n",
            "2019-11-16 12:52:24,601 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:24,605 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:24,620 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:24,621 : INFO : EPOCH - 65 : training on 653029 raw words (465911 effective words) took 1.0s, 480088 effective words/s\n",
            "2019-11-16 12:52:25,555 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:25,578 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:25,585 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:25,586 : INFO : EPOCH - 66 : training on 653029 raw words (465840 effective words) took 0.9s, 491156 effective words/s\n",
            "2019-11-16 12:52:26,537 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:26,539 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:26,548 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:26,549 : INFO : EPOCH - 67 : training on 653029 raw words (465915 effective words) took 1.0s, 488436 effective words/s\n",
            "2019-11-16 12:52:27,485 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:27,494 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:27,496 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:27,497 : INFO : EPOCH - 68 : training on 653029 raw words (466188 effective words) took 0.9s, 498163 effective words/s\n",
            "2019-11-16 12:52:28,434 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:28,436 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:28,447 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:28,448 : INFO : EPOCH - 69 : training on 653029 raw words (465693 effective words) took 0.9s, 499677 effective words/s\n",
            "2019-11-16 12:52:29,399 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:29,411 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:29,413 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:29,415 : INFO : EPOCH - 70 : training on 653029 raw words (466052 effective words) took 0.9s, 491522 effective words/s\n",
            "2019-11-16 12:52:30,344 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:30,346 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:30,351 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:30,353 : INFO : EPOCH - 71 : training on 653029 raw words (465756 effective words) took 0.9s, 505454 effective words/s\n",
            "2019-11-16 12:52:31,270 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:31,272 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:31,284 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:31,286 : INFO : EPOCH - 72 : training on 653029 raw words (465618 effective words) took 0.9s, 506180 effective words/s\n",
            "2019-11-16 12:52:32,218 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:32,225 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:32,242 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:32,243 : INFO : EPOCH - 73 : training on 653029 raw words (465662 effective words) took 0.9s, 492405 effective words/s\n",
            "2019-11-16 12:52:33,171 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:33,179 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:33,181 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:33,182 : INFO : EPOCH - 74 : training on 653029 raw words (466023 effective words) took 0.9s, 500198 effective words/s\n",
            "2019-11-16 12:52:34,028 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:34,034 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:34,039 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:34,043 : INFO : EPOCH - 75 : training on 653029 raw words (465707 effective words) took 0.8s, 549892 effective words/s\n",
            "2019-11-16 12:52:34,958 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:34,959 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:34,977 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:34,979 : INFO : EPOCH - 76 : training on 653029 raw words (465725 effective words) took 0.9s, 503328 effective words/s\n",
            "2019-11-16 12:52:35,923 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:35,925 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:35,935 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:35,937 : INFO : EPOCH - 77 : training on 653029 raw words (465709 effective words) took 0.9s, 492949 effective words/s\n",
            "2019-11-16 12:52:36,865 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:36,873 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:36,880 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:36,881 : INFO : EPOCH - 78 : training on 653029 raw words (465582 effective words) took 0.9s, 503436 effective words/s\n",
            "2019-11-16 12:52:37,823 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:37,825 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:37,839 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:37,840 : INFO : EPOCH - 79 : training on 653029 raw words (466091 effective words) took 0.9s, 494164 effective words/s\n",
            "2019-11-16 12:52:38,780 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:38,788 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:38,794 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:38,795 : INFO : EPOCH - 80 : training on 653029 raw words (466264 effective words) took 0.9s, 491759 effective words/s\n",
            "2019-11-16 12:52:39,739 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:39,742 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:39,752 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:39,753 : INFO : EPOCH - 81 : training on 653029 raw words (466189 effective words) took 1.0s, 490335 effective words/s\n",
            "2019-11-16 12:52:40,695 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:40,703 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:40,716 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:40,717 : INFO : EPOCH - 82 : training on 653029 raw words (465914 effective words) took 0.9s, 493235 effective words/s\n",
            "2019-11-16 12:52:41,638 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:41,640 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:41,651 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:41,651 : INFO : EPOCH - 83 : training on 653029 raw words (465517 effective words) took 0.9s, 501999 effective words/s\n",
            "2019-11-16 12:52:42,491 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:42,499 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:42,514 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:42,515 : INFO : EPOCH - 84 : training on 653029 raw words (466078 effective words) took 0.8s, 548795 effective words/s\n",
            "2019-11-16 12:52:43,372 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:43,377 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:43,382 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:43,383 : INFO : EPOCH - 85 : training on 653029 raw words (465801 effective words) took 0.9s, 543342 effective words/s\n",
            "2019-11-16 12:52:44,320 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:44,327 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:44,346 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:44,348 : INFO : EPOCH - 86 : training on 653029 raw words (466286 effective words) took 0.9s, 493925 effective words/s\n",
            "2019-11-16 12:52:45,287 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:45,289 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:45,307 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:45,308 : INFO : EPOCH - 87 : training on 653029 raw words (465472 effective words) took 0.9s, 493081 effective words/s\n",
            "2019-11-16 12:52:46,246 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:46,265 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:46,277 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:46,278 : INFO : EPOCH - 88 : training on 653029 raw words (466159 effective words) took 1.0s, 486346 effective words/s\n",
            "2019-11-16 12:52:47,206 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:47,211 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:47,227 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:47,229 : INFO : EPOCH - 89 : training on 653029 raw words (465729 effective words) took 0.9s, 495949 effective words/s\n",
            "2019-11-16 12:52:48,175 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:48,180 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:48,189 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:48,191 : INFO : EPOCH - 90 : training on 653029 raw words (466277 effective words) took 0.9s, 491389 effective words/s\n",
            "2019-11-16 12:52:49,123 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:49,135 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:49,151 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:49,152 : INFO : EPOCH - 91 : training on 653029 raw words (466055 effective words) took 0.9s, 491403 effective words/s\n",
            "2019-11-16 12:52:50,085 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:50,087 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:50,103 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:50,104 : INFO : EPOCH - 92 : training on 653029 raw words (465904 effective words) took 0.9s, 498013 effective words/s\n",
            "2019-11-16 12:52:51,049 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:51,061 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:51,065 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:51,069 : INFO : EPOCH - 93 : training on 653029 raw words (465852 effective words) took 1.0s, 488700 effective words/s\n",
            "2019-11-16 12:52:52,019 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:52,027 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:52,032 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:52,033 : INFO : EPOCH - 94 : training on 653029 raw words (465772 effective words) took 1.0s, 489794 effective words/s\n",
            "2019-11-16 12:52:52,970 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:52,981 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:52,989 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:52,990 : INFO : EPOCH - 95 : training on 653029 raw words (465569 effective words) took 0.9s, 493840 effective words/s\n",
            "2019-11-16 12:52:53,946 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:53,948 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:53,958 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:53,960 : INFO : EPOCH - 96 : training on 653029 raw words (465968 effective words) took 1.0s, 486435 effective words/s\n",
            "2019-11-16 12:52:54,891 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:54,893 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:54,897 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:54,898 : INFO : EPOCH - 97 : training on 653029 raw words (465865 effective words) took 0.9s, 500905 effective words/s\n",
            "2019-11-16 12:52:55,845 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:55,847 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:55,851 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:55,852 : INFO : EPOCH - 98 : training on 653029 raw words (465839 effective words) took 0.9s, 496503 effective words/s\n",
            "2019-11-16 12:52:56,827 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:56,838 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:56,846 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:56,847 : INFO : EPOCH - 99 : training on 653029 raw words (466067 effective words) took 1.0s, 473140 effective words/s\n",
            "2019-11-16 12:52:57,802 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
            "2019-11-16 12:52:57,816 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
            "2019-11-16 12:52:57,831 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
            "2019-11-16 12:52:57,832 : INFO : EPOCH - 100 : training on 653029 raw words (465765 effective words) took 1.0s, 478098 effective words/s\n",
            "2019-11-16 12:52:57,833 : INFO : training on a 65302900 raw words (46592282 effective words) took 95.5s, 487807 effective words/s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result embedding shape: (22707, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lubkEgR692Fl",
        "colab_type": "code",
        "outputId": "e0b7bf3d-aaed-47ac-c2eb-ce08ef5cf221",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "# Saving the model \n",
        "model_path = \"sherlock_model\"\n",
        "word_model.save(model_path)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-11-16 12:52:57,846 : INFO : saving Word2Vec object under sherlock_model, separately None\n",
            "2019-11-16 12:52:57,848 : INFO : not storing attribute vectors_norm\n",
            "2019-11-16 12:52:57,852 : INFO : not storing attribute cum_table\n",
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n",
            "2019-11-16 12:52:58,212 : INFO : saved sherlock_model\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w6TDN7Y5IPgs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Retriving 'topn' words similar to 'word'\n",
        "def get_most_similar(word,topn):\n",
        "  try:\n",
        "    print(word_model.most_similar(word,topn=topn))\n",
        "  except:\n",
        "    print(\"Word <\",word,\"> not in vocabulary\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ho_mEYCBI2Is",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Method to return index of 'word' in the word2vec model\n",
        "# Returns -1 if the 'word' is not present in the vocab \n",
        "def word2idx(word):\n",
        "  try:\n",
        "   index = word_model.wv.vocab[word].index\n",
        "  except:\n",
        "   index = -1\n",
        "  return index   \n",
        "\n",
        "# Method to return 'word' at a particular index in the word2vec model\n",
        "# Returns empty string if the index is out of bounds\n",
        "def idx2word(idx):\n",
        "  try:\n",
        "    word = word_model.wv.index2word[idx]\n",
        "  except:\n",
        "    word = \"\"  \n",
        "  return word"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzU65KYdI4Y5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Preparing the input and coresspondings output for training the LSTMs\n",
        "train_x = np.zeros([len(sentences), max_sentence_len], dtype=np.int32)\n",
        "train_y = np.zeros([len(sentences)], dtype=np.int32)\n",
        "for i, sentence in enumerate(sentences):\n",
        "  for t, word in enumerate(sentence[:-1]):\n",
        "    train_x[i, t] = word2idx(word)\n",
        "  train_y[i] = word2idx(sentence[-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmLX-e-kI4S7",
        "colab_type": "code",
        "outputId": "09d38ab1-f19a-4490-8d29-8e620b5c1df8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "source": [
        "# LSTM model \n",
        "lstm_model = tf.keras.models.Sequential()\n",
        "lstm_model.add(tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=emdedding_size, weights=[pretrained_weights]))\n",
        "lstm_model.add(tf.keras.layers.LSTM(units=emdedding_size))\n",
        "lstm_model.add(tf.keras.layers.Dense(units=vocab_size))\n",
        "lstm_model.add(tf.keras.layers.Activation('softmax'))\n",
        "lstm_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-16 12:52:58,744 : WARNING : From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-16 12:52:58,774 : WARNING : From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zu7Sj0xdJNE-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Method to generate next samples using a temprature, \n",
        "# instead of vanila argmax\n",
        "def sample(preds, temperature=1.0):\n",
        "  if temperature <= 0:\n",
        "    return np.argmax(preds)\n",
        "  preds = np.asarray(preds).astype('float64')\n",
        "  preds = np.log(preds) / temperature\n",
        "  exp_preds = np.exp(preds)\n",
        "  preds = exp_preds / np.sum(exp_preds)\n",
        "  probas = np.random.multinomial(1, preds, 1)\n",
        "  return np.argmax(probas)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MwwDsQPjJSEI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Method to generate next 'count' words if a 'text' is provided\n",
        "def generate_next(lstm_model, text, count=10):\n",
        "  word_idxs = [word2idx(word) for word in text.lower().split()]\n",
        "  for i in range(count):\n",
        "    prediction = lstm_model.predict(x=np.array(word_idxs))\n",
        "    idx = sample(prediction[-1], temperature=0.7)\n",
        "    word_idxs.append(idx)  \n",
        "  return ' '.join(idx2word(idx) for idx in word_idxs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6zip0i7JVwE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Creating a callback to save the LSTM model after every epoch\n",
        "filepath=\"weights-improvement-{epoch:02d}-.hdf5\"\n",
        "checkpoint = ModelCheckpoint(filepath,verbose=1, save_best_only=False, save_weights_only=False, mode='auto')\n",
        "callbacks_list = [checkpoint]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aXqXRLCpJX6_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Training the LSTM model\n",
        "lstm_model.fit(train_x, train_y,\n",
        "          batch_size=128,\n",
        "          epochs=25,\n",
        "          callbacks =callbacks_list, verbose=1)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}