Visual_Embeddings_Newsgroup.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Visual Newsgroup.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "E1MBXjyvmLMg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 972
        },
        "outputId": "bcfeb0d4-ad6c-4808-f922-1e4cdf91d89f"
      },
      "cell_type": "code",
      "source": [
        "!pip install -U keras gensim \n",
        "!pip install -U -q PyDrive"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting keras\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/34/7d/b1dedde8af99bd82f20ed7e9697aac0597de3049b1f786aa2aac3b9bd4da/Keras-2.2.2-py2.py3-none-any.whl (299kB)\n",
            "\u001b[K    100% |████████████████████████████████| 307kB 5.4MB/s \n",
            "\u001b[?25hCollecting gensim\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/e4/1c763ebd2141d0adcb1e31e24d296f0e6451c5451f250bd988d2e8ee6d95/gensim-3.5.0-cp27-cp27mu-manylinux1_x86_64.whl (23.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 23.5MB 1.0MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy>=1.9.1 in /usr/local/lib/python2.7/dist-packages (from keras) (1.14.5)\n",
            "Requirement already satisfied, skipping upgrade: scipy>=0.14 in /usr/local/lib/python2.7/dist-packages (from keras) (0.19.1)\n",
            "Collecting keras-preprocessing==1.0.2 (from keras)\n",
            "  Downloading https://files.pythonhosted.org/packages/71/26/1e778ebd737032749824d5cba7dbd3b0cf9234b87ab5ec79f5f0403ca7e9/Keras_Preprocessing-1.0.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python2.7/dist-packages (from keras) (2.8.0)\n",
            "Collecting keras-applications==1.0.4 (from keras)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/54/90/8f327deaa37a71caddb59b7b4aaa9d4b3e90c0e76f8c2d1572005278ddc5/Keras_Applications-1.0.4-py2.py3-none-any.whl (43kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 10.9MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: pyyaml in /usr/local/lib/python2.7/dist-packages (from keras) (3.13)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.9.0 in /usr/local/lib/python2.7/dist-packages (from keras) (1.11.0)\n",
            "Collecting smart-open>=1.2.1 (from gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/cf/3d/5f3a9a296d0ba8e00e263a8dee76762076b9eb5ddc254ccaa834651c8d65/smart_open-1.6.0.tar.gz\n",
            "Collecting boto>=2.32 (from smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/23/10/c0b78c27298029e4454a472a1919bde20cb182dab1662cec7f2ca1dcc523/boto-2.49.0-py2.py3-none-any.whl (1.4MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.4MB 4.2MB/s \n",
            "\u001b[?25hCollecting bz2file (from smart-open>=1.2.1->gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/61/39/122222b5e85cd41c391b68a99ee296584b2a2d1d233e7ee32b4532384f2d/bz2file-0.98.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python2.7/dist-packages (from smart-open>=1.2.1->gensim) (2.18.4)\n",
            "Collecting boto3 (from smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6d/69/1d148c026d926333a2179c2ad3fddf80c902a8745b627d42c9b1c48e3997/boto3-1.8.8-py2.py3-none-any.whl (128kB)\n",
            "\u001b[K    100% |████████████████████████████████| 133kB 20.5MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python2.7/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.6)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python2.7/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.22)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python2.7/dist-packages (from requests->smart-open>=1.2.1->gensim) (2018.8.24)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python2.7/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
            "Collecting jmespath<1.0.0,>=0.7.1 (from boto3->smart-open>=1.2.1->gensim)\n",
            "  Downloading https://files.pythonhosted.org/packages/b7/31/05c8d001f7f87f0f07289a5fc0fc3832e9a57f2dbd4d3b0fee70e0d51365/jmespath-0.9.3-py2.py3-none-any.whl\n",
            "Collecting s3transfer<0.2.0,>=0.1.10 (from boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/14/2a0004d487464d120c9fb85313a75cd3d71a7506955be458eebfe19a6b1d/s3transfer-0.1.13-py2.py3-none-any.whl (59kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 16.5MB/s \n",
            "\u001b[?25hCollecting botocore<1.12.0,>=1.11.8 (from boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/43/96/f5c5fd630f8f8c36800ec5e1daa474c065a75ea1d011f26d1fa934137e18/botocore-1.11.8-py2.py3-none-any.whl (4.6MB)\n",
            "\u001b[K    100% |████████████████████████████████| 4.7MB 3.3MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: futures<4.0.0,>=2.2.0; python_version == \"2.6\" or python_version == \"2.7\" in /usr/local/lib/python2.7/dist-packages (from s3transfer<0.2.0,>=0.1.10->boto3->smart-open>=1.2.1->gensim) (3.2.0)\n",
            "Collecting docutils>=0.10 (from botocore<1.12.0,>=1.11.8->boto3->smart-open>=1.2.1->gensim)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/09/c53398e0005b11f7ffb27b7aa720c617aba53be4fb4f4f3f06b9b5c60f28/docutils-0.14-py2-none-any.whl (543kB)\n",
            "\u001b[K    100% |████████████████████████████████| 552kB 7.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python2.7/dist-packages (from botocore<1.12.0,>=1.11.8->boto3->smart-open>=1.2.1->gensim) (2.5.3)\n",
            "Building wheels for collected packages: smart-open, bz2file\n",
            "  Running setup.py bdist_wheel for smart-open ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/73/f1/9b/ccf93d4ba073b6f79b1ed9df68ab5ce048d8136d0efcf90b30\n",
            "  Running setup.py bdist_wheel for bz2file ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/81/75/d6/e1317bf09bf1af5a30befc2a007869fa6e1f516b8f7c591cb9\n",
            "Successfully built smart-open bz2file\n",
            "Installing collected packages: keras-preprocessing, keras-applications, keras, boto, bz2file, jmespath, docutils, botocore, s3transfer, boto3, smart-open, gensim\n",
            "  Found existing installation: Keras 2.1.6\n",
            "    Uninstalling Keras-2.1.6:\n",
            "      Successfully uninstalled Keras-2.1.6\n",
            "Successfully installed boto-2.49.0 boto3-1.8.8 botocore-1.11.8 bz2file-0.98 docutils-0.14 gensim-3.5.0 jmespath-0.9.3 keras-2.2.2 keras-applications-1.0.4 keras-preprocessing-1.0.2 s3transfer-0.1.13 smart-open-1.6.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-BfrSMkYPqRN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "7b9e8079-29a6-44a3-9321-d1ebe32d13fd"
      },
      "cell_type": "code",
      "source": [
        "!wget http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.tar.gz"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2018-09-06 14:43:55--  http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.tar.gz\n",
            "Resolving www.cs.cmu.edu (www.cs.cmu.edu)... 128.2.42.95\n",
            "Connecting to www.cs.cmu.edu (www.cs.cmu.edu)|128.2.42.95|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 17329808 (17M) [application/x-tar]\n",
            "Saving to: ‘news20.tar.gz’\n",
            "\n",
            "news20.tar.gz       100%[===================>]  16.53M  1.72MB/s    in 9.8s    \n",
            "\n",
            "2018-09-06 14:44:05 (1.68 MB/s) - ‘news20.tar.gz’ saved [17329808/17329808]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xUqqMKbqP1PL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!tar xzf news20.tar.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DjHxEtQHROYf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "4b3b9f86-da2e-45a0-d25a-938f33fb6c95"
      },
      "cell_type": "code",
      "source": [
        "!wget http://nlp.stanford.edu/data/glove.6B.zip"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2018-09-06 14:44:11--  http://nlp.stanford.edu/data/glove.6B.zip\r\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2018-09-06 14:44:11--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  39.5MB/s    in 18s     \n",
            "\n",
            "2018-09-06 14:44:29 (46.3 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "l_og9VhqRKR6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "5a6b0a12-6fff-495e-b0e6-771960c97161"
      },
      "cell_type": "code",
      "source": [
        "!unzip glove.6B.zip"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  glove.6B.zip\n",
            "  inflating: glove.6B.50d.txt        \n",
            "  inflating: glove.6B.100d.txt       \n",
            "  inflating: glove.6B.200d.txt       \n",
            "  inflating: glove.6B.300d.txt       \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hyCs1vTXRy7V",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "a2e25253-1297-48f9-b152-8cfe9581e337"
      },
      "cell_type": "code",
      "source": [
        "from __future__ import print_function\n",
        "\n",
        "import os\n",
        "import sys\n",
        "import numpy as np\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.layers import Dense, Input, GlobalMaxPooling1D\n",
        "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
        "from keras.models import Model\n",
        "from keras.initializers import Constant\n",
        "import cPickle\n",
        "\n",
        "\n",
        "BASE_DIR = ''\n",
        "GLOVE_DIR = ''\n",
        "TEXT_DATA_DIR = os.path.join(BASE_DIR, '20_newsgroup')\n",
        "MAX_SEQUENCE_LENGTH = 1000\n",
        "MAX_NUM_WORDS = 20000\n",
        "EMBEDDING_DIM = 100\n",
        "VALIDATION_SPLIT = 0.4\n",
        "\n",
        "# first, build index mapping words in the embeddings set\n",
        "# to their embedding vector\n",
        "\n",
        "print('Indexing word vectors.')\n",
        "\n",
        "embeddings_index = {}\n",
        "with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt')) as f:\n",
        "    for line in f:\n",
        "        values = line.split()\n",
        "        word = values[0]\n",
        "        coefs = np.asarray(values[1:], dtype='float32')\n",
        "        embeddings_index[word] = coefs\n",
        "\n",
        "print('Found %s word vectors.' % len(embeddings_index))\n",
        "\n",
        "# second, prepare text samples and their labels\n",
        "print('Processing text dataset')\n",
        "\n",
        "texts = []  # list of text samples\n",
        "labels_index = {}  # dictionary mapping label name to numeric id\n",
        "labels = []  # list of label ids\n",
        "for name in sorted(os.listdir(TEXT_DATA_DIR)):\n",
        "    path = os.path.join(TEXT_DATA_DIR, name)\n",
        "    if os.path.isdir(path):\n",
        "        label_id = len(labels_index)\n",
        "        labels_index[name] = label_id\n",
        "        for fname in sorted(os.listdir(path)):\n",
        "            if fname.isdigit():\n",
        "                fpath = os.path.join(path, fname)\n",
        "                args = {} if sys.version_info < (3,) else {'encoding': 'latin-1'}\n",
        "                with open(fpath, **args) as f:\n",
        "                    t = f.read()\n",
        "                    i = t.find('\\n\\n')  # skip header\n",
        "                    if 0 < i:\n",
        "                        t = t[i:]\n",
        "                    texts.append(t)\n",
        "                labels.append(label_id)\n",
        "\n",
        "print('Found %s texts.' % len(texts))\n",
        "\n",
        "# finally, vectorize the text samples into a 2D integer tensor\n",
        "tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)\n",
        "tokenizer.fit_on_texts(texts)\n",
        "sequences = tokenizer.texts_to_sequences(texts)\n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n",
        "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "labels = to_categorical(np.asarray(labels))\n",
        "print('Shape of data tensor:', data.shape)\n",
        "print('Shape of label tensor:', labels.shape)\n",
        "\n",
        "# split the data into a training set and a validation set\n",
        "indices = np.arange(data.shape[0])\n",
        "np.random.shuffle(indices)\n",
        "data = data[indices]\n",
        "labels = labels[indices]\n",
        "num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
        "\n",
        "x_train = data[:-num_validation_samples]\n",
        "y_train = labels[:-num_validation_samples]\n",
        "x_val = data[-num_validation_samples:]\n",
        "y_val = labels[-num_validation_samples:]\n",
        "\n",
        "print('Preparing embedding matrix.')\n",
        "\n",
        "# prepare embedding matrix\n",
        "num_words = min(MAX_NUM_WORDS, len(word_index) + 1)\n",
        "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
        "for word, i in word_index.items():\n",
        "    if i >= MAX_NUM_WORDS:\n",
        "        continue\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = embedding_vector\n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Indexing word vectors.\n",
            "Found 400000 word vectors.\n",
            "Processing text dataset\n",
            "Found 19997 texts.\n",
            "Found 174105 unique tokens.\n",
            "Shape of data tensor: (19997, 1000)\n",
            "Shape of label tensor: (19997, 20)\n",
            "Preparing embedding matrix.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "r4f5CN_GSpav",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "bf62182d-7d35-489b-89f6-f670d6def552"
      },
      "cell_type": "code",
      "source": [
        "# load pre-trained word embeddings into an Embedding layer\n",
        "# note that we set trainable = False so as to keep the embeddings fixed\n",
        "pretrained_embedding_layer = Embedding(num_words,\n",
        "                                EMBEDDING_DIM,\n",
        "                                embeddings_initializer=Constant(embedding_matrix),\n",
        "                                input_length=MAX_SEQUENCE_LENGTH,\n",
        "                                trainable=False)\n",
        "\n",
        "print('Training model.')\n",
        "\n",
        "# train a 1D convnet with global maxpooling\n",
        "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
        "embedded_sequences = pretrained_embedding_layer(sequence_input)\n",
        "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = GlobalMaxPooling1D()(x)\n",
        "x = Dense(128, activation='relu')(x)\n",
        "preds = Dense(len(labels_index), activation='softmax')(x)\n",
        "\n",
        "model = Model(sequence_input, preds)\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='rmsprop',\n",
        "              metrics=['acc'])\n",
        "\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=128,\n",
        "          epochs=10,\n",
        "          validation_data=(x_val, y_val))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training model.\n",
            "Train on 11999 samples, validate on 7998 samples\n",
            "Epoch 1/10\n",
            "11999/11999 [==============================] - 155s 13ms/step - loss: 2.5810 - acc: 0.1744 - val_loss: 2.3600 - val_acc: 0.2256\n",
            "Epoch 2/10\n",
            "11999/11999 [==============================] - 154s 13ms/step - loss: 1.7368 - acc: 0.3930 - val_loss: 1.7241 - val_acc: 0.4102\n",
            "Epoch 3/10\n",
            "11999/11999 [==============================] - 155s 13ms/step - loss: 1.3533 - acc: 0.5262 - val_loss: 1.2453 - val_acc: 0.5618\n",
            "Epoch 4/10\n",
            "11999/11999 [==============================] - 154s 13ms/step - loss: 1.1175 - acc: 0.6111 - val_loss: 1.2066 - val_acc: 0.5898\n",
            "Epoch 5/10\n",
            "11999/11999 [==============================] - 155s 13ms/step - loss: 0.9545 - acc: 0.6771 - val_loss: 1.0347 - val_acc: 0.6577\n",
            "Epoch 6/10\n",
            "11999/11999 [==============================] - 154s 13ms/step - loss: 0.8164 - acc: 0.7197 - val_loss: 1.1297 - val_acc: 0.6144\n",
            "Epoch 7/10\n",
            "11999/11999 [==============================] - 156s 13ms/step - loss: 0.6940 - acc: 0.7638 - val_loss: 1.0048 - val_acc: 0.6760\n",
            "Epoch 8/10\n",
            "11999/11999 [==============================] - 155s 13ms/step - loss: 0.5927 - acc: 0.8001 - val_loss: 1.0053 - val_acc: 0.6872\n",
            "Epoch 9/10\n",
            "11999/11999 [==============================] - 156s 13ms/step - loss: 0.5238 - acc: 0.8287 - val_loss: 0.9523 - val_acc: 0.7079\n",
            "Epoch 10/10\n",
            "11999/11999 [==============================] - 153s 13ms/step - loss: 0.4465 - acc: 0.8492 - val_loss: 1.0430 - val_acc: 0.6983\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f9ef91e4bd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "kZ258iDaUD0P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "08c1fcaa-8802-4ff1-9dbd-2979121a3d6a"
      },
      "cell_type": "code",
      "source": [
        "random_embedding_layer = Embedding(num_words,\n",
        "                                EMBEDDING_DIM,\n",
        "                                input_length=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "print('Training model.')\n",
        "\n",
        "# train a 1D convnet with global maxpooling\n",
        "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
        "embedded_sequences = random_embedding_layer(sequence_input)\n",
        "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = GlobalMaxPooling1D()(x)\n",
        "x = Dense(128, activation='relu')(x)\n",
        "preds = Dense(len(labels_index), activation='softmax')(x)\n",
        "\n",
        "model = Model(sequence_input, preds)\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='rmsprop',\n",
        "              metrics=['acc'])\n",
        "\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=128,\n",
        "          epochs=10,\n",
        "          validation_data=(x_val, y_val))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training model.\n",
            "Train on 11999 samples, validate on 7998 samples\n",
            "Epoch 1/10\n",
            "11999/11999 [==============================] - 182s 15ms/step - loss: 2.7592 - acc: 0.0990 - val_loss: 2.3958 - val_acc: 0.1614\n",
            "Epoch 2/10\n",
            "11999/11999 [==============================] - 185s 15ms/step - loss: 2.2289 - acc: 0.1878 - val_loss: 2.1106 - val_acc: 0.2142\n",
            "Epoch 3/10\n",
            "11999/11999 [==============================] - 183s 15ms/step - loss: 1.9787 - acc: 0.2557 - val_loss: 1.9712 - val_acc: 0.2548\n",
            "Epoch 4/10\n",
            "11999/11999 [==============================] - 185s 15ms/step - loss: 1.6896 - acc: 0.3502 - val_loss: 1.7153 - val_acc: 0.3772\n",
            "Epoch 5/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 1.3630 - acc: 0.4739 - val_loss: 1.5474 - val_acc: 0.4684\n",
            "Epoch 6/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 1.0884 - acc: 0.5945 - val_loss: 1.4860 - val_acc: 0.5068\n",
            "Epoch 7/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 0.8437 - acc: 0.6969 - val_loss: 1.4757 - val_acc: 0.5495\n",
            "Epoch 8/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 0.6485 - acc: 0.7710 - val_loss: 1.4966 - val_acc: 0.5835\n",
            "Epoch 9/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 0.4919 - acc: 0.8358 - val_loss: 1.5751 - val_acc: 0.6108\n",
            "Epoch 10/10\n",
            "11999/11999 [==============================] - 184s 15ms/step - loss: 0.3990 - acc: 0.8651 - val_loss: 1.8137 - val_acc: 0.5756\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f120b73ecd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "metadata": {
        "id": "P44ARw2EFjU6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install -U -q PyDrive\n",
        "import os\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# 1. Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TmVvam9SYk7l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6882aaba-b91e-48f9-ca3e-e6afe4c78bb9"
      },
      "cell_type": "code",
      "source": [
        "listed = drive.ListFile({'q': \"'12C0S8nP6RTyF0lPbUnUA-5V0chnN3UqG' in parents\"}).GetList()\n",
        "for file in listed:\n",
        "    print('title {}, id {}'.format(file['title'], file['id']))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "title visual_keyed_vectors.cpickle, id 1NA5OuE6ZJpPwogioKYF3fSSeJiW1Nb01\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "cEiSMxKNZcRt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "download_path = os.path.expanduser('~/data2')\n",
        "os.makedirs(download_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b4RjyicqZl9A",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "output_file = os.path.join(download_path, 'visual_keyed_vectors.cpickle')\n",
        "temp_file = drive.CreateFile({'id': '1NA5OuE6ZJpPwogioKYF3fSSeJiW1Nb01'})\n",
        "temp_file.GetContentFile(output_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IM-hLxeTZ9tM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "visual_embeddings = cPickle.load(open(download_path + '/visual_keyed_vectors.cpickle'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uGGW4_yldRdb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        },
        "outputId": "6085c410-62df-4152-a13a-3f52e1111ebe"
      },
      "cell_type": "code",
      "source": [
        "print(visual_embeddings.shape)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0mTraceback (most recent call last)",
            "\u001b[0;32m<ipython-input-27-c646aae1e838>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvisual_embeddings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'Word2VecKeyedVectors' object has no attribute 'shape'"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "TmEoWpExaOOm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1bdeed55-d414-4c5f-bc1d-d80eda006bb6"
      },
      "cell_type": "code",
      "source": [
        "words_not_in_model = []\n",
        "visual_embedding_matrix_orig = np.zeros((num_words, 2048))\n",
        "for word, i in word_index.items():\n",
        "    if i >= MAX_NUM_WORDS:\n",
        "        continue\n",
        "    \n",
        "    if word in visual_embeddings:\n",
        "        visual_embedding_vector = visual_embeddings[word]\n",
        "        visual_embedding_matrix_orig[i] = visual_embedding_vector\n",
        "    else:\n",
        "        words_not_in_model.append(word)\n",
        "\n",
        "len(words_not_in_model)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "891"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "metadata": {
        "id": "c3v3_ng6dpew",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "50248fa0-7c13-4045-c7cc-32b8c78a69d2"
      },
      "cell_type": "code",
      "source": [
        "visual_embedding_matrix_orig.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20000, 2048)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "metadata": {
        "id": "WrfAnMLaayJj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xtoVX0dZdkpX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7a89f3a9-9c03-40aa-9afe-f954190ffa3b"
      },
      "cell_type": "code",
      "source": [
        "visual_embedding_matrix = PCA(n_components=100).fit_transform(visual_embedding_matrix_orig)\n",
        "visual_embedding_matrix.shape"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20000, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "metadata": {
        "id": "lf_eLku1dtGM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        },
        "outputId": "43d05691-047f-4af8-c96a-aeb5090999e8"
      },
      "cell_type": "code",
      "source": [
        "visual_embedding_layer = Embedding(num_words,\n",
        "                                2048,\n",
        "                                embeddings_initializer=Constant(visual_embedding_matrix_orig),\n",
        "                                input_length=MAX_SEQUENCE_LENGTH,\n",
        "                                trainable=False)\n",
        "\n",
        "print('Training model.')\n",
        "\n",
        "# train a 1D convnet with global maxpooling\n",
        "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
        "embedded_sequences = visual_embedding_layer(sequence_input)\n",
        "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = MaxPooling1D(5)(x)\n",
        "x = Conv1D(128, 5, activation='relu')(x)\n",
        "x = GlobalMaxPooling1D()(x)\n",
        "x = Dense(128, activation='relu')(x)\n",
        "preds = Dense(len(labels_index), activation='softmax')(x)\n",
        "\n",
        "model = Model(sequence_input, preds)\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='rmsprop',\n",
        "              metrics=['acc'])\n",
        "\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=128,\n",
        "          epochs=15,\n",
        "          validation_data=(x_val, y_val))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training model.\n",
            "Train on 11999 samples, validate on 7998 samples\n",
            "Epoch 1/15\n",
            "11999/11999 [==============================] - 55s 5ms/step - loss: 3.0930 - acc: 0.0679 - val_loss: 2.9072 - val_acc: 0.0850\n",
            "Epoch 2/15\n",
            "11999/11999 [==============================] - 54s 4ms/step - loss: 2.8181 - acc: 0.1128 - val_loss: 2.7333 - val_acc: 0.1383\n",
            "Epoch 3/15\n",
            "11999/11999 [==============================] - 54s 4ms/step - loss: 2.5617 - acc: 0.1853 - val_loss: 2.3999 - val_acc: 0.2111\n",
            "Epoch 4/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 2.3890 - acc: 0.2325 - val_loss: 2.3110 - val_acc: 0.2246\n",
            "Epoch 5/15\n",
            "11999/11999 [==============================] - 54s 4ms/step - loss: 2.2564 - acc: 0.2707 - val_loss: 2.2551 - val_acc: 0.2561\n",
            "Epoch 6/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 2.1075 - acc: 0.3124 - val_loss: 2.1118 - val_acc: 0.2953\n",
            "Epoch 7/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.8984 - acc: 0.3808 - val_loss: 2.2540 - val_acc: 0.2911\n",
            "Epoch 8/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.7147 - acc: 0.4422 - val_loss: 1.9648 - val_acc: 0.3700\n",
            "Epoch 9/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.5234 - acc: 0.4994 - val_loss: 1.9029 - val_acc: 0.3843\n",
            "Epoch 10/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.3261 - acc: 0.5629 - val_loss: 1.7478 - val_acc: 0.4334\n",
            "Epoch 11/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.1520 - acc: 0.6182 - val_loss: 1.8463 - val_acc: 0.4404\n",
            "Epoch 12/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 1.0344 - acc: 0.6666 - val_loss: 1.9894 - val_acc: 0.4287\n",
            "Epoch 13/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 0.8800 - acc: 0.7148 - val_loss: 1.7933 - val_acc: 0.4621\n",
            "Epoch 14/15\n",
            "11999/11999 [==============================] - 54s 5ms/step - loss: 0.7525 - acc: 0.7559 - val_loss: 3.1984 - val_acc: 0.3300\n",
            "Epoch 15/15\n",
            " 7680/11999 [==================>...........] - ETA: 14s - loss: 0.6976 - acc: 0.7845"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}