Text_Classification/self_Attn_on_seperate_fets_of_2embds.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Treating embeddings as seperate continuous features\n",
    "\n",
    "The 2 embeddings used here are glove and fasttext. In the model they are passed to different embedding layers and 2 different layers of feature extraction are present for each of the embeddings.\n",
    "\n",
    "Sequence feature is extracted by first spatially dropping out the embedding vectors. Than globally max and average fetaures are pooled from the selfattended RNN output.\n",
    "\n",
    "The max pooled features for both the embeddings is than concatenated and passed to dense layer for feature extraction.\n",
    "\n",
    "Same is done for the average pooled features and, than these pooled features are concatenated and passed to fully connected network for classification. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('2.2.2',\n",
       " '1.10.0',\n",
       " '3.6.6 |Anaconda, Inc.| (default, Jun 28 2018, 11:27:44) [MSC v.1900 64 bit (AMD64)]')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras, tensorflow, sys\n",
    "keras.__version__, tensorflow.__version__, sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Deep Learning 3033\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# import required packages\n",
    "import sys\n",
    "import warnings\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\" \n",
    "\n",
    "if not sys.warnoptions:\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers import CuDNNLSTM, CuDNNGRU, BatchNormalization, Dense, Dropout, Activation, Embedding, Input\n",
    "from keras.layers import Bidirectional,SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Concatenate\n",
    "\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "from keras_self_attention import  SeqSelfAttention\n",
    "\n",
    "from sklearn.metrics import confusion_matrix,f1_score, precision_score, recall_score, roc_auc_score, accuracy_score\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from glob import glob\n",
    "\n",
    "import math\n",
    "from snapshot import SnapshotCallbackBuilder\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_imdb_dataset():\n",
    "\n",
    "    # Load the dataset\n",
    "    train = pd.DataFrame(columns=[\"text\", \"positive\"])\n",
    "    test = pd.DataFrame(columns=[\"text\", \"positive\"])\n",
    "    ctr = 0\n",
    "    cte = 0\n",
    "    for fil in ['train/', 'test/']:\n",
    "        for cls in ['pos', 'neg']:\n",
    "            dset_path = \"./\" + fil + cls\n",
    "            for fname in sorted(os.listdir(dset_path)):\n",
    "                if fname.endswith('.txt'):\n",
    "                    with open(os.path.join(dset_path, fname), encoding=\"utf8\") as f:\n",
    "                        if fil == 'train/':\n",
    "                            train.loc[ctr] = (f.read(), int(cls == \"pos\"))\n",
    "                            ctr+=1\n",
    "                        else:\n",
    "                            test.loc[cte] = (f.read(), int(cls == \"pos\"))\n",
    "                            cte+=1\n",
    "                            \n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape (25000, 2)\n",
      "Test data shape (25000, 2)\n"
     ]
    }
   ],
   "source": [
    "train, test = load_imdb_dataset()\n",
    "\n",
    "print (\"Train data shape\", train.shape)\n",
    "print (\"Test data shape\", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data class distbn 1    12500\n",
      "0    12500\n",
      "Name: positive, dtype: int64\n",
      "Test data class distbn 1    12500\n",
      "0    12500\n",
      "Name: positive, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data class distbn\", train.positive.value_counts())\n",
    "print(\"Test data class distbn\", test.positive.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bromwell High is a cartoon comedy. It ran at t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Homelessness (or Houselessness as George Carli...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Brilliant over-acting by Lesley Ann Warren. Be...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>This is easily the most underrated film inn th...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>This is not the typical Mel Brooks film. It wa...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text positive\n",
       "0  Bromwell High is a cartoon comedy. It ran at t...        1\n",
       "1  Homelessness (or Houselessness as George Carli...        1\n",
       "2  Brilliant over-acting by Lesley Ann Warren. Be...        1\n",
       "3  This is easily the most underrated film inn th...        1\n",
       "4  This is not the typical Mel Brooks film. It wa...        1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I went and saw this movie last night after bei...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Actor turned director Bill Paxton follows up h...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>As a recreational golfer with some knowledge o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I saw this film in a sneak preview, and it is ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bill Paxton has taken the true story of the 19...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text positive\n",
       "0  I went and saw this movie last night after bei...        1\n",
       "1  Actor turned director Bill Paxton follows up h...        1\n",
       "2  As a recreational golfer with some knowledge o...        1\n",
       "3  I saw this film in a sneak preview, and it is ...        1\n",
       "4  Bill Paxton has taken the true story of the 19...        1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Sequence length distribution:\n",
      "\n",
      "count    25000.000000\n",
      "mean       233.787200\n",
      "std        173.733032\n",
      "min         10.000000\n",
      "25%        127.000000\n",
      "50%        174.000000\n",
      "75%        284.000000\n",
      "max       2470.000000\n",
      "dtype: float64\n",
      "\n",
      "\n",
      "Test Sequence length distribution:\n",
      "\n",
      "count    25000.000000\n",
      "mean       228.526680\n",
      "std        168.883693\n",
      "min          4.000000\n",
      "25%        126.000000\n",
      "50%        172.000000\n",
      "75%        277.000000\n",
      "max       2278.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Average number of words per review \n",
    "tr_l = [len(x.split()) for x in train.text]\n",
    "te_l = [len(x.split()) for x in test.text]\n",
    "print(\"Train Sequence length distribution:\\n\")\n",
    "print(pd.Series(tr_l).describe())\n",
    "print(\"\\n\\nTest Sequence length distribution:\\n\")\n",
    "print(pd.Series(te_l).describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size 88582\n"
     ]
    }
   ],
   "source": [
    "# Number of unique words by finding the length of dictionary of words mapped with unique tokens (integers)\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(list(train.text))\n",
    "print(\"Vocab size\", len(tokenizer.word_counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 300 \n",
    "\n",
    "# mean number of words per sentence in the train set is taken as maximum sentence length.\n",
    "max_sent_len = int(np.percentile(tr_l, 50)) \n",
    "\n",
    "num_words = len(tokenizer.word_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converte sentence text to list of token represented sentences, required for training\n",
    "X = tokenizer.texts_to_sequences(train.text)\n",
    "X = pad_sequences(X, maxlen=max_sent_len)\n",
    "\n",
    "x_test = tokenizer.texts_to_sequences(test.text)\n",
    "x_test = pad_sequences(x_test, maxlen=max_sent_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((22500, 174), (2500, 174))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split into train and validation data\n",
    "x_train, x_val, y_train, y_val = train_test_split(X, train.positive, test_size=0.1, random_state=3)\n",
    "x_train.shape, x_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Functions to load different embeddings\n",
    "\n",
    "def load_glove(word_index):\n",
    "    EMBEDDING_FILE = '../embeddings/glove.840B.300d/glove.840B.300d.txt'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\"))\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "    \n",
    "    nb_words = min(num_words, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= num_words: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "    return embedding_matrix \n",
    "\n",
    "def load_fasttext(word_index):    \n",
    "    EMBEDDING_FILE = '../embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\") if len(o)>100)\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    nb_words = min(num_words, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= num_words: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get word_indexes (tokens) for each of the word in vocabulary\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "embedding_matrix_glove = load_glove(word_index)\n",
    "embedding_matrix_ft = load_fasttext(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a snaphot of the model after (nb_epochs/ M) epochs. Also cosine anneal the learning rate.\n",
    "\n",
    "M = 2\n",
    "nb_epoch = T = 50\n",
    "alpha_zero = 5e-4\n",
    "snapshot = SnapshotCallbackBuilder(T, M, alpha_zero)\n",
    "timestr = time.strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
    "model_prefix = './imdb{}'.format(timestr)\n",
    "\n",
    "callbacks = snapshot.get_callbacks(model_prefix=model_prefix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 0 , Accuracy_score: 0.88964\n",
      "Model: 1 , Accuracy_score: 0.89272\n",
      "Model: 2 , Accuracy_score: 0.8908\n",
      "Model: 3 , Accuracy_score: 0.88996\n",
      "Model: 4 , Accuracy_score: 0.89064\n"
     ]
    }
   ],
   "source": [
    "pred_avg = []\n",
    "real = list(test.positive)\n",
    "# Performing cross validation of 5\n",
    "for cv in range(5):\n",
    "    \n",
    "    # Embedding layer to use glove embeddings\n",
    "    embedding_layer_g = Embedding(num_words, embed_size, input_length=max_sent_len, trainable=False,\n",
    "                                  weights=[embedding_matrix_glove])\n",
    "    sequence_input_g = Input(shape=(max_sent_len,), dtype='int32')\n",
    "\n",
    "    embedded_sequences_g = embedding_layer_g(sequence_input_g)\n",
    "    embedded_sequences_g = SpatialDropout1D(0.2)(embedded_sequences_g)\n",
    "\n",
    "    x_g = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat')(embedded_sequences_g)\n",
    "    x_g_a = SeqSelfAttention()(x_g)\n",
    "\n",
    "    x_g = Concatenate()([x_g, x_g_a])\n",
    "\n",
    "    x_g_a = GlobalAveragePooling1D()(x_g)\n",
    "    x_g = GlobalMaxPooling1D()(x_g)\n",
    "\n",
    "\n",
    "    # Embedding layer to use fasttext embeddings\n",
    "    embedding_layer_f = Embedding(num_words, embed_size, input_length=max_sent_len, trainable=False,\n",
    "                                  weights=[embedding_matrix_ft])\n",
    "    sequence_input_f = Input(shape=(max_sent_len,), dtype='int32')\n",
    "\n",
    "    embedded_sequences_f = embedding_layer_f(sequence_input_f)\n",
    "    embedded_sequences_f = SpatialDropout1D(0.2)(embedded_sequences_f)\n",
    "\n",
    "    x_f = Bidirectional(CuDNNGRU(64, return_sequences=True), merge_mode='concat')(embedded_sequences_f)\n",
    "    x_f_a = SeqSelfAttention()(x_f)\n",
    "\n",
    "    x_f = Concatenate()([x_f, x_f_a])\n",
    "\n",
    "    x_f_a = GlobalAveragePooling1D()(x_f)\n",
    "    x_f = GlobalMaxPooling1D()(x_f)\n",
    "\n",
    "\n",
    "    # Concatenate the globally pooled features from each of the embeddings\n",
    "    x_g = Concatenate()([x_g, x_f])\n",
    "\n",
    "    x_g = Dense(128, activation=\"relu\", kernel_initializer=\"glorot_normal\")(x_g)\n",
    "    x_g = BatchNormalization()(x_g)\n",
    "    x_g = Dropout(0.4)(x_g)\n",
    "\n",
    "    \n",
    "    # Concatenate the globally averaged features from each of the embeddings\n",
    "    x_a = Concatenate()([x_g_a, x_f_a])\n",
    "\n",
    "    x_a = Dense(128, activation=\"relu\", kernel_initializer=\"glorot_normal\")(x_a)\n",
    "    x_a = BatchNormalization()(x_a)\n",
    "    x_a = Dropout(0.4)(x_a)\n",
    "\n",
    "    x = Concatenate()([x_g, x_a])\n",
    "    \n",
    "    \n",
    "    # Fully connected layers to classify using features from both the embeddings.\n",
    "    x = Dense(128, activation=\"relu\", kernel_initializer=\"glorot_normal\")(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(0.4)(x)\n",
    "\n",
    "    x = Dense(64, activation=\"relu\", kernel_initializer=\"glorot_normal\")(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(0.4)(x)\n",
    "\n",
    "    x = Dense(16, activation=\"relu\", kernel_initializer=\"glorot_normal\")(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(0.4)(x)\n",
    "\n",
    "    out = Dense(1, activation=\"sigmoid\", kernel_initializer=\"glorot_normal\")(x)\n",
    "    \n",
    "    model = Model([sequence_input_g, sequence_input_f], out)\n",
    "\n",
    "    model.compile(loss=\"binary_crossentropy\", optimizer=Adam(5e-5),metrics=['accuracy'])\n",
    "    model.fit([x_train, x_train], y_train, validation_data=([x_val, x_val], y_val), epochs=nb_epoch, verbose=0,\n",
    "              batch_size=100, shuffle=True, callbacks=callbacks)\n",
    "    pred = model.predict(x=[x_test, x_test])\n",
    "    pred = pred > 0.5\n",
    "    pred = [int(p[0]) for p in pred]\n",
    "    pred_avg.append(pred)\n",
    "    print(\"Model:\", cv, \", Accuracy_score:\", accuracy_score(real, pred))\n",
    "    del model\n",
    "\n",
    "pred = np.mean(pred_avg, axis=0)\n",
    "pred = pred > 0.5\n",
    "pred = [int(p) for p in pred]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## F1 Score: 90.59 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      " [[11306  1194]\n",
      " [ 1161 11339]]\n",
      "f1_score: 0.9059241800822915 precision_score: 0.9047315088167238 recall_score: 0.90712 accuracy_score: 0.9058\n"
     ]
    }
   ],
   "source": [
    "print(\"Confusion Matrix:\\n\", confusion_matrix(real, pred))\n",
    "print(\"f1_score:\",f1_score(real, pred), \"precision_score:\",precision_score(real, pred),\n",
    "          \"recall_score:\",recall_score(real, pred), \"accuracy_score:\",accuracy_score(real, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}