3.Q_Mean_W2V.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MRP-fAQedMTd"
   },
   "source": [
    "<h2>Featurizing text data with tfidf weighted word-vectors </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-3IbomL8dMTi",
    "outputId": "3fa8eb7c-ddf2-4f98-edee-0c49db6502e8"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import time\n",
    "import warnings\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import sys\n",
    "import os \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "# exctract word2vec vectors\n",
    "# https://github.com/explosion/spaCy/issues/1721\n",
    "# http://landinghub.visualstudio.com/visual-cpp-build-tools\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j5XNgVyLdMT7"
   },
   "outputs": [],
   "source": [
    "# avoid decoding problems\n",
    "df = pd.read_csv(\"train.csv\")\n",
    " \n",
    "# encode questions to unicode\n",
    "# https://stackoverflow.com/a/6812069\n",
    "# ----------------- python 2 ---------------------\n",
    "# df['question1'] = df['question1'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# df['question2'] = df['question2'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# ----------------- python 3 ---------------------\n",
    "df['question1'] = df['question1'].apply(lambda x: str(x))\n",
    "df['question2'] = df['question2'].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HbiMFpgRdMUJ",
    "outputId": "21c00698-7f2a-4ce4-e665-f7a2feaab6fa"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RU3HqJXwdMUj"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# merge texts\n",
    "questions = list(df['question1']) + list(df['question2'])\n",
    "\n",
    "tfidf = TfidfVectorizer(lowercase=False)\n",
    "tfidf.fit_transform(questions)\n",
    "\n",
    "# dict key:word and value:tf-idf score\n",
    "word2tfidf = dict(zip(tfidf.get_feature_names(), tfidf.idf_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "collapsed": true,
    "id": "2JKI2yT4dMUv"
   },
   "source": [
    "- After we find TF-IDF scores, we convert each question to a weighted average of word2vec vectors by these scores.\n",
    "- here we use a pre-trained GLOVE model which comes free with \"Spacy\".  https://spacy.io/usage/vectors-similarity\n",
    "- It is trained on Wikipedia and therefore, it is stronger in terms of word semantics. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# en_vectors_web_lg, which includes over 1 million unique vectors.\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PFS6m8z5dMUz",
    "outputId": "3c4fb6fd-7f86-4955-8b8f-762b5969ecce"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d1dbf8aaf204a738985be1ea1d1c8d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=404290), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vecs1 = []\n",
    "# https://github.com/noamraph/tqdm\n",
    "# tqdm is used to print the progress bar\n",
    "for qu1 in tqdm(list(df['question1'])):\n",
    "    doc1 = nlp(qu1) \n",
    "    # 384 is the number of dimensions of vectors \n",
    "    mean_vec1 = np.zeros([len(doc1), 384])\n",
    "    for word1 in doc1:\n",
    "        # word2vec\n",
    "        vec1 = word1.vector\n",
    "        # fetch df score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word1)]\n",
    "        except:\n",
    "            idf = 0\n",
    "        # compute final vec\n",
    "        mean_vec1 += vec1 * idf\n",
    "    mean_vec1 = mean_vec1.mean(axis=0)\n",
    "    vecs1.append(mean_vec1)\n",
    "df['q1_feats_m'] = list(vecs1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "62GEF-RbdMVB",
    "outputId": "60a4f5f8-5582-4886-befd-2ab6ed99c753"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "124371752e7841279a1a573c802da544",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=404290), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vecs2 = []\n",
    "for qu2 in tqdm(list(df['question2'])):\n",
    "    doc2 = nlp(qu2) \n",
    "    mean_vec2 = np.zeros([len(doc2), 384])\n",
    "    for word2 in doc2:\n",
    "        # word2vec\n",
    "        vec2 = word2.vector\n",
    "        # fetch df score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word2)]\n",
    "        except:\n",
    "            #print word\n",
    "            idf = 0\n",
    "        # compute final vec\n",
    "        mean_vec2 += vec2 * idf\n",
    "    mean_vec2 = mean_vec2.mean(axis=0)\n",
    "    vecs2.append(mean_vec2)\n",
    "df['q2_feats_m'] = list(vecs2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a38GBlGWdMVQ"
   },
   "outputs": [],
   "source": [
    "#prepro_features_train.csv (Simple Preprocessing Feartures)\n",
    "#nlp_features_train.csv (NLP Features)\n",
    "if os.path.isfile('nlp_features_train.csv'):\n",
    "    dfnlp = pd.read_csv(\"nlp_features_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download nlp_features_train.csv from drive or run previous notebook\")\n",
    "\n",
    "if os.path.isfile('df_fe_without_preprocessing_train.csv'):\n",
    "    dfppro = pd.read_csv(\"df_fe_without_preprocessing_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download df_fe_without_preprocessing_train.csv from drive or run previous notebook\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "apdRa1kndMVb"
   },
   "outputs": [],
   "source": [
    "df1 = dfnlp.drop(['qid1','qid2','question1','question2'],axis=1)\n",
    "df2 = dfppro.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "df3 = df.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "df3_q1 = pd.DataFrame(df3.q1_feats_m.values.tolist(), index= df3.index)\n",
    "df3_q2 = pd.DataFrame(df3.q2_feats_m.values.tolist(), index= df3.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xzWAqGegdMVp",
    "outputId": "2f88eeda-244f-4bbb-a51c-a8680fe8fb92"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>token_set_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.833319</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.916659</td>\n",
       "      <td>0.785709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>100</td>\n",
       "      <td>93</td>\n",
       "      <td>93</td>\n",
       "      <td>100</td>\n",
       "      <td>0.982759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12.5</td>\n",
       "      <td>86</td>\n",
       "      <td>63</td>\n",
       "      <td>66</td>\n",
       "      <td>75</td>\n",
       "      <td>0.596154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.249997</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "      <td>43</td>\n",
       "      <td>47</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28</td>\n",
       "      <td>24</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>0.039216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.199998</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.307690</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>67</td>\n",
       "      <td>47</td>\n",
       "      <td>35</td>\n",
       "      <td>56</td>\n",
       "      <td>0.175000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  is_duplicate   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "0   0             0  0.999980  0.833319  0.999983  0.999983  0.916659   \n",
       "1   1             0  0.799984  0.399996  0.749981  0.599988  0.699993   \n",
       "2   2             0  0.399992  0.333328  0.399992  0.249997  0.399996   \n",
       "3   3             0  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4   4             0  0.399992  0.199998  0.999950  0.666644  0.571420   \n",
       "\n",
       "    ctc_max  last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0  0.785709           0.0            1.0           2.0      13.0   \n",
       "1  0.466664           0.0            1.0           5.0      12.5   \n",
       "2  0.285712           0.0            1.0           4.0      12.0   \n",
       "3  0.000000           0.0            0.0           2.0      12.0   \n",
       "4  0.307690           0.0            1.0           6.0      10.0   \n",
       "\n",
       "   token_set_ratio  token_sort_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "0              100                93          93                 100   \n",
       "1               86                63          66                  75   \n",
       "2               63                63          43                  47   \n",
       "3               28                24           9                  14   \n",
       "4               67                47          35                  56   \n",
       "\n",
       "   longest_substr_ratio  \n",
       "0              0.982759  \n",
       "1              0.596154  \n",
       "2              0.166667  \n",
       "3              0.039216  \n",
       "4              0.175000  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataframe of nlp features\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N4DQnDtndMV4",
    "outputId": "2e288eed-e8fa-4ec3-a9b9-4e4daba52fc1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>freq_qid1</th>\n",
       "      <th>freq_qid2</th>\n",
       "      <th>q1len</th>\n",
       "      <th>q2len</th>\n",
       "      <th>q1_n_words</th>\n",
       "      <th>q2_n_words</th>\n",
       "      <th>word_Common</th>\n",
       "      <th>word_Total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>freq_q1+q2</th>\n",
       "      <th>freq_q1-q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>57</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>10.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>88</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>59</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>76</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  freq_qid1  freq_qid2  q1len  q2len  q1_n_words  q2_n_words  \\\n",
       "0   0          1          1     66     57          14          12   \n",
       "1   1          4          1     51     88           8          13   \n",
       "2   2          1          1     73     59          14          10   \n",
       "3   3          1          1     50     65          11           9   \n",
       "4   4          3          1     76     39          13           7   \n",
       "\n",
       "   word_Common  word_Total  word_share  freq_q1+q2  freq_q1-q2  \n",
       "0         10.0        23.0    0.434783           2           0  \n",
       "1          4.0        20.0    0.200000           5           3  \n",
       "2          4.0        24.0    0.166667           2           0  \n",
       "3          0.0        19.0    0.000000           2           0  \n",
       "4          2.0        20.0    0.100000           4           2  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data before preprocessing \n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_1YIPtTwdMWC",
    "outputId": "510f4c73-0706-4633-d706-e0d348ebfa71"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>374</th>\n",
       "      <th>375</th>\n",
       "      <th>376</th>\n",
       "      <th>377</th>\n",
       "      <th>378</th>\n",
       "      <th>379</th>\n",
       "      <th>380</th>\n",
       "      <th>381</th>\n",
       "      <th>382</th>\n",
       "      <th>383</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>121.929942</td>\n",
       "      <td>100.083882</td>\n",
       "      <td>72.497907</td>\n",
       "      <td>115.641808</td>\n",
       "      <td>-48.370869</td>\n",
       "      <td>34.619060</td>\n",
       "      <td>-172.057791</td>\n",
       "      <td>-92.502618</td>\n",
       "      <td>113.223268</td>\n",
       "      <td>50.562426</td>\n",
       "      <td>...</td>\n",
       "      <td>12.397645</td>\n",
       "      <td>40.909526</td>\n",
       "      <td>8.150258</td>\n",
       "      <td>-15.170695</td>\n",
       "      <td>18.007705</td>\n",
       "      <td>6.167001</td>\n",
       "      <td>-30.124161</td>\n",
       "      <td>3.700892</td>\n",
       "      <td>-1.757701</td>\n",
       "      <td>-1.818056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-78.070958</td>\n",
       "      <td>54.843768</td>\n",
       "      <td>82.738470</td>\n",
       "      <td>98.191845</td>\n",
       "      <td>-51.234832</td>\n",
       "      <td>55.013504</td>\n",
       "      <td>-39.140738</td>\n",
       "      <td>-82.692363</td>\n",
       "      <td>45.161486</td>\n",
       "      <td>-9.556293</td>\n",
       "      <td>...</td>\n",
       "      <td>-21.987074</td>\n",
       "      <td>-12.389282</td>\n",
       "      <td>20.667988</td>\n",
       "      <td>2.202711</td>\n",
       "      <td>-17.142451</td>\n",
       "      <td>-5.880971</td>\n",
       "      <td>-10.123959</td>\n",
       "      <td>-4.890664</td>\n",
       "      <td>-13.018388</td>\n",
       "      <td>-5.219297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-5.355039</td>\n",
       "      <td>73.671827</td>\n",
       "      <td>14.376383</td>\n",
       "      <td>104.130224</td>\n",
       "      <td>1.433505</td>\n",
       "      <td>35.229106</td>\n",
       "      <td>-148.519387</td>\n",
       "      <td>-97.124606</td>\n",
       "      <td>41.972181</td>\n",
       "      <td>50.948725</td>\n",
       "      <td>...</td>\n",
       "      <td>3.027700</td>\n",
       "      <td>14.025775</td>\n",
       "      <td>-2.960309</td>\n",
       "      <td>-3.206542</td>\n",
       "      <td>4.355142</td>\n",
       "      <td>2.936156</td>\n",
       "      <td>-20.199560</td>\n",
       "      <td>9.816350</td>\n",
       "      <td>11.894366</td>\n",
       "      <td>-8.798817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.778360</td>\n",
       "      <td>-34.712032</td>\n",
       "      <td>48.999641</td>\n",
       "      <td>59.699240</td>\n",
       "      <td>40.661269</td>\n",
       "      <td>-41.658738</td>\n",
       "      <td>-36.808587</td>\n",
       "      <td>24.170641</td>\n",
       "      <td>0.235596</td>\n",
       "      <td>-29.407287</td>\n",
       "      <td>...</td>\n",
       "      <td>13.100011</td>\n",
       "      <td>1.405663</td>\n",
       "      <td>-1.891072</td>\n",
       "      <td>-7.882639</td>\n",
       "      <td>18.000563</td>\n",
       "      <td>12.106917</td>\n",
       "      <td>-10.507835</td>\n",
       "      <td>5.243826</td>\n",
       "      <td>10.158345</td>\n",
       "      <td>5.886347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51.138227</td>\n",
       "      <td>38.587261</td>\n",
       "      <td>123.639507</td>\n",
       "      <td>53.333048</td>\n",
       "      <td>-47.062795</td>\n",
       "      <td>37.356194</td>\n",
       "      <td>-298.722733</td>\n",
       "      <td>-106.421107</td>\n",
       "      <td>106.248904</td>\n",
       "      <td>65.880710</td>\n",
       "      <td>...</td>\n",
       "      <td>13.906533</td>\n",
       "      <td>43.461716</td>\n",
       "      <td>11.519204</td>\n",
       "      <td>-22.468289</td>\n",
       "      <td>45.431129</td>\n",
       "      <td>8.161225</td>\n",
       "      <td>-35.373912</td>\n",
       "      <td>7.728862</td>\n",
       "      <td>9.592856</td>\n",
       "      <td>5.447332</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 384 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0           1           2           3          4          5    \\\n",
       "0  121.929942  100.083882   72.497907  115.641808 -48.370869  34.619060   \n",
       "1  -78.070958   54.843768   82.738470   98.191845 -51.234832  55.013504   \n",
       "2   -5.355039   73.671827   14.376383  104.130224   1.433505  35.229106   \n",
       "3    5.778360  -34.712032   48.999641   59.699240  40.661269 -41.658738   \n",
       "4   51.138227   38.587261  123.639507   53.333048 -47.062795  37.356194   \n",
       "\n",
       "          6           7           8          9    ...        374        375  \\\n",
       "0 -172.057791  -92.502618  113.223268  50.562426  ...  12.397645  40.909526   \n",
       "1  -39.140738  -82.692363   45.161486  -9.556293  ... -21.987074 -12.389282   \n",
       "2 -148.519387  -97.124606   41.972181  50.948725  ...   3.027700  14.025775   \n",
       "3  -36.808587   24.170641    0.235596 -29.407287  ...  13.100011   1.405663   \n",
       "4 -298.722733 -106.421107  106.248904  65.880710  ...  13.906533  43.461716   \n",
       "\n",
       "         376        377        378        379        380       381        382  \\\n",
       "0   8.150258 -15.170695  18.007705   6.167001 -30.124161  3.700892  -1.757701   \n",
       "1  20.667988   2.202711 -17.142451  -5.880971 -10.123959 -4.890664 -13.018388   \n",
       "2  -2.960309  -3.206542   4.355142   2.936156 -20.199560  9.816350  11.894366   \n",
       "3  -1.891072  -7.882639  18.000563  12.106917 -10.507835  5.243826  10.158345   \n",
       "4  11.519204 -22.468289  45.431129   8.161225 -35.373912  7.728862   9.592856   \n",
       "\n",
       "        383  \n",
       "0 -1.818056  \n",
       "1 -5.219297  \n",
       "2 -8.798817  \n",
       "3  5.886347  \n",
       "4  5.447332  \n",
       "\n",
       "[5 rows x 384 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Questions 1 tfidf weighted word2vec\n",
    "df3_q1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wUMdkJTNdMWL",
    "outputId": "69e3e256-cbb8-4fe2-aaf2-9088c3868b29"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>374</th>\n",
       "      <th>375</th>\n",
       "      <th>376</th>\n",
       "      <th>377</th>\n",
       "      <th>378</th>\n",
       "      <th>379</th>\n",
       "      <th>380</th>\n",
       "      <th>381</th>\n",
       "      <th>382</th>\n",
       "      <th>383</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>125.983297</td>\n",
       "      <td>95.636478</td>\n",
       "      <td>42.114727</td>\n",
       "      <td>95.450001</td>\n",
       "      <td>-37.386297</td>\n",
       "      <td>39.400069</td>\n",
       "      <td>-148.116056</td>\n",
       "      <td>-87.851468</td>\n",
       "      <td>110.371951</td>\n",
       "      <td>62.272810</td>\n",
       "      <td>...</td>\n",
       "      <td>16.165599</td>\n",
       "      <td>33.030674</td>\n",
       "      <td>7.019994</td>\n",
       "      <td>-14.793956</td>\n",
       "      <td>15.437509</td>\n",
       "      <td>8.199660</td>\n",
       "      <td>-25.070836</td>\n",
       "      <td>1.571609</td>\n",
       "      <td>1.603733</td>\n",
       "      <td>0.305656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-106.871914</td>\n",
       "      <td>80.290392</td>\n",
       "      <td>79.066298</td>\n",
       "      <td>59.302086</td>\n",
       "      <td>-42.175382</td>\n",
       "      <td>117.616717</td>\n",
       "      <td>-144.364301</td>\n",
       "      <td>-127.131524</td>\n",
       "      <td>22.962531</td>\n",
       "      <td>25.397591</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.901130</td>\n",
       "      <td>-4.565389</td>\n",
       "      <td>41.520751</td>\n",
       "      <td>-0.727564</td>\n",
       "      <td>-16.413772</td>\n",
       "      <td>-7.373778</td>\n",
       "      <td>2.638877</td>\n",
       "      <td>-7.403463</td>\n",
       "      <td>2.703062</td>\n",
       "      <td>0.408053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.072902</td>\n",
       "      <td>15.513371</td>\n",
       "      <td>1.846921</td>\n",
       "      <td>85.937604</td>\n",
       "      <td>-33.808812</td>\n",
       "      <td>94.702354</td>\n",
       "      <td>-122.256856</td>\n",
       "      <td>-114.009527</td>\n",
       "      <td>53.922339</td>\n",
       "      <td>60.131817</td>\n",
       "      <td>...</td>\n",
       "      <td>8.359976</td>\n",
       "      <td>-2.165974</td>\n",
       "      <td>10.936576</td>\n",
       "      <td>-16.531656</td>\n",
       "      <td>14.681223</td>\n",
       "      <td>15.633755</td>\n",
       "      <td>-1.210891</td>\n",
       "      <td>14.183817</td>\n",
       "      <td>11.703129</td>\n",
       "      <td>10.148076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39.421522</td>\n",
       "      <td>44.136995</td>\n",
       "      <td>-24.010941</td>\n",
       "      <td>85.265888</td>\n",
       "      <td>-0.339025</td>\n",
       "      <td>-9.323141</td>\n",
       "      <td>-60.499645</td>\n",
       "      <td>-37.044788</td>\n",
       "      <td>49.407835</td>\n",
       "      <td>-23.350166</td>\n",
       "      <td>...</td>\n",
       "      <td>3.311411</td>\n",
       "      <td>3.788879</td>\n",
       "      <td>13.398605</td>\n",
       "      <td>-6.592596</td>\n",
       "      <td>6.437358</td>\n",
       "      <td>5.993291</td>\n",
       "      <td>2.732391</td>\n",
       "      <td>-3.727645</td>\n",
       "      <td>5.614124</td>\n",
       "      <td>6.023691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31.950130</td>\n",
       "      <td>62.854123</td>\n",
       "      <td>1.778176</td>\n",
       "      <td>36.218745</td>\n",
       "      <td>-45.130849</td>\n",
       "      <td>66.674900</td>\n",
       "      <td>-106.342322</td>\n",
       "      <td>-22.901017</td>\n",
       "      <td>59.835925</td>\n",
       "      <td>62.663934</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.403873</td>\n",
       "      <td>11.991199</td>\n",
       "      <td>8.088481</td>\n",
       "      <td>-15.090202</td>\n",
       "      <td>8.375163</td>\n",
       "      <td>1.727223</td>\n",
       "      <td>-6.601127</td>\n",
       "      <td>11.317408</td>\n",
       "      <td>11.544600</td>\n",
       "      <td>2.478690</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 384 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1          2          3          4           5    \\\n",
       "0  125.983297  95.636478  42.114727  95.450001 -37.386297   39.400069   \n",
       "1 -106.871914  80.290392  79.066298  59.302086 -42.175382  117.616717   \n",
       "2    7.072902  15.513371   1.846921  85.937604 -33.808812   94.702354   \n",
       "3   39.421522  44.136995 -24.010941  85.265888  -0.339025   -9.323141   \n",
       "4   31.950130  62.854123   1.778176  36.218745 -45.130849   66.674900   \n",
       "\n",
       "          6           7           8          9    ...        374        375  \\\n",
       "0 -148.116056  -87.851468  110.371951  62.272810  ...  16.165599  33.030674   \n",
       "1 -144.364301 -127.131524   22.962531  25.397591  ...  -4.901130  -4.565389   \n",
       "2 -122.256856 -114.009527   53.922339  60.131817  ...   8.359976  -2.165974   \n",
       "3  -60.499645  -37.044788   49.407835 -23.350166  ...   3.311411   3.788879   \n",
       "4 -106.342322  -22.901017   59.835925  62.663934  ...  -2.403873  11.991199   \n",
       "\n",
       "         376        377        378        379        380        381  \\\n",
       "0   7.019994 -14.793956  15.437509   8.199660 -25.070836   1.571609   \n",
       "1  41.520751  -0.727564 -16.413772  -7.373778   2.638877  -7.403463   \n",
       "2  10.936576 -16.531656  14.681223  15.633755  -1.210891  14.183817   \n",
       "3  13.398605  -6.592596   6.437358   5.993291   2.732391  -3.727645   \n",
       "4   8.088481 -15.090202   8.375163   1.727223  -6.601127  11.317408   \n",
       "\n",
       "         382        383  \n",
       "0   1.603733   0.305656  \n",
       "1   2.703062   0.408053  \n",
       "2  11.703129  10.148076  \n",
       "3   5.614124   6.023691  \n",
       "4  11.544600   2.478690  \n",
       "\n",
       "[5 rows x 384 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Questions 2 tfidf weighted word2vec\n",
    "df3_q2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ozz83vh4dMWU",
    "outputId": "e5b30f77-2849-4b08-9949-0912ec0db418"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features in nlp dataframe : 17\n",
      "Number of features in preprocessed dataframe : 12\n",
      "Number of features in question1 w2v  dataframe : 384\n",
      "Number of features in question2 w2v  dataframe : 384\n",
      "Number of features in final dataframe  : 797\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of features in nlp dataframe :\", df1.shape[1])\n",
    "print(\"Number of features in preprocessed dataframe :\", df2.shape[1])\n",
    "print(\"Number of features in question1 w2v  dataframe :\", df3_q1.shape[1])\n",
    "print(\"Number of features in question2 w2v  dataframe :\", df3_q2.shape[1])\n",
    "print(\"Number of features in final dataframe  :\", df1.shape[1]+df2.shape[1]+df3_q1.shape[1]+df3_q2.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HmfZ5Q1zdMWl"
   },
   "outputs": [],
   "source": [
    "# storing the final features to csv file\n",
    "if not os.path.isfile('final_features.csv'):\n",
    "    df3_q1['id']=df1['id']\n",
    "    df3_q2['id']=df1['id']\n",
    "    df1  = df1.merge(df2, on='id',how='left')\n",
    "    df2  = df3_q1.merge(df3_q2, on='id',how='left')\n",
    "    result  = df1.merge(df2, on='id',how='left')\n",
    "    result.to_csv('final_features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of final dataset: (404290, 796)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Shape of final dataset: {result.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>...</th>\n",
       "      <th>374_y</th>\n",
       "      <th>375_y</th>\n",
       "      <th>376_y</th>\n",
       "      <th>377_y</th>\n",
       "      <th>378_y</th>\n",
       "      <th>379_y</th>\n",
       "      <th>380_y</th>\n",
       "      <th>381_y</th>\n",
       "      <th>382_y</th>\n",
       "      <th>383_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.833319</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.916659</td>\n",
       "      <td>0.785709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>16.165599</td>\n",
       "      <td>33.030674</td>\n",
       "      <td>7.019994</td>\n",
       "      <td>-14.793956</td>\n",
       "      <td>15.437509</td>\n",
       "      <td>8.199660</td>\n",
       "      <td>-25.070836</td>\n",
       "      <td>1.571609</td>\n",
       "      <td>1.603733</td>\n",
       "      <td>0.305656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.901130</td>\n",
       "      <td>-4.565389</td>\n",
       "      <td>41.520751</td>\n",
       "      <td>-0.727564</td>\n",
       "      <td>-16.413772</td>\n",
       "      <td>-7.373778</td>\n",
       "      <td>2.638877</td>\n",
       "      <td>-7.403463</td>\n",
       "      <td>2.703062</td>\n",
       "      <td>0.408053</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 796 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  is_duplicate   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "0   0             0  0.999980  0.833319  0.999983  0.999983  0.916659   \n",
       "1   1             0  0.799984  0.399996  0.749981  0.599988  0.699993   \n",
       "\n",
       "    ctc_max  last_word_eq  first_word_eq  ...      374_y      375_y  \\\n",
       "0  0.785709           0.0            1.0  ...  16.165599  33.030674   \n",
       "1  0.466664           0.0            1.0  ...  -4.901130  -4.565389   \n",
       "\n",
       "       376_y      377_y      378_y     379_y      380_y     381_y     382_y  \\\n",
       "0   7.019994 -14.793956  15.437509  8.199660 -25.070836  1.571609  1.603733   \n",
       "1  41.520751  -0.727564 -16.413772 -7.373778   2.638877 -7.403463  2.703062   \n",
       "\n",
       "      383_y  \n",
       "0  0.305656  \n",
       "1  0.408053  \n",
       "\n",
       "[2 rows x 796 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head(2)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "3.Q_Mean_W2V.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}