Topic Link Dectection  With Word2Vec & TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic Link Detection TDT5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "#from __future__ import print_function, division\n",
    "\n",
    "if not os.path.exists(\"data/embed.dat\"):\n",
    "    print(\"Caching word embeddings in memmapped format...\")\n",
    "    \n",
    "    wv = Word2Vec.load_word2vec_format(\n",
    "        \"/home/skillachie/Downloads/GoogleNews-vectors-negative300.bin\",\n",
    "        binary=True)\n",
    "    wv.init_sims(replace=False)\n",
    "    \n",
    "    fp = np.memmap(\"data/embed.dat\", dtype=np.double, mode='w+', shape=wv.syn0norm.shape)\n",
    "    fp[:] = wv.syn0norm[:]\n",
    "    with open(\"data/embed.vocab\", \"w\") as f:\n",
    "        for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()):\n",
    "            print(w, file=f)\n",
    "    del fp, wv\n",
    "\n",
    "W = np.memmap(\"data/embed.dat\", dtype=np.double, mode=\"r\", shape=(3000000, 300))\n",
    "with open(\"data/embed.vocab\") as f:\n",
    "    vocab_list = map(str.strip, f.readlines())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print(\"Features:\",  \", \".join(train_vect.get_feature_names()))\n",
    "vocab_dict = {w: k for k, w in enumerate(vocab_list)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<B><B><B><B><B><B><B><B><B><B>']\n"
     ]
    }
   ],
   "source": [
    "from parse_tdt5 import *\n",
    "import sys\n",
    "\n",
    "print([10*\"<B>\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Tokenize and clean\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "\n",
    "def preprocessing(text,stem=False,stop=False,sent=False):\n",
    "    \n",
    "    \n",
    "    # Remove punctuations\n",
    "    exclude = set(string.punctuation)\n",
    "    text = ''.join(ch for ch in text if ch not in exclude)\n",
    "    \n",
    "    tokens = word_tokenize(text)\n",
    "    \n",
    "    if stop:\n",
    "        stop = stopwords.words('english')\n",
    "        tokens =[word for word in tokens if word not in stop]\n",
    "        tokens = [word.lower() for word in tokens]\n",
    "\n",
    "    if stem:\n",
    "        stemmer = PorterStemmer()\n",
    "        tokens = [stemmer.stem(t) for t in tokens]\n",
    "        \n",
    "            \n",
    "    if sent:\n",
    "        tokens = ' '.join(tokens)\n",
    "        \n",
    "        \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdt_annotation_dir = \"/home/skillachie/Downloads/tdt/annotations/tdt5_topic_annot/data/annotations\"\n",
    "tdt_corpus_dir = \"/home/skillachie/Downloads/tdt/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "link_det_answers = read_link_detection_answer_key(tdt_annotation_dir,'lnk_SR=nwt_TE=man,eng_1000.key')\n",
    "link_texts = []\n",
    "link_pairs = []\n",
    "link_answers = []\n",
    "\n",
    "for link in link_det_answers:\n",
    "    doc1 = read_doc(tdt_corpus_dir,link['file1_id'],link['file1_docno'],\\\n",
    "                            task_type='LINK')\n",
    "    \n",
    "    doc2 = read_doc(tdt_corpus_dir,link['file2_id'],link['file2_docno'],\\\n",
    "                            task_type='LINK')\n",
    "    \n",
    "    if doc2 is None:\n",
    "        continue\n",
    "    \n",
    "    if doc1 is None:\n",
    "        continue \n",
    "    \n",
    "    \n",
    "    link_texts.append(doc1)\n",
    "    link_texts.append(doc2)\n",
    "    \n",
    "    link_pairs.append((doc1,doc2))\n",
    "    link_answers.append(link['answer'])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "992\n"
     ]
    }
   ],
   "source": [
    "print(len(link_answers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Split train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_articles_pre, test_articles, train_target_pre, test_target = \\\n",
    "                        train_test_split(link_pairs,link_answers, test_size=0.20, random_state=13)\n",
    "    \n",
    "#Divide into Dev and Train\n",
    "train_articles, dev_articles, train_target, dev_target =\\\n",
    "    train_test_split(train_articles_pre,train_target_pre, test_size=0.20, random_state=13)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import Doc2Vec, Word2Vec\n",
    "import gensim.models.doc2vec\n",
    "import numpy as np\n",
    "\n",
    "from collections import OrderedDict\n",
    "import multiprocessing\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "import sys\n",
    "\n",
    "cores = multiprocessing.cpu_count()\n",
    "assert gensim.models.doc2vec.FAST_VERSION > -1, \"this will be painfully slow otherwise\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " British  Prime  Minister's  Office  announced  today  that  the  7th \n",
      " U.S.  President  Bush  will  visit  Northern  Ireland,  and  British \n",
      " Prime  Minister  Tony  Blair  met  to  discuss  Iraq  war.  They \n",
      " will  also  discuss  the  Middle  East  situation  and  the  Northern \n",
      " Ireland.  This  will  be  Bush  and  Blair  three  times  in  recent \n",
      " weeks  hold  a  summit.  In  Iraq  before  war  broke  out  on  March \n",
      " 20,  who  met  in  Azores  is.  Last  week  in  the  United  States \n",
      " had  been  held  at  Camp  David  talks. \n",
      "\n",
      "1268\n",
      "318\n"
     ]
    }
   ],
   "source": [
    "# Create vectorizer using all the vocab of the dataset\n",
    "def get_all_articles_text(articles):\n",
    "    all_articles = []\n",
    "\n",
    "    for article in articles:\n",
    "        all_articles.append(article[0])\n",
    "        all_articles.append(article[1])\n",
    "    return all_articles\n",
    "\n",
    "train_articles_txt = get_all_articles_text(train_articles)\n",
    "dev_articles_txt = get_all_articles_text(dev_articles)\n",
    "\n",
    "print(train_articles_txt[66])\n",
    "\n",
    "#train_vect = create_vectorizer(train_articles_txt + test_articles_txt)\n",
    "print(len(train_articles_txt))\n",
    "print(len(dev_articles_txt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<318x6209 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 51397 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#Test\n",
    "tfidf_vec_test = TfidfVectorizer(sublinear_tf=True,\n",
    "                            use_idf=True)\n",
    "\n",
    "\n",
    "tfidf_vec_test.fit_transform(train_articles_txt)\n",
    "\n",
    "#Dev\n",
    "tfidf_vec_dev = TfidfVectorizer(sublinear_tf=True,\n",
    "                            use_idf=True)\n",
    "\n",
    "tfidf_vec_dev.fit_transform(dev_articles_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2vec_model = Word2Vec.load_word2vec_format('/home/skillachie/Downloads/GoogleNews-vectors-negative300.bin', binary=True)\n",
    "word2vec_model.init_sims(replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def get_word_vectors(tokens):\n",
    "    \n",
    "    article_word_vectors = []\n",
    "    article_model_vocab = []\n",
    "    \n",
    "    for token in tokens:\n",
    "        #print(token)\n",
    "        word_vec = []\n",
    "        try:\n",
    "            word_vec = word2vec_model[token]\n",
    "            article_model_vocab.append(token)\n",
    "            #print(token)\n",
    "        except Exception:\n",
    "                #print word\n",
    "            continue\n",
    "            #word_vec = np.zeros(300)\n",
    "                       \n",
    "    \n",
    "        #article_word_vectors.append(word_vec)\n",
    "        \n",
    "        \n",
    "    return article_model_vocab\n",
    "\n",
    "\n",
    "def aggr_word_vectors(word_vectors):\n",
    "    #print(len(word_vectors))\n",
    "    stack = np.vstack(word_vectors)\n",
    "    doc_vec = np.mean(stack,axis=0)\n",
    "    return doc_vec\n",
    "\n",
    "def gen_doc2vec_vocab_features(articles,tf_vec):\n",
    "     \n",
    "    new_feature_vectors = []\n",
    "    \n",
    "    for article_tuple in articles:\n",
    "\n",
    "        \n",
    "        #print(article_tuple)\n",
    "        #print(\"-------------------------\")\n",
    "        feat_dict = defaultdict(dict)\n",
    "        \n",
    "        article1_txt = article_tuple[0]\n",
    "        article2_txt = article_tuple[1]\n",
    "        \n",
    "        if article_tuple[0] is None:\n",
    "            article1_txt = 'bad data' #hack remove\n",
    "             \n",
    "        \n",
    "        if article_tuple[1] is None:\n",
    "            article2_txt = 'bad data' #hack remove\n",
    "        \n",
    "        \n",
    "        article1_tf_idf = tf_vec.transform([article1_txt])\n",
    "        article2_tf_idf = tf_vec.transform([article2_txt])\n",
    "        \n",
    "        feat_dict['article1_tfidf'] = article1_tf_idf\n",
    "        feat_dict['article2_tfidf'] = article2_tf_idf\n",
    "        \n",
    "        article1_tokens = word_tokenize(article1_txt)\n",
    "        article2_tokens = word_tokenize(article2_txt)\n",
    "        \n",
    "        article1_wordvec_vobab = get_word_vectors(article1_tokens)\n",
    "        article2_wordvec_vobab = get_word_vectors(article2_tokens)\n",
    "        \n",
    "        #print(article1_wordvec_vobab)\n",
    "        #sys.exit(1)\n",
    "          \n",
    "        #doc1_vec = aggr_word_vectors(article1_wordvecs)\n",
    "        #doc2_vec = aggr_word_vectors(article2_wordvecs)\n",
    "\n",
    "        \n",
    "        #feat_dict['article1_vec'] = doc1_vec\n",
    "        feat_dict['article1_vocab'] = article1_wordvec_vobab\n",
    "        #feat_dict['article2_vec'] = doc2_vec\n",
    "        feat_dict['article2_vocab'] = article2_wordvec_vobab\n",
    "        \n",
    "        new_feature_vectors.append(feat_dict)\n",
    "        \n",
    "    return new_feature_vectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def check_link(articles,method=None):\n",
    "           \n",
    "    pred_answer = None\n",
    "    \n",
    "    if method is None:\n",
    "        sim = word2vec_model.n_similarity(articles['article1_vocab'], articles['article2_vocab'])\n",
    "        \n",
    "        if sim >= 0.88:\n",
    "            pred_answer = \"TARGET\"\n",
    "        else:\n",
    "            pred_answer = \"NONTARGET\"\n",
    "        \n",
    "    else:\n",
    "        sim  = cosine_similarity(articles['article1_tfidf'], articles['article2_tfidf'] )[0][0]\n",
    "    \n",
    "        \n",
    "        if sim >= 0.2:\n",
    "            pred_answer = \"TARGET\"\n",
    "        else:\n",
    "            pred_answer = \"NONTARGET\"\n",
    "        \n",
    "    return pred_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def connect_links(features,method=None):\n",
    "    \n",
    "    pred_answers = []\n",
    "    for article_features in features:\n",
    "        answer = check_link(article_features,method)\n",
    "        pred_answers.append(answer)\n",
    "    return pred_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Dev\n",
    "word2vec_features_dev = gen_doc2vec_vocab_features(dev_articles,tfidf_vec_dev)\n",
    "\n",
    "\n",
    "pred_answers_word2vec_dev = connect_links(word2vec_features_dev)\n",
    "pred_answers_tfidf_dev = connect_links(word2vec_features_dev,\"TF-IDF\")\n",
    "\n",
    "\n",
    "#Test\n",
    "word2vec_features_test = gen_doc2vec_vocab_features(test_articles,tfidf_vec_test)\n",
    "pred_answers_word2vec_test = connect_links(word2vec_features_test)\n",
    "pred_answers_tfidf_test = connect_links(word2vec_features_test,\"TF-IDF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159\n",
      "199\n",
      "634\n"
     ]
    }
   ],
   "source": [
    "print(len(pred_answers_word2vec_dev))\n",
    "print(len(pred_answers_word2vec_test))\n",
    "print(len(train_articles))\n",
    "#print(len(pred_answers_tfidf))\n",
    "\n",
    "\n",
    "#print(len(test_target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def event_pair_evaluation(predicted_answers, answers):\n",
    "    \n",
    "    neg_correct = 0\n",
    "    neg_incorrect = 0\n",
    "    pos_correct = 0\n",
    "    pos_incorrect = 0\n",
    "    \n",
    "    negatives = 0\n",
    "    positives = 0\n",
    "    \n",
    "    total = len(answers)\n",
    "\n",
    "    for pred_ans,ans in zip(predicted_answers,answers):\n",
    "        \n",
    "        if ans == \"NONTARGET\":\n",
    "            negatives +=1\n",
    "\n",
    "            if pred_ans == ans:\n",
    "                neg_correct += 1\n",
    "            else:\n",
    "                neg_incorrect += 1\n",
    "    \n",
    "        if ans == \"TARGET\":\n",
    "            positives +=1\n",
    "    \n",
    "            if pred_ans == ans:\n",
    "                pos_correct += 1\n",
    "            else:\n",
    "                pos_incorrect += 1\n",
    "    \n",
    "     \n",
    "        \n",
    "    pos_correct = np.float64(pos_correct)\n",
    "    neg_correct = np.float64(neg_correct)\n",
    "        \n",
    "            \n",
    "    print(\"Negative: %f \" %( neg_correct/negatives) )\n",
    "    \n",
    "    print(neg_correct)\n",
    "    print(negatives)\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    print(\"Positive: %f \" %( pos_correct/positives) )\n",
    "    print(pos_correct)\n",
    "    print(positives)\n",
    "    #print(/np.float64(positives))\n",
    "    \n",
    "    correct = neg_correct + pos_correct\n",
    "        \n",
    "    return (float(correct) / total)*100   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#/Word2vec Event Pair Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative: 0.941520 \n",
      "161.0\n",
      "171\n",
      "Positive: 0.821429 \n",
      "23.0\n",
      "28\n",
      "92.46231155778895\n"
     ]
    }
   ],
   "source": [
    "#word2vec_score_dev = event_pair_evaluation(pred_answers_word2vec_dev,dev_target)\n",
    "#print(word2vec_score_dev)\n",
    "\n",
    "word2vec_score_test = event_pair_evaluation(pred_answers_word2vec_test,test_target)\n",
    "print(word2vec_score_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative: 1.000000 \n",
      "134.0\n",
      "134\n",
      "Positive: 0.880000 \n",
      "22.0\n",
      "25\n",
      "98.11320754716981\n"
     ]
    }
   ],
   "source": [
    "tfidf_score_dev = event_pair_evaluation(pred_answers_tfidf_dev,dev_target)\n",
    "print(tfidf_score_dev)\n",
    "\n",
    "#tfidf_score_test = event_pair_evaluation(pred_answers_tfidf_test,test_target)\n",
    "#print(tfidf_score_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.968553 , Recall: 1.000000 , F1-Score: 0.909091\n",
      "Accuracy: 0.981132 , Recall: 0.880000 , F1-Score: 0.936170\n",
      "-------------------------------\n",
      "Accuracy: 0.924623 , Recall: 0.821429 , F1-Score: 0.754098\n",
      "Accuracy: 0.949749 , Recall: 0.714286 , F1-Score: 0.800000\n"
     ]
    }
   ],
   "source": [
    "def binarize_answers(answers):\n",
    "    \n",
    "    bin_answers = []\n",
    "    \n",
    "    for answer in answers:\n",
    "        if answer == 'TARGET':\n",
    "            bin_answers.append(1)\n",
    "        else:\n",
    "            bin_answers.append(0)\n",
    "    return bin_answers \n",
    "\n",
    "\n",
    "def get_metrics(target,predict):\n",
    "    \n",
    "    accuracy = accuracy_score(target,predict)\n",
    "    recall = recall_score(target,predict)\n",
    "    f1 = f1_score(target,predict)\n",
    "\n",
    "    #print(accuracy)\n",
    "    print(\"Accuracy: %f , Recall: %f , F1-Score: %f\" %(accuracy, recall, f1))\n",
    "    \n",
    "\n",
    "    \n",
    "#Dev    \n",
    "bin_pred_word2vec_dev = binarize_answers(pred_answers_word2vec_dev)\n",
    "bin_pred_tfidf_dev = binarize_answers(pred_answers_tfidf_dev)\n",
    "bin_dev_target = binarize_answers(dev_target)\n",
    "\n",
    "\n",
    "get_metrics(bin_dev_target,bin_pred_word2vec_dev)\n",
    "get_metrics(bin_dev_target,bin_pred_tfidf_dev)\n",
    "    \n",
    "\n",
    "    \n",
    "print(\"-------------------------------\")\n",
    "\n",
    "#Test\n",
    "bin_pred_word2vec_test = binarize_answers(pred_answers_word2vec_test)\n",
    "bin_pred_tfidf_test = binarize_answers(pred_answers_tfidf_test)\n",
    "bin_test_target = binarize_answers(test_target)\n",
    "\n",
    "\n",
    "get_metrics(bin_test_target,bin_pred_word2vec_test)\n",
    "get_metrics(bin_test_target,bin_pred_tfidf_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}