playground-benchmarking-multihop-qa-systems.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(180000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 180 seconds\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:25: The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading the params file\n",
      "Input encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Input decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "Output encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Output decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:3673: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 202, 256)     25088       input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 202, 128)     12544       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, 202, 256)     525312      embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 202, 256)     263168      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 202, 202)     0           lstm_2[0][0]                     \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "attention (Activation)          (None, 202, 202)     0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 202, 256)     0           attention[0][0]                  \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 202, 512)     0           dot_2[0][0]                      \n",
      "                                                                 lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 202, 128)     65664       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 202, 98)      12642       time_distributed_1[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 904,418\n",
      "Trainable params: 904,418\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import time\n",
    "#import convex as cx\n",
    "import tmqa31 as tmqa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "conversations_path = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/data/convex/test_set/test_set_ALL.json\"\n",
    "with open(conversations_path, \"r\") as data:\n",
    "    conversations = json.load(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pickle_data(df, filename):\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    filename = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/\"+filename+'.pickle.bz2'\n",
    "    #df.summary = df.summary.map(sanitize_str)\n",
    "    print(\"Saving Dataframe Done!\",filename)\n",
    "    return df.to_pickle(filename, compression='bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Q13133'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lang: en, fr, de, it, es, zh\n",
    "#kb: dbpedia, wikidata, dblp, freebase\n",
    "def ask_qanswer(question):\n",
    "    data = {'query': question,'lang': 'en','kb': 'wikidata'}\n",
    "    headers = {\"Authorization\":\"Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIzNDIiLCJpYXQiOjE1NzkyNTYxNDQsImV4cCI6MTU3OTg2MDk0NH0.YPFBZ-Xc8OI7eeTTkQaVT5a-CA5VONiCr_VIViG3t8tjVv7eRKgz_X_1KWDnly_F08rLXwpPcDUMBt8_M8-S8w\"}\n",
    "    query = requests.post('http://qanswer-core1.univ-st-etienne.fr/api/gerbil', data=data, headers=headers)\n",
    "    \n",
    "#    var settings = {\n",
    "#  \"async\": true,\n",
    "#  \"crossDomain\": true,\n",
    "#  \"url\": \"http://qanswer-core1.univ-st-etienne.fr/api/qa/full?question=what%20is%20a%20margerita&lang=en&kb=cocktails\",\n",
    "#  \"method\": \"GET\",\n",
    "#  \"headers\": {\n",
    "#    \"Authorization\": \"Bearer eyJhbGciOiJIUzUxMi.....\",\n",
    "#  }\n",
    "#}\n",
    "    \n",
    "    if not query:\n",
    "        return False\n",
    "    if (query.json()['questions'][0]['question']['answers']) == None:\n",
    "        return False\n",
    "    #if (query.json()['questions'][0]['question']['answers'].replace('\\n', '')) == None:\n",
    "    #    return False\n",
    "    #print(query.json()['questions'][0]['question']['answers'].replace('\\n', '').get(\"results\"))\n",
    "    try:\n",
    "        response = (json.loads(query.json()\n",
    "                .get(\"questions\")[0]\n",
    "                .get(\"question\")\n",
    "                .get(\"answers\")\n",
    "                .replace('\\n', ''))\n",
    "         .get(\"results\").get(\"bindings\"))\n",
    "    except:\n",
    "        return False\n",
    "    \n",
    "    if response:\n",
    "        result = response[0].get(\"o1\").get(\"value\")[len(\"http://www.wikidata.org/entity/\"):] if response[0].get(\"o1\") is not None else False\n",
    "        return result\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "#ask_qanswer(\"Who is the wife of Barack Obama\")\n",
    "#ask_qanwser(\"Which equestrian was born in dublin?\")\n",
    "#ask_qanswer(\"what is the main language spoken in a ghentar si muore facile\")\n",
    "#ask_qanswer(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_qanswer(\"how does engelbert zaschka identify\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_platypus(question):\n",
    "    headers = {'Accept': 'application/json','Accept-Language': 'en',}\n",
    "    params = (('q', question),('lang', 'en'))\n",
    "\n",
    "    response = requests.get('https://qa.askplatyp.us/v0/ask', headers=headers, params=params)\n",
    "    if response:\n",
    "        if type(response.json()['member']) is list:\n",
    "            #print(response.json()['member'][0]['result'])\n",
    "            if response.json()['member'] != []:\n",
    "                if '@id' in (json.dumps(response.json()['member'][0]['result'])):\n",
    "                    try:\n",
    "                        ps_result = (json.dumps(response.json()['member'][0]['result']['@id']))\n",
    "                    except:\n",
    "                        return False\n",
    "                else: return False\n",
    "            else: return False\n",
    "        else:\n",
    "            try:\n",
    "                if '@id' in (json.dumps(response.json()['member']['result'])):\n",
    "                    ps_result = (json.dumps(response.json()[\"member\"]['result']['@id']))\n",
    "                else: return False\n",
    "            except:\n",
    "                return False\n",
    "    else: return False\n",
    "    ps_result = ps_result[4:-1]\n",
    "    #print(result[:1])\n",
    "    if ps_result[:1] != 'P' and ps_result[:1] != 'Q':\n",
    "        return False\n",
    "    return ps_result\n",
    "#ask_platypus(\"Which genre of album is harder.....faster?\")\n",
    "#ask_platypus(\"how does engelbert zaschka identify\")\n",
    "#ask_platypus(\"Which Swiss conductor's cause of death is myoc...\")\n",
    "#ask_platypus(\"where was padraic mcguinness's place of death\")\n",
    "#ask_platypus(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_platypus(\"Who created the show life on earth\")\n",
    "#ask_platypus(\"Who is the wife of Barack Obama\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_convex(question):\n",
    "    cx_result = cx.answer_complete_question(question, cx.tagmeToken)['answers'][0]['answer']\n",
    "    #print(cx_result)\n",
    "    #answer = str(cx.wd.wikidata_id_to_label(result['answers'][0]['answer']))\n",
    "    try:\n",
    "        if not cx_result:\n",
    "            return cx_result\n",
    "        if cx_result[:1] != 'P' and cx_result[:1] != 'Q':\n",
    "            return False\n",
    "        return cx_result\n",
    "    except:\n",
    "        return False\n",
    "\n",
    "#ask_convex(\"Which actor voiced the Unicorn in The Last Unicorn?\")\n",
    "#ask_convex(\"Which genre of album is harder.....faster?\")\n",
    "#ask_convex(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_convex(\"Who is the wife of Barack Obama\")\n",
    "#ask_convex(\"100% senorita is a television show in what language?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tmqa' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-832cef302837>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mask_tmqa\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshow_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtmqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbanning_str\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mtmqa_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manswer_question\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshow_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbanning_str\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbanning_str\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m#print(tmqa_result)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtmqa_result\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tmqa' is not defined"
     ]
    }
   ],
   "source": [
    "def ask_tmqa(question, verbose=False, timer=False, show_graph=False, cores=tmqa.mp.cpu_count(), banning_str=False):\n",
    "    tmqa_result = tmqa.answer_question(question, verbose=verbose, timer=timer, show_graph=show_graph, cores=cores, banning_str=banning_str)\n",
    "    #print(tmqa_result)\n",
    "    if not tmqa_result:\n",
    "        return False\n",
    "    if tmqa_result == (False,False):\n",
    "        return False\n",
    "    #if tmqa_result[]\n",
    "    #if tmqa_result[0][0][:1] != 'P' and tmqa_result[0][0][:1] != 'Q':\n",
    "    #    return False\n",
    "    return tmqa_result[0][0]\n",
    "\n",
    "#answer = ask_tmqa_1(\"Which actor voiced the Unicorn in The Last Unicorn?\", verbose=True)\n",
    "#answer = ask_tmqa_1(\"what's akbar tandjung's ethnicity\", verbose=True)\n",
    "#ask_tmqa_1(\"Which genre of album is harder.....faster?\")\n",
    "#ask_tmqa_1(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_tmqa(\"Who is the wife of Barack Obama\")\n",
    "#print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#HEADERS = ['question', 'answer', 'qanswer', 'platypus', 'convex',\n",
    "#           'tm', \"tm_time\", \"tm_top2\", \"tm_top3\", \"tm_top4\", \"tm_top5\", \"tm_topall\"]\n",
    "#df = pd.DataFrame(columns=HEADERS)\n",
    "#LOADING\n",
    "df = pd.read_pickle(\"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-2240.pickle.bz2\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['What is the name of the writer of The Secret Garden?', 'Where does the story take place?', 'When was the book published?', 'What kind of book is it?', 'What magazine first published the book?']\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'tmqa' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-e2f6e2af14a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mquestions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mturn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'question'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mturn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconversation\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'questions'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mgolden_answers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtmqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwikidata_url_to_wikidata_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mturn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'answer'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mturn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconversation\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'questions'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgolden_answers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-e2f6e2af14a5>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mquestions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mturn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'question'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mturn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconversation\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'questions'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mgolden_answers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtmqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwikidata_url_to_wikidata_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mturn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'answer'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mturn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconversation\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'questions'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgolden_answers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tmqa' is not defined"
     ]
    }
   ],
   "source": [
    "for conversation in conversations:\n",
    "    questions = [turn['question'] for turn in conversation['questions']]\n",
    "    print(questions)\n",
    "    golden_answers = [tmqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "    print(golden_answers)\n",
    "    \n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t>>> Processing 392/2240: (Q39829) By whom was Misery written?                                  Asking Platypus\n",
      "Asking Convex\n",
      "\n",
      " 392 -> Platypus Q39829\n",
      "Asking GraphQA\n",
      "User input: By whom was Misery written?\n",
      "--> Auto correcting question in progress...\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "-> Auto corrected q_nlp: By whom was Misery written \n",
      "-> q_themes: ([(Misery, ['Q596874', 'Q379446']), (misery, ['Q3317029'])], [By was Misery, by be misery write])\n",
      "-> q_themes_enhanced: [('write', ['Q29465908']), ('Write', ['Q1215628'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(be, ['P31']), (written, ['P50'])]\n",
      "-> q_predicates \tRunning time is 15.09s\n",
      "--> Potential meaningful keywords for the sentence: ['Misery', 'misery', 'write', 'Write']\n",
      "q_focused_parts: []\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 18.95s\n",
      "-->  57 nodes and 54 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 57 nodes and 54 edges\n",
      "---> Rebuilding the graph with k_deep 5 ... Previously: 57 nodes or 54 edges was below the limit of 100\n",
      "->New graph \tRunning time is 18.75s\n",
      "-->  63 nodes and 60 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 63 nodes and 60 edges\n",
      "---> Rebuilding the graph with k_deep 7 ... Previously: 63 nodes or 60 edges was below the limit of 100\n",
      "->New graph \tRunning time is 18.51s\n",
      "-->  66 nodes and 64 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 66 nodes and 64 edges\n",
      "---> Rebuilding the graph with k_deep 9 ... Previously: 66 nodes or 64 edges was below the limit of 100\n",
      "->New graph \tRunning time is 18.34s\n",
      "-->  73 nodes and 72 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 73 nodes and 72 edges\n",
      "---> Rebuilding the graph with k_deep 11 ... Previously: 73 nodes or 72 edges was below the limit of 100\n",
      "->New graph \tRunning time is 18.6s\n",
      "-->  84 nodes and 84 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 84 nodes and 84 edges\n",
      "---> Rebuilding the graph with k_deep 12 ... Previously: 84 nodes or 84 edges was below the limit of 100\n",
      "->New graph \tRunning time is 20.25s\n",
      "-->  84 nodes and 84 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 84 nodes and 84 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "-> predicates_dict: {'P1013': 1, 'P1441': 1, 'P156': 2, 'P585': 3, 'P166': 2, 'P1082': 2, 'P407': 2, 'P571': 1, 'P51': 1, 'P131': 4, 'P50': 1, 'P421': 2, 'P31': 3, 'P361': 1, 'P577': 1, 'P495': 1, 'P800': 1, 'P144': 1, 'P136': 4, 'P155': 1, 'P150': 1, 'P47': 6}\n",
      "-> paths_keywords: (['written', 'misery'], {'instance of': [instance of, ['P31']], 'author': [author, ['P50']]}, [whom])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 170\n",
      "->Computing possible paths \tRunning time is 22.89s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 164\n",
      "->\tRunning time is 3.92s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q39829', 34.03272541266038], ['Q484170', 1.1243258510535779], ['Q8785', 0.8351466648111323], ['Q8261', 0.7657646096900868], ['Q1860', 0.7139884289432892], ['Q722192', 0.7128129658306115], ['Q29465908', 0.5302441298458512], ['Q14579', 0.3476752938610911], ['2017-02-14T00:00:00Z', 0.053720558833521115]]\n",
      "->Computing hypothesises \tRunning time is 15.99s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 24\n",
      "->\tRunning time is 7.92s\n",
      "--> len(cleared_golden_paths): 15\n",
      "---> First path: ['Q39829', 'P50', 'Q596874', 'P31', 'Q8261']\n",
      "->\tTotal Running time is 181.99s\n",
      "\n",
      "\n",
      " 392 -> tmqa Q39829\n",
      "\n",
      " 392 -> tmqa in answers ['Q39829', 'Q484170', 'Q8785', 'Q8261', 'Q1860', 'Q722192', 'Q29465908', 'Q14579', '2017-02-14T00:00:00Z']\n",
      "                        question  answer qanswer platypus convex      tm  \\\n",
      "392  By whom was Misery written?  Q39829   False   Q39829  Q1860  Q39829   \n",
      "\n",
      "     tm_time tm_top2 tm_top3 tm_top4 tm_top5 tm_topall  \n",
      "392   182.28    True    True    True    True      True  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-393.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 393/2240: (1959-01-01T00:00:00Z) In what year did The Twilight Zone first start airing?                                  Asking Platypus\n",
      "Asking Convex\n",
      "Asking GraphQA\n",
      "User input: In what year did The Twilight Zone first start airing?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: In what date did The Twilight Zone first start airing \n",
      "-> q_themes: ([(The Twilight Zone, ['Q742103', 'Q559270']), (first, ['Q19269277']), (date, ['Q858423', 'Q3016931']), (Twilight Zone, ['Q2047418', 'Q26326875']), (The Twilight, ['Q14755895']), (Date, ['Q10467097', 'Q18614434']), (air, ['Q7391292', 'Q318452'])], [Airing, In what date did The])\n",
      "-> q_themes_enhanced: [('The Do', ['Q7730457'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: air\n",
      "-> q_predicates: [(did, ['P248']), (start, ['P580', 'P571']), (airing, [])]\n",
      "-> q_predicates \tRunning time is 14.34s\n",
      "--> Potential meaningful keywords for the sentence: ['The Twilight Zone', 'first', 'date', 'Twilight Zone', 'The Twilight', 'Date', 'air', 'The Do']\n",
      "q_focused_parts: [(date, ['Q1652093', 'P837', 'Q3016931'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n"
     ]
    }
   ],
   "source": [
    "### Evaluate\n",
    "banning_str = False\n",
    "\n",
    "start_time = time.time()\n",
    "conversations_len = len(conversations)\n",
    "\n",
    "for i, conversation in enumerate(conversations):\n",
    "    if i >= len(df):\n",
    "        questions = [turn['question'] for turn in conversation['questions']]\n",
    "        question = questions[0]\n",
    "        answers = [tmqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "        answer = answers[0]\n",
    "        print(\"\\r\\t>>> Processing {}/{}: ({}) {}\".format(i,conversations_len,answer,question), end='                                  ')\n",
    "        \n",
    "        #print(\"Asking qAnswer\")\n",
    "        result_qanswer = False #ask_qanswer(question)\n",
    "        print(\"Asking Platypus\")\n",
    "        result_platypus = ask_platypus(question)\n",
    "        print(\"Asking Convex\")\n",
    "        result_convex = ask_convex(question)\n",
    "        print(\"\\n\",i, \"-> qAnswer\", result_qanswer) if result_qanswer == answer else False\n",
    "        print(\"\\n\",i, \"-> Platypus\", result_platypus) if result_platypus == answer else False\n",
    "        print(\"\\n\",i, \"-> Convex\", result_convex) if result_convex == answer else False\n",
    "\n",
    "        df_tm = False\n",
    "        df_tm_top2 = False\n",
    "        df_tm_top3 = False\n",
    "        df_tm_top4 = False\n",
    "        df_tm_top5 = False\n",
    "        df_tm_topall = False\n",
    "        \n",
    "        print(\"Asking GraphQA\")\n",
    "        start_time_tmqa = time.time()\n",
    "        result_tmqa = ask_tmqa(question, verbose=True, timer=True, banning_str=banning_str)   \n",
    "        if result_tmqa:\n",
    "            df_tm = result_tmqa[0]\n",
    "            if answer in result_tmqa[:2]:\n",
    "                df_tm_top2 = True\n",
    "            if answer in result_tmqa[:3]:\n",
    "                df_tm_top3 = True\n",
    "            if answer in result_tmqa[:4]:\n",
    "                df_tm_top4 = True\n",
    "            if answer in result_tmqa[:5]:\n",
    "                df_tm_top5 = True\n",
    "            if answer in result_tmqa:\n",
    "                df_tm_topall = True\n",
    "            \n",
    "        df_tm_time = round(time.time()-start_time_tmqa,2)\n",
    "        \n",
    "        df = df.append({\"question\":question, 'answer':answer, 'qanswer':result_qanswer, 'platypus':result_platypus, 'convex':result_convex,\n",
    "                       \"tm\":df_tm, \"tm_time\":df_tm_time, \"tm_top2\":df_tm_top2, \"tm_top3\":df_tm_top3, \"tm_top4\":df_tm_top4, \"tm_top5\":df_tm_top5, \"tm_topall\":df_tm_topall},\n",
    "                       ignore_index=True)\n",
    "\n",
    "        print(\"\\n\",i, \"-> tmqa\", df_tm) if str(df_tm) == str(answer) else False\n",
    "        print(\"\\n\",i, \"-> tmqa in answers\", result_tmqa) if df_tm_topall == True else False\n",
    "        \n",
    "        print(df.tail(1))\n",
    "        \n",
    "        pickle_data(df, \"benchmarking-qanswer-platypus-convex-qagraph-from-0-to-\"+str(len(df)))\n",
    "        \n",
    "        print(\"\\n\")\n",
    "    \n",
    "    #break\n",
    "\n",
    "print(\"->\\tRunning time is {}s\".format(round(time.time()-start_time,2)))\n",
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t>>> Processing 0/2240: (Q276028) What is the name of the writer of The Secret Garden?                                  Asking qAnswer\n",
      "\n",
      " 0 -> qAnswer Q276028\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-0-redo-qanswer.pickle.bz2\n",
      "question     What is the name of the writer of The Secret G...\n",
      "answer                                                 Q276028\n",
      "qanswer                                                   True\n",
      "platypus                                                 False\n",
      "convex                                                   False\n",
      "tm                                                     Q783521\n",
      "tm_time                                                 480.56\n",
      "tm_top2                                                  False\n",
      "tm_top3                                                  False\n",
      "tm_top4                                                  False\n",
      "tm_top5                                                  False\n",
      "tm_topall                                                 True\n",
      "Name: 0, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 1/2240: (Q23589) What network was Dexter on?                                  Asking qAnswer\n",
      "\n",
      " 1 -> qAnswer Q23589\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-1-redo-qanswer.pickle.bz2\n",
      "question     What network was Dexter on?\n",
      "answer                            Q23589\n",
      "qanswer                             True\n",
      "platypus                          Q23577\n",
      "convex                            Q23589\n",
      "tm                               Q835696\n",
      "tm_time                            74.54\n",
      "tm_top2                            False\n",
      "tm_top3                            False\n",
      "tm_top4                            False\n",
      "tm_top5                            False\n",
      "tm_topall                          False\n",
      "Name: 1, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 2/2240: (1978-06-13T00:00:00Z) When did the movie Grease come out?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-2-redo-qanswer.pickle.bz2\n",
      "question     When did the movie Grease come out?\n",
      "answer                      1978-06-13T00:00:00Z\n",
      "qanswer                                     True\n",
      "platypus                                   False\n",
      "convex                                     False\n",
      "tm                          1978-09-28T00:00:00Z\n",
      "tm_time                                   141.08\n",
      "tm_top2                                    False\n",
      "tm_top3                                    False\n",
      "tm_top4                                     True\n",
      "tm_top5                                     True\n",
      "tm_topall                                   True\n",
      "Name: 2, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 3/2240: (Q4985) Who is the author that wrote the book Moby-Dick?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-3-redo-qanswer.pickle.bz2\n",
      "question     Who is the author that wrote the book Moby-Dick?\n",
      "answer                                                  Q4985\n",
      "qanswer                                                  True\n",
      "platypus                                                False\n",
      "convex                                                  Q4985\n",
      "tm                                                      Q4985\n",
      "tm_time                                                 69.39\n",
      "tm_top2                                                  True\n",
      "tm_top3                                                  True\n",
      "tm_top4                                                  True\n",
      "tm_top5                                                  True\n",
      "tm_topall                                                True\n",
      "Name: 3, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 4/2240: (Q484523) Who is the lead singer of 30 seconds to Mars?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-4-redo-qanswer.pickle.bz2\n",
      "question     Who is the lead singer of 30 seconds to Mars?\n",
      "answer                                             Q484523\n",
      "qanswer                                               True\n",
      "platypus                                             False\n",
      "convex                                           Q14179172\n",
      "tm                                                   Q7187\n",
      "tm_time                                             103.67\n",
      "tm_top2                                              False\n",
      "tm_top3                                              False\n",
      "tm_top4                                              False\n",
      "tm_top5                                              False\n",
      "tm_topall                                            False\n",
      "Name: 4, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 5/2240: (1994-01-01T00:00:00Z) What was the first year Friends aired?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-5-redo-qanswer.pickle.bz2\n",
      "question     What was the first year Friends aired?\n",
      "answer                         1994-01-01T00:00:00Z\n",
      "qanswer                                        True\n",
      "platypus                                    Q387066\n",
      "convex                                        False\n",
      "tm                             2012-10-03T00:00:00Z\n",
      "tm_time                                      209.63\n",
      "tm_top2                                       False\n",
      "tm_top3                                       False\n",
      "tm_top4                                       False\n",
      "tm_top5                                       False\n",
      "tm_topall                                     False\n",
      "Name: 5, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 6/2240: (2005-01-01T00:00:00Z) In which year was the theatrical release of the movie Wedding Crashers?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-6-redo-qanswer.pickle.bz2\n",
      "question     In which year was the theatrical release of th...\n",
      "answer                                    2005-01-01T00:00:00Z\n",
      "qanswer                                                   True\n",
      "platypus                                                 False\n",
      "convex                                                   False\n",
      "tm                                                        Q183\n",
      "tm_time                                                 165.91\n",
      "tm_top2                                                  False\n",
      "tm_top3                                                  False\n",
      "tm_top4                                                  False\n",
      "tm_top5                                                  False\n",
      "tm_topall                                                False\n",
      "Name: 6, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 7/2240: (Q429828) Which character shares part of her name with the title of Grey's Anatomy?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-7-redo-qanswer.pickle.bz2\n",
      "question     Which character shares part of her name with t...\n",
      "answer                                                 Q429828\n",
      "qanswer                                                   True\n",
      "platypus                                                 False\n",
      "convex                                                   Q1860\n",
      "tm                                                     Q151411\n",
      "tm_time                                                 368.25\n",
      "tm_top2                                                  False\n",
      "tm_top3                                                  False\n",
      "tm_top4                                                  False\n",
      "tm_top5                                                  False\n",
      "tm_topall                                                False\n",
      "Name: 7, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 8/2240: (Q169889) Which network originally broadcasted Lost?                                  Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-8-redo-qanswer.pickle.bz2\n",
      "question     Which network originally broadcasted Lost?\n",
      "answer                                          Q169889\n",
      "qanswer                                            True\n",
      "platypus                                          False\n",
      "convex                                          Q169889\n",
      "tm                                               Q23567\n",
      "tm_time                                           89.85\n",
      "tm_top2                                           False\n",
      "tm_top3                                           False\n",
      "tm_top4                                           False\n",
      "tm_top5                                           False\n",
      "tm_topall                                         False\n",
      "Name: 8, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 9/2240: (1900-01-01T00:00:00Z) When was The Wonderful Wizard of Oz published?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-9-redo-qanswer.pickle.bz2\n",
      "question     When was The Wonderful Wizard of Oz published?\n",
      "answer                                 1900-01-01T00:00:00Z\n",
      "qanswer                                                True\n",
      "platypus                                              False\n",
      "convex                                                False\n",
      "tm                                     1939-01-01T00:00:00Z\n",
      "tm_time                                              319.08\n",
      "tm_top2                                               False\n",
      "tm_top3                                               False\n",
      "tm_top4                                               False\n",
      "tm_top5                                               False\n",
      "tm_topall                                              True\n",
      "Name: 9, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 10/2240: (1991-01-30T00:00:00Z) When was The Silence of the Lambs released?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-10-redo-qanswer.pickle.bz2\n",
      "question     When was The Silence of the Lambs released?\n",
      "answer                              1991-01-30T00:00:00Z\n",
      "qanswer                                             True\n",
      "platypus                                           False\n",
      "convex                                             False\n",
      "tm                                  1991-04-11T00:00:00Z\n",
      "tm_time                                           900.52\n",
      "tm_top2                                            False\n",
      "tm_top3                                            False\n",
      "tm_top4                                            False\n",
      "tm_top5                                             True\n",
      "tm_topall                                           True\n",
      "Name: 10, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 11/2240: (Q842256) What genre is Camp Rock?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-11-redo-qanswer.pickle.bz2\n",
      "question     What genre is Camp Rock?\n",
      "answer                        Q842256\n",
      "qanswer                          True\n",
      "platypus                        False\n",
      "convex                        Q842256\n",
      "tm                           Q4104520\n",
      "tm_time                         137.4\n",
      "tm_top2                         False\n",
      "tm_top3                         False\n",
      "tm_top4                         False\n",
      "tm_top5                          True\n",
      "tm_topall                        True\n",
      "Name: 11, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 12/2240: (Q838742) In which episode did Homer Simpson visit a store called Sneed's Feed & Seed?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-12-redo-qanswer.pickle.bz2\n",
      "question     In which episode did Homer Simpson visit a sto...\n",
      "answer                                                 Q838742\n",
      "qanswer                                                   True\n",
      "platypus                                                 False\n",
      "convex                                                 Q151076\n",
      "tm                                                        Q183\n",
      "tm_time                                                 323.38\n",
      "tm_top2                                                  False\n",
      "tm_top3                                                  False\n",
      "tm_top4                                                  False\n",
      "tm_top5                                                  False\n",
      "tm_topall                                                False\n",
      "Name: 12, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 13/2240: (Q80955) What was the first team he played for?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-13-redo-qanswer.pickle.bz2\n",
      "question     What was the first team he played for?\n",
      "answer                                       Q80955\n",
      "qanswer                                        True\n",
      "platypus                                    Q387066\n",
      "convex                                     Q1744559\n",
      "tm                                        Q46047837\n",
      "tm_time                                       54.83\n",
      "tm_top2                                       False\n",
      "tm_top3                                       False\n",
      "tm_top4                                       False\n",
      "tm_top5                                       False\n",
      "tm_topall                                     False\n",
      "Name: 13, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 14/2240: (Q838742) What's the episode with Sneed's Feed & Seed in The Simpsons?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-14-redo-qanswer.pickle.bz2\n",
      "question     What's the episode with Sneed's Feed & Seed in...\n",
      "answer                                                 Q838742\n",
      "qanswer                                                   True\n",
      "platypus                                                 False\n",
      "convex                                                      Q5\n",
      "tm                                                         618\n",
      "tm_time                                                 683.99\n",
      "tm_top2                                                  False\n",
      "tm_top3                                                  False\n",
      "tm_top4                                                  False\n",
      "tm_top5                                                  False\n",
      "tm_topall                                                False\n",
      "Name: 14, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 15/2240: (Q34660) Who is the author of the Harry Potter series?                                  Asking qAnswer\n",
      "\n",
      " 15 -> qAnswer Q34660\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-15-redo-qanswer.pickle.bz2\n",
      "question     Who is the author of the Harry Potter series?\n",
      "answer                                              Q34660\n",
      "qanswer                                               True\n",
      "platypus                                            Q34660\n",
      "convex                                              Q34660\n",
      "tm                                                  Q34660\n",
      "tm_time                                             224.07\n",
      "tm_top2                                               True\n",
      "tm_top3                                               True\n",
      "tm_top4                                               True\n",
      "tm_top5                                               True\n",
      "tm_topall                                             True\n",
      "Name: 15, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 16/2240: (1943-01-01T00:00:00Z) What year was Janis Joplin born?                                  Asking qAnswer\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-16-redo-qanswer.pickle.bz2\n",
      "question     What year was Janis Joplin born?\n",
      "answer                   1943-01-01T00:00:00Z\n",
      "qanswer                                  True\n",
      "platypus                                False\n",
      "convex                                  False\n",
      "tm                                   Q8060656\n",
      "tm_time                                 69.97\n",
      "tm_top2                                 False\n",
      "tm_top3                                 False\n",
      "tm_top4                                 False\n",
      "tm_top5                                 False\n",
      "tm_topall                               False\n",
      "Name: 16, dtype: object\n",
      "\n",
      "\n",
      "\t>>> Processing 17/2240: (Q276028) Who wrote the book The Secret Garden?                                  Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 17 -> qAnswer Q276028\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex/benchmarking-qanswer-platypus-convex-qagraph-from-0-to-17-redo-qanswer.pickle.bz2\n",
      "question     Who wrote the book The Secret Garden?\n",
      "answer                                     Q276028\n",
      "qanswer                                       True\n",
      "platypus                                     False\n",
      "convex                                     Q276028\n",
      "tm                                         Q238378\n",
      "tm_time                                      66.68\n",
      "tm_top2                                       True\n",
      "tm_top3                                       True\n",
      "tm_top4                                       True\n",
      "tm_top5                                       True\n",
      "tm_topall                                     True\n",
      "Name: 17, dtype: object\n",
      "\n",
      "\n",
      "\r",
      "\t>>> Processing 18/2240: (Q429828) Who is the character in Grey's Anatomy that shares her last name with part of the title?                                  Asking qAnswer\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 377\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-18de0587e341>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Asking qAnswer\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m         \u001b[0mresult_qanswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mask_qanswer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"-> qAnswer\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult_qanswer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresult_qanswer\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-cbf722d87f20>\u001b[0m in \u001b[0;36mask_qanswer\u001b[0;34m(question)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'query'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'lang'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'en'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'kb'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'wikidata'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mheaders\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"Authorization\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\"Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIzNDIiLCJpYXQiOjE1NzkyNTYxNDQsImV4cCI6MTU3OTg2MDk0NH0.YPFBZ-Xc8OI7eeTTkQaVT5a-CA5VONiCr_VIViG3t8tjVv7eRKgz_X_1KWDnly_F08rLXwpPcDUMBt8_M8-S8w\"\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mquery\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://qanswer-core1.univ-st-etienne.fr/api/gerbil'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m#    var settings = {\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/requests/api.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m    114\u001b[0m     \"\"\"\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'post'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    531\u001b[0m         }\n\u001b[1;32m    532\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    534\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    535\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 646\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    447\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    598\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1342\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1344\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1345\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1346\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    304\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    305\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 306\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    307\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    308\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 267\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    268\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "### Evaluate only qAnswer\n",
    "banning_str = False\n",
    "\n",
    "start_time = time.time()\n",
    "conversations_len = len(conversations)\n",
    "\n",
    "for i, conversation in enumerate(conversations):\n",
    "    if i >= 0:\n",
    "        questions = [turn['question'] for turn in conversation['questions']]\n",
    "        question = questions[0]\n",
    "        answers = [tmqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "        answer = answers[0]\n",
    "        print(\"\\r\\t>>> Processing {}/{}: ({}) {}\".format(i,conversations_len,answer,question), end='                                  ')\n",
    "        \n",
    "        print(\"Asking qAnswer\")\n",
    "        result_qanswer = ask_qanswer(question)\n",
    "        print(\"\\n\",i, \"-> qAnswer\", result_qanswer) if result_qanswer == answer else False\n",
    "        \n",
    "        df['qanswer'][i] = result_qanswer\n",
    "        \n",
    "        pickle_data(df, \"benchmarking-qanswer-platypus-convex-qagraph-from-0-to-\"+str(i)+\"-redo-qanswer\")\n",
    "        \n",
    "        print(df.loc[i,:])\n",
    "        print(\"\\n\")\n",
    "    \n",
    "    #break\n",
    "\n",
    "print(\"->\\tRunning time is {}s\".format(round(time.time()-start_time,2)))\n",
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question      object\n",
      "answer        object\n",
      "qanswer       object\n",
      "platypus      object\n",
      "convex        object\n",
      "tm            object\n",
      "tm_time      float64\n",
      "tm_top2       object\n",
      "tm_top3       object\n",
      "tm_top4       object\n",
      "tm_top5       object\n",
      "tm_topall     object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1800"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result_convex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>qanswer</th>\n",
       "      <th>platypus</th>\n",
       "      <th>convex</th>\n",
       "      <th>tm</th>\n",
       "      <th>tm_time</th>\n",
       "      <th>tm_top2</th>\n",
       "      <th>tm_top3</th>\n",
       "      <th>tm_top4</th>\n",
       "      <th>tm_top5</th>\n",
       "      <th>tm_topall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2230</th>\n",
       "      <td>Who wrote Lord of the Rings?</td>\n",
       "      <td>Q892</td>\n",
       "      <td>False</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q1860</td>\n",
       "      <td>143.01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2231</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>False</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>158.89</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2232</th>\n",
       "      <td>Who commissioned at the start of MLS?</td>\n",
       "      <td>Q5300675</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q36467673</td>\n",
       "      <td>130.10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2233</th>\n",
       "      <td>Who is the head coach of the Orlando City SC?</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>False</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>False</td>\n",
       "      <td>182.29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2234</th>\n",
       "      <td>Which person is the author of the Lord of the ...</td>\n",
       "      <td>Q892</td>\n",
       "      <td>False</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>432.09</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2235</th>\n",
       "      <td>What was the name of the first album by Metall...</td>\n",
       "      <td>Q3284678</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q726251</td>\n",
       "      <td>False</td>\n",
       "      <td>98.33</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2236</th>\n",
       "      <td>Who directed Interstellar?</td>\n",
       "      <td>Q13417189</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>146.39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2237</th>\n",
       "      <td>What was the name of the first soccer team he ...</td>\n",
       "      <td>Q80955</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q3608832</td>\n",
       "      <td>False</td>\n",
       "      <td>141.99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2238</th>\n",
       "      <td>What was the first Pixar movie?</td>\n",
       "      <td>Q171048</td>\n",
       "      <td>False</td>\n",
       "      <td>Q387066</td>\n",
       "      <td>False</td>\n",
       "      <td>Q23337</td>\n",
       "      <td>258.00</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2239</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>False</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>154.94</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question     answer qanswer  \\\n",
       "2230                       Who wrote Lord of the Rings?       Q892   False   \n",
       "2231                 What country is Lionel Messi from?       Q414   False   \n",
       "2232              Who commissioned at the start of MLS?   Q5300675   False   \n",
       "2233      Who is the head coach of the Orlando City SC?   Q1337446   False   \n",
       "2234  Which person is the author of the Lord of the ...       Q892   False   \n",
       "2235  What was the name of the first album by Metall...   Q3284678   False   \n",
       "2236                         Who directed Interstellar?  Q13417189   False   \n",
       "2237  What was the name of the first soccer team he ...     Q80955   False   \n",
       "2238                    What was the first Pixar movie?    Q171048   False   \n",
       "2239                 What country is Lionel Messi from?       Q414   False   \n",
       "\n",
       "      platypus    convex         tm  tm_time tm_top2 tm_top3 tm_top4 tm_top5  \\\n",
       "2230      Q892      Q892      Q1860   143.01   False   False   False   False   \n",
       "2231      Q615       Q29      False   158.89   False   False   False   False   \n",
       "2232     False     False  Q36467673   130.10   False   False   False   False   \n",
       "2233  Q1337446  Q1337446      False   182.29   False   False   False   False   \n",
       "2234      Q892      Q892       Q892   432.09    True    True    True    True   \n",
       "2235     False   Q726251      False    98.33   False   False   False   False   \n",
       "2236     False    Q25191     Q25191   146.39   False   False   False   False   \n",
       "2237     False  Q3608832      False   141.99   False   False   False   False   \n",
       "2238   Q387066     False     Q23337   258.00   False   False   False   False   \n",
       "2239      Q615       Q29      False   154.94   False   False   False   False   \n",
       "\n",
       "     tm_topall  \n",
       "2230     False  \n",
       "2231     False  \n",
       "2232     False  \n",
       "2233     False  \n",
       "2234      True  \n",
       "2235     False  \n",
       "2236     False  \n",
       "2237     False  \n",
       "2238     False  \n",
       "2239     False  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>qanswer</th>\n",
       "      <th>platypus</th>\n",
       "      <th>convex</th>\n",
       "      <th>tm</th>\n",
       "      <th>tm_time</th>\n",
       "      <th>tm_top2</th>\n",
       "      <th>tm_top3</th>\n",
       "      <th>tm_top4</th>\n",
       "      <th>tm_top5</th>\n",
       "      <th>tm_topall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2230</th>\n",
       "      <td>Who wrote Lord of the Rings?</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q1860</td>\n",
       "      <td>143.01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2231</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>158.89</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2232</th>\n",
       "      <td>Who commissioned at the start of MLS?</td>\n",
       "      <td>Q5300675</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q36467673</td>\n",
       "      <td>130.10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2233</th>\n",
       "      <td>Who is the head coach of the Orlando City SC?</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>False</td>\n",
       "      <td>182.29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2234</th>\n",
       "      <td>Which person is the author of the Lord of the ...</td>\n",
       "      <td>Q892</td>\n",
       "      <td>False</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>432.09</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2235</th>\n",
       "      <td>What was the name of the first album by Metall...</td>\n",
       "      <td>Q3284678</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>Q726251</td>\n",
       "      <td>False</td>\n",
       "      <td>98.33</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2236</th>\n",
       "      <td>Who directed Interstellar?</td>\n",
       "      <td>Q13417189</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>False</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>146.39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2237</th>\n",
       "      <td>What was the name of the first soccer team he ...</td>\n",
       "      <td>Q80955</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q3608832</td>\n",
       "      <td>False</td>\n",
       "      <td>141.99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2238</th>\n",
       "      <td>What was the first Pixar movie?</td>\n",
       "      <td>Q171048</td>\n",
       "      <td></td>\n",
       "      <td>Q387066</td>\n",
       "      <td>False</td>\n",
       "      <td>Q23337</td>\n",
       "      <td>258.00</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2239</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>154.94</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question     answer   qanswer  \\\n",
       "2230                       Who wrote Lord of the Rings?       Q892      Q892   \n",
       "2231                 What country is Lionel Messi from?       Q414      Q414   \n",
       "2232              Who commissioned at the start of MLS?   Q5300675             \n",
       "2233      Who is the head coach of the Orlando City SC?   Q1337446  Q1337446   \n",
       "2234  Which person is the author of the Lord of the ...       Q892     False   \n",
       "2235  What was the name of the first album by Metall...   Q3284678             \n",
       "2236                         Who directed Interstellar?  Q13417189    Q25191   \n",
       "2237  What was the name of the first soccer team he ...     Q80955     False   \n",
       "2238                    What was the first Pixar movie?    Q171048             \n",
       "2239                 What country is Lionel Messi from?       Q414      Q414   \n",
       "\n",
       "      platypus    convex         tm  tm_time tm_top2 tm_top3 tm_top4 tm_top5  \\\n",
       "2230      Q892      Q892      Q1860   143.01   False   False   False   False   \n",
       "2231      Q615       Q29      False   158.89   False   False   False   False   \n",
       "2232     False     False  Q36467673   130.10   False   False   False   False   \n",
       "2233  Q1337446  Q1337446      False   182.29   False   False   False   False   \n",
       "2234      Q892      Q892       Q892   432.09    True    True    True    True   \n",
       "2235     False   Q726251      False    98.33   False   False   False   False   \n",
       "2236     False    Q25191     Q25191   146.39   False   False   False   False   \n",
       "2237     False  Q3608832      False   141.99   False   False   False   False   \n",
       "2238   Q387066     False     Q23337   258.00   False   False   False   False   \n",
       "2239      Q615       Q29      False   154.94   False   False   False   False   \n",
       "\n",
       "     tm_topall  \n",
       "2230     False  \n",
       "2231     False  \n",
       "2232     False  \n",
       "2233     False  \n",
       "2234      True  \n",
       "2235     False  \n",
       "2236     False  \n",
       "2237     False  \n",
       "2238     False  \n",
       "2239     False  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>qanswer</th>\n",
       "      <th>platypus</th>\n",
       "      <th>convex</th>\n",
       "      <th>tm</th>\n",
       "      <th>tm_time</th>\n",
       "      <th>tm_top2</th>\n",
       "      <th>tm_top3</th>\n",
       "      <th>tm_top4</th>\n",
       "      <th>tm_top5</th>\n",
       "      <th>tm_topall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2230</th>\n",
       "      <td>Who wrote Lord of the Rings?</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q1860</td>\n",
       "      <td>143.01</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2231</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>158.89</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2232</th>\n",
       "      <td>Who commissioned at the start of MLS?</td>\n",
       "      <td>Q5300675</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q36467673</td>\n",
       "      <td>130.10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2233</th>\n",
       "      <td>Who is the head coach of the Orlando City SC?</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>Q1337446</td>\n",
       "      <td>False</td>\n",
       "      <td>182.29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2234</th>\n",
       "      <td>Which person is the author of the Lord of the ...</td>\n",
       "      <td>Q892</td>\n",
       "      <td>False</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>Q892</td>\n",
       "      <td>432.09</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2235</th>\n",
       "      <td>What was the name of the first album by Metall...</td>\n",
       "      <td>Q3284678</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q726251</td>\n",
       "      <td>False</td>\n",
       "      <td>98.33</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2236</th>\n",
       "      <td>Who directed Interstellar?</td>\n",
       "      <td>Q13417189</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>False</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>Q25191</td>\n",
       "      <td>146.39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2237</th>\n",
       "      <td>What was the name of the first soccer team he ...</td>\n",
       "      <td>Q80955</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Q3608832</td>\n",
       "      <td>False</td>\n",
       "      <td>141.99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2238</th>\n",
       "      <td>What was the first Pixar movie?</td>\n",
       "      <td>Q171048</td>\n",
       "      <td>False</td>\n",
       "      <td>Q387066</td>\n",
       "      <td>False</td>\n",
       "      <td>Q23337</td>\n",
       "      <td>258.00</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2239</th>\n",
       "      <td>What country is Lionel Messi from?</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q414</td>\n",
       "      <td>Q615</td>\n",
       "      <td>Q29</td>\n",
       "      <td>False</td>\n",
       "      <td>154.94</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question     answer   qanswer  \\\n",
       "2230                       Who wrote Lord of the Rings?       Q892      Q892   \n",
       "2231                 What country is Lionel Messi from?       Q414      Q414   \n",
       "2232              Who commissioned at the start of MLS?   Q5300675     False   \n",
       "2233      Who is the head coach of the Orlando City SC?   Q1337446  Q1337446   \n",
       "2234  Which person is the author of the Lord of the ...       Q892     False   \n",
       "2235  What was the name of the first album by Metall...   Q3284678     False   \n",
       "2236                         Who directed Interstellar?  Q13417189    Q25191   \n",
       "2237  What was the name of the first soccer team he ...     Q80955     False   \n",
       "2238                    What was the first Pixar movie?    Q171048     False   \n",
       "2239                 What country is Lionel Messi from?       Q414      Q414   \n",
       "\n",
       "      platypus    convex         tm  tm_time  tm_top2  tm_top3  tm_top4  \\\n",
       "2230      Q892      Q892      Q1860   143.01    False    False    False   \n",
       "2231      Q615       Q29      False   158.89    False    False    False   \n",
       "2232     False     False  Q36467673   130.10    False    False    False   \n",
       "2233  Q1337446  Q1337446      False   182.29    False    False    False   \n",
       "2234      Q892      Q892       Q892   432.09     True     True     True   \n",
       "2235     False   Q726251      False    98.33    False    False    False   \n",
       "2236     False    Q25191     Q25191   146.39    False    False    False   \n",
       "2237     False  Q3608832      False   141.99    False    False    False   \n",
       "2238   Q387066     False     Q23337   258.00    False    False    False   \n",
       "2239      Q615       Q29      False   154.94    False    False    False   \n",
       "\n",
       "      tm_top5  tm_topall  \n",
       "2230    False      False  \n",
       "2231    False      False  \n",
       "2232    False      False  \n",
       "2233    False      False  \n",
       "2234     True       True  \n",
       "2235    False      False  \n",
       "2236    False      False  \n",
       "2237    False      False  \n",
       "2238    False      False  \n",
       "2239    False      False  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RECALL\n",
      "qanswer: 0.3053571428571429\n",
      "platypus: 0.22455357142857144\n",
      "convex: 0.6089285714285714\n",
      "tm1: 0.81875\n",
      "PRECISION\n",
      "qanswer: 0.15535714285714286\n",
      "platypus: 0.09375\n",
      "convex: 0.22053571428571428\n",
      "tm1: 0.09241071428571429\n",
      "F1\n",
      "qanswer: 0.20593853820598007\n",
      "platypus: 0.13227559607293127\n",
      "convex: 0.32380055359065046\n",
      "tm1: 0.16607667809897111\n"
     ]
    }
   ],
   "source": [
    "max_row = len(df_copy)\n",
    "\n",
    "df_qanswer_recall = (max_row-len(df_copy[(df_copy.index<=max_row) & (df_copy.qanswer == False)]))/max_row\n",
    "df_platypus_recall = (max_row-len(df_copy[(df_copy.index<=max_row) & (df_copy.platypus == False)]))/max_row\n",
    "df_convex_recall = (max_row-len(df_copy[(df_copy.index<=max_row) & (df_copy.convex == False)]))/max_row\n",
    "df_tm1_recall = (max_row-len(df_copy[(df_copy.index<=max_row) & (df_copy.tm == False)]))/max_row\n",
    "\n",
    "df_qanswer_precision = len(df_copy[(df_copy.index<=max_row) & (df_copy.qanswer == df_copy.answer)])/max_row\n",
    "df_platypus_precision = len(df_copy[(df_copy.index<=max_row) & (df_copy.platypus == df_copy.answer)])/max_row\n",
    "df_convex_precision = len(df_copy[(df_copy.index<=max_row) & (df_copy.convex == df_copy.answer)])/max_row\n",
    "df_tm1_precision = len(df_copy[(df_copy.index<=max_row) & (df_copy.tm == df_copy.answer)])/max_row\n",
    "\n",
    "\n",
    "\n",
    "print(\"RECALL\")\n",
    "print(\"qanswer:\", df_qanswer_recall)\n",
    "print(\"platypus:\", df_platypus_recall)\n",
    "print(\"convex:\", df_convex_recall)\n",
    "print(\"tm1:\", df_tm1_recall)\n",
    "                            \n",
    "                            \n",
    "print(\"PRECISION\")\n",
    "print(\"qanswer:\", df_qanswer_precision)\n",
    "print(\"platypus:\", df_platypus_precision)\n",
    "print(\"convex:\", df_convex_precision)\n",
    "print(\"tm1:\", df_tm1_precision)\n",
    "\n",
    "print(\"F1\")\n",
    "print(\"qanswer:\", 2*((df_qanswer_precision*df_qanswer_recall)/(df_qanswer_precision+df_qanswer_recall)))\n",
    "print(\"platypus:\", 2*((df_platypus_precision*df_platypus_recall)/(df_platypus_precision+df_platypus_recall)))\n",
    "print(\"convex:\", 2*((df_convex_precision*df_convex_recall)/(df_convex_precision+df_convex_recall)))\n",
    "print(\"tm1:\", 2*((df_tm1_precision*df_tm1_recall)/(df_tm1_precision+df_tm1_recall)))\n",
    "\n",
    "\n",
    "#df[ & (df.qanswer == df.source)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.5"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(1+2+3+4)/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.5"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(5+6+7+8)/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.5"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(1+2+3+4+5+6+7+8)/8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.5"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(2.5+6.5)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SAVING\n",
    "#pickle_data(df_loaded, \"benchmarking-qanswer-platypus-convex-tm1-from-0-to-\"+str(len(df_loaded)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOADING\n",
    "#df_loaded = pd.read_pickle(\"/data/users/romain.claret/tm/wikidata-simplequestions/benchmark_pickles/benchmarking-qanswer-platypus-convex-tm1-from-0-to-9961.pickle.bz2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['qanswer'][34] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del df_loaded['tm2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.rename({'mine':'tm1'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['tm1_top4'] = \"False\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded[['question','source','qanswer','platypus','convex','tm1','tm1_time','tm1_top2','tm1_top3','tm1_top4','tm1_top5','tm1_topall']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded_len = len(df_loaded)\n",
    "#for i, question in enumerate(df_loaded['question']):\n",
    "#    if i >= 0:\n",
    "#    #if i >= 497:\n",
    "#        source = str(df_loaded['source'][i])\n",
    "#        print(str(i)+\"/\"+str(df_loaded_len),question,\"-> source:\",source)\n",
    "#        \n",
    "#        start_time = time.time()\n",
    "#        result_tmqa_1 = ask_tmqa_1(question, verbose=True)\n",
    "#        \n",
    "#        if result_tmqa_1:\n",
    "#            df_loaded['tm1'][i] = result_tmqa_1[0]\n",
    "#            if source in result_tmqa_1[:2]:\n",
    "#                df_loaded['tm1_top2'][i] = True\n",
    "#            if source in result_tmqa_1[:3]:\n",
    "#                df_loaded['tm1_top3'][i] = True\n",
    "#            if source in result_tmqa_1[:4]:\n",
    "#                df_loaded['tm1_top4'][i] = True\n",
    "#            if source in result_tmqa_1[:5]:\n",
    "#                df_loaded['tm1_top5'][i] = True\n",
    "#            if source in result_tmqa_1:\n",
    "#                df_loaded['tm1_topall'][i] = True\n",
    "#        else:\n",
    "#            df_loaded['tm1'][i] = False\n",
    "#        end_time = time.time()\n",
    "#        df_loaded['tm1_time'][i] = round(end_time-start_time,2)\n",
    "#        print(\"->\\tRunning time is {}s\".format(round(end_time-start_time,2)))\n",
    "#        print(str(str(df_loaded['tm1'][i])==str(source)),\"---> result_tmqa_1:\",str(result_tmqa_1)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded.copy()\n",
    "#df = df.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_backup = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_row = 496\n",
    "#df_len = len(df)\n",
    "#df_qanswer_max = df[(df.index<=max_row) & (df.qanswer == df.source)]\n",
    "#df_qanswer_max_len = len(df_qanswer_max)\n",
    "#\n",
    "#df_platypus_max = df[(df.index<=max_row) & (df.platypus == df.source)]\n",
    "#df_platypus_max_len = len(df_platypus_max)\n",
    "#\n",
    "#df_convex_max = df[(df.index<=max_row) & (df.convex == df.source)]\n",
    "#df_convex_max_len = len(df_convex_max)\n",
    "#\n",
    "#df_tm1_max = df[(df.index<=max_row) & (df.tm1 == df.source)]\n",
    "#df_tm1_max_len = len(df_tm1_max)\n",
    "#\n",
    "#df_tm1_max_top2 = df[(df.index<=max_row) & (df.tm1_top2 == True)]\n",
    "#df_tm1_max_top2_len = len(df_tm1_max_top2)\n",
    "#\n",
    "#df_tm1_max_top3 = df[(df.index<=max_row) & (df.tm1_top3 == True)]\n",
    "#df_tm1_max_top3_len = len(df_tm1_max_top3)\n",
    "#\n",
    "#df_tm1_max_top4 = df[(df.index<=max_row) & (df.tm1_top4 == True)]\n",
    "#df_tm1_max_top4_len = len(df_tm1_max_top4)\n",
    "#\n",
    "#df_tm1_max_top5 = df[(df.index<=max_row) & (df.tm1_top5 == True)]\n",
    "#df_tm1_max_top5_len = len(df_tm1_max_top5)\n",
    "#\n",
    "#df_tm1_max_topall = df[(df.index<=max_row) & (df.tm1_topall == True)]\n",
    "#df_tm1_max_topall_len = len(df_tm1_max_topall)\n",
    "#\n",
    "#print(\"qanswer:\", df_qanswer_max_len,df_qanswer_max_len/max_row)\n",
    "#print(\"platypus:\", df_platypus_max_len, df_platypus_max_len/max_row)\n",
    "#print(\"convex:\", df_convex_max_len, df_convex_max_len/max_row)\n",
    "#print(\"tm1:\", df_tm1_max_len, df_tm1_max_len/max_row)\n",
    "#print(\"tm1_top2:\", df_tm1_max_top2_len, df_tm1_max_top2_len/max_row)\n",
    "#print(\"tm1_top3:\", df_tm1_max_top3_len, df_tm1_max_top3_len/max_row)\n",
    "#print(\"tm1_top4:\", df_tm1_max_top4_len, df_tm1_max_top4_len/max_row)\n",
    "#print(\"tm1_top5:\", df_tm1_max_top5_len, df_tm1_max_top5_len/max_row)\n",
    "#print(\"tm1_topall:\", df_tm1_max_topall_len, df_tm1_max_topall_len/max_row)\n",
    "#\n",
    "#df[ & (df.qanswer == df.source)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"qanswer:\",len(df[df.qanswer == df.source]),len(df[df.qanswer == df.source])/len(df))\n",
    "#print(\"platypus:\",len(df[df.platypus == df.source]),len(df[df.platypus == df.source])/len(df))\n",
    "#print(\"convex:\",len(df[df.convex == df.source]),len(df[df.convex == df.source])/len(df))\n",
    "#print(\"tm1:\",len(df[df.tm1 == df.source]),len(df[df.tm1 == df.source])/len(df))\n",
    "#print(\"tm1_top2:\",len(df[df.tm1_top2 == True]),len(df[df.tm1_top2 == True])/len(df))\n",
    "#print(\"tm1_top3:\",len(df[df.tm1_top3 == True]),len(df[df.tm1_top3 == True])/len(df))\n",
    "#print(\"tm1_top4:\",len(df[df.tm1_top4 == True]),len(df[df.tm1_top4 == True])/len(df))\n",
    "#print(\"tm1_top5:\",len(df[df.tm1_top5 == True]),len(df[df.tm1_top5 == True])/len(df))\n",
    "#print(\"tm1_topall:\",len(df[df.tm1_topall == True]),len(df[df.tm1_topall == True])/len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tm1_top2 = df.tm1_top3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:qa]",
   "language": "python",
   "name": "conda-env-qa-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}