playground-benchmarking-conversational-multihop-qa-systems-Copy2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(180000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 180 seconds\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:25: The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/txt2txt/txt2txt.py:27: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading the params file\n",
      "Input encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Input decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "Output encoding {'o': 2, '{': 3, '.': 4, 'J': 5, '0': 6, '1': 7, '<': 8, 'B': 9, 'd': 10, '£': 11, 'e': 12, '6': 13, '!': 14, 'O': 15, 'M': 16, 'X': 17, 'f': 18, 't': 19, 'C': 20, 'V': 21, 'z': 22, 'K': 23, '\\\\': 24, '9': 25, 'P': 26, 'S': 27, '/': 28, '₹': 29, 'F': 30, 'G': 31, '=': 32, '8': 33, ')': 34, '+': 35, ']': 36, 'U': 37, \"'\": 38, '\"': 39, 'g': 40, 'N': 41, 'r': 42, 'u': 43, '&': 44, '$': 45, 'x': 46, '%': 47, ':': 48, '@': 49, '^': 50, 'I': 51, 'L': 52, 'Z': 53, 'h': 54, 'W': 55, 'A': 56, 'v': 57, '?': 58, '2': 59, '~': 60, 's': 61, 'T': 62, 'R': 63, ',': 64, '|': 65, '4': 66, '>': 67, 'y': 68, '(': 69, '[': 70, 'k': 71, 'H': 72, 'l': 73, 'j': 74, '7': 75, 'n': 76, 'i': 77, 'D': 78, 'Q': 79, ' ': 80, 'm': 81, 'Y': 82, '*': 83, '}': 84, '#': 85, 'p': 86, 'q': 87, '5': 88, 'c': 89, '`': 90, 'a': 91, 'b': 92, 'w': 93, '3': 94, 'E': 95, ';': 96, '-': 97}\n",
      "Output decoding {2: 'o', 3: '{', 4: '.', 5: 'J', 6: '0', 7: '1', 8: '<', 9: 'B', 10: 'd', 11: '£', 12: 'e', 13: '6', 14: '!', 15: 'O', 16: 'M', 17: 'X', 18: 'f', 19: 't', 20: 'C', 21: 'V', 22: 'z', 23: 'K', 24: '\\\\', 25: '9', 26: 'P', 27: 'S', 28: '/', 29: '₹', 30: 'F', 31: 'G', 32: '=', 33: '8', 34: ')', 35: '+', 36: ']', 37: 'U', 38: \"'\", 39: '\"', 40: 'g', 41: 'N', 42: 'r', 43: 'u', 44: '&', 45: '$', 46: 'x', 47: '%', 48: ':', 49: '@', 50: '^', 51: 'I', 52: 'L', 53: 'Z', 54: 'h', 55: 'W', 56: 'A', 57: 'v', 58: '?', 59: '2', 60: '~', 61: 's', 62: 'T', 63: 'R', 64: ',', 65: '|', 66: '4', 67: '>', 68: 'y', 69: '(', 70: '[', 71: 'k', 72: 'H', 73: 'l', 74: 'j', 75: '7', 76: 'n', 77: 'i', 78: 'D', 79: 'Q', 80: ' ', 81: 'm', 82: 'Y', 83: '*', 84: '}', 85: '#', 86: 'p', 87: 'q', 88: '5', 89: 'c', 90: '`', 91: 'a', 92: 'b', 93: 'w', 94: '3', 95: 'E', 96: ';', 97: '-'}\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:3673: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 202)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 202, 256)     25088       input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 202, 128)     12544       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, 202, 256)     525312      embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 202, 256)     263168      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 202, 202)     0           lstm_2[0][0]                     \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "attention (Activation)          (None, 202, 202)     0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 202, 256)     0           attention[0][0]                  \n",
      "                                                                 bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 202, 512)     0           dot_2[0][0]                      \n",
      "                                                                 lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 202, 128)     65664       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 202, 98)      12642       time_distributed_1[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 904,418\n",
      "Trainable params: 904,418\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Hi! My PID is 5024\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import time\n",
    "from datetime import datetime\n",
    "import networkx as nx\n",
    "import convex as cx\n",
    "import tmqa35 as graphqa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import importlib\n",
    "#import tmqa35 as graphqa\n",
    "#importlib.reload(graphqa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "conversations_path = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/data/convex/test_set/test_set_ALL.json\"\n",
    "with open(conversations_path, \"r\") as data:\n",
    "    conversations = json.load(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pickle_data(df, filename):\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    filename = \"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/\"+filename+'.pickle.bz2'\n",
    "    #df.summary = df.summary.map(sanitize_str)\n",
    "    print(\"Saving Dataframe Done!\",filename)\n",
    "    return df.to_pickle(filename, compression='bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardize_graph(graph):\n",
    "    this_graph = graph.copy()\n",
    "    for n in this_graph.nodes():\n",
    "        n_pos = n.find(\"-\")\n",
    "        n_name = n\n",
    "        if n_pos != -1: n_name = n[:n_pos]\n",
    "        this_graph.nodes[n][\"name\"] = graphqa.get_wd_label(n_name)\n",
    "        this_graph.nodes[n][\"weight\"] = 1\n",
    "        \n",
    "    return this_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lang: en, fr, de, it, es, zh\n",
    "#kb: dbpedia, wikidata, dblp, freebase\n",
    "def ask_qanswer(question):\n",
    "    data = {'query': question,'lang': 'en','kb': 'wikidata'}\n",
    "    headers = {\"Authorization\":\"Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIzNDIiLCJpYXQiOjE1NzkyNTYxNDQsImV4cCI6MTU3OTg2MDk0NH0.YPFBZ-Xc8OI7eeTTkQaVT5a-CA5VONiCr_VIViG3t8tjVv7eRKgz_X_1KWDnly_F08rLXwpPcDUMBt8_M8-S8w\"}\n",
    "    query = requests.post('http://qanswer-core1.univ-st-etienne.fr/api/gerbil', data=data, headers=headers)\n",
    "    \n",
    "#    var settings = {\n",
    "#  \"async\": true,\n",
    "#  \"crossDomain\": true,\n",
    "#  \"url\": \"http://qanswer-core1.univ-st-etienne.fr/api/qa/full?question=what%20is%20a%20margerita&lang=en&kb=cocktails\",\n",
    "#  \"method\": \"GET\",\n",
    "#  \"headers\": {\n",
    "#    \"Authorization\": \"Bearer eyJhbGciOiJIUzUxMi.....\",\n",
    "#  }\n",
    "#}\n",
    "    \n",
    "    if not query:\n",
    "        return False,False\n",
    "    if (query.json()['questions'][0]['question']['answers']) == None:\n",
    "        return False,False\n",
    "    #if (query.json()['questions'][0]['question']['answers'].replace('\\n', '')) == None:\n",
    "    #    return False\n",
    "    #print(query.json()['questions'][0]['question']['answers'].replace('\\n', '').get(\"results\"))\n",
    "    try:\n",
    "        response = (json.loads(query.json()\n",
    "                .get(\"questions\")[0]\n",
    "                .get(\"question\")\n",
    "                .get(\"answers\")\n",
    "                .replace('\\n', ''))\n",
    "         .get(\"results\").get(\"bindings\"))\n",
    "    except:\n",
    "        return False,False\n",
    "    \n",
    "    if response:\n",
    "        result = response[0].get(\"o1\").get(\"value\")[len(\"http://www.wikidata.org/entity/\"):] if response[0].get(\"o1\") is not None else False\n",
    "        return result,False\n",
    "    else:\n",
    "        return False,False\n",
    "\n",
    "#ask_qanswer(\"Who is the wife of Barack Obama\")\n",
    "#ask_qanwser(\"Which equestrian was born in dublin?\")\n",
    "#ask_qanswer(\"what is the main language spoken in a ghentar si muore facile\")\n",
    "#ask_qanswer(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_qanswer(\"how does engelbert zaschka identify\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_platypus(question):\n",
    "    headers = {'Accept': 'application/json','Accept-Language': 'en',}\n",
    "    params = (('q', question),('lang', 'en'))\n",
    "\n",
    "    response = requests.get('https://qa.askplatyp.us/v0/ask', headers=headers, params=params)\n",
    "    if response:\n",
    "        if type(response.json()['member']) is list:\n",
    "            #print(response.json()['member'][0]['result'])\n",
    "            if response.json()['member'] != []:\n",
    "                if '@id' in (json.dumps(response.json()['member'][0]['result'])):\n",
    "                    try:\n",
    "                        ps_result = (json.dumps(response.json()['member'][0]['result']['@id']))\n",
    "                    except:\n",
    "                        return False, False\n",
    "                else: return False, False\n",
    "            else: return False, False\n",
    "        else:\n",
    "            try:\n",
    "                if '@id' in (json.dumps(response.json()['member']['result'])):\n",
    "                    ps_result = (json.dumps(response.json()[\"member\"]['result']['@id']))\n",
    "                else: return False, False\n",
    "            except:\n",
    "                return False, False\n",
    "    else: return False, False\n",
    "    ps_result = ps_result[4:-1]\n",
    "    #print(result[:1])\n",
    "    #if ps_result[:1] != 'P' and ps_result[:1] != 'Q':\n",
    "    #    return False, False\n",
    "    return ps_result,False\n",
    "\n",
    "#ask_platypus(\"Which genre of album is harder.....faster?\")\n",
    "#ask_platypus(\"how does engelbert zaschka identify\")\n",
    "#ask_platypus(\"Which Swiss conductor's cause of death is myoc...\")\n",
    "#ask_platypus(\"where was padraic mcguinness's place of death\")\n",
    "#ask_platypus(\"was the film helpmates in color or black-and-white?\")\n",
    "#ask_platypus(\"Who created the show life on earth\")\n",
    "#ask_platypus(\"Who is the wife of Barack Obama\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_convex(question):\n",
    "    cx_result = cx.answer_complete_question(question, cx.tagmeToken)\n",
    "    graph = cx.gp.expand_context_with_statements(None, [cx_result['context']], qa=True) \n",
    "    graph = standardize_graph(graph)\n",
    "    #print(cx_result)\n",
    "    #answer = str(cx.wd.wikidata_id_to_label(result['answers'][0]['answer']))\n",
    "    try:\n",
    "        if not cx_result:\n",
    "            return False, False\n",
    "        return [[r[\"answer\"] for r in cx_result['answers']],\n",
    "                [cx_result['context'][\"entity\"][\"id\"],cx_result['context'][\"predicate\"][\"id\"],cx_result['context'][\"object\"][\"id\"]]], graph\n",
    "    except:\n",
    "        return False, False\n",
    "\n",
    "#ask_convex(\"Which actor voiced the Unicorn in The Last Unicorn?\")\n",
    "#ask_convex(\"Which genre of album is harder.....faster?\")\n",
    "#ask_convex(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_convex(\"Who is the wife of Barack Obama\")\n",
    "#ask_convex(\"100% senorita is a television show in what language?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_graphqa(question, verbose=True, timer=True, show_graph=False, cores=graphqa.mp.cpu_count(), banning_str=False,\n",
    "            answer_context=False, context_graph=False, use_convex=False, turn=1):\n",
    "    \n",
    "    frontier_detection=[0.9, 0.6, 0.3] #random_access\n",
    "    answer_detection=[0.9, 0.1] #total_distance_qa_nodes, total_distance_frontiers\n",
    "    frontiers=3\n",
    "    \n",
    "    if use_convex:\n",
    "        if not context_graph:\n",
    "            context_graph=nx.Graph()\n",
    "            if answer_context:\n",
    "                if answer_context[0]:\n",
    "                    context_graph.add_node(answer_context[0][0], name=graphqa.get_wd_label(answer_context[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                \n",
    "        answer_context_convex,context_graph = cx.answer_follow_up_question(question, turn, context_graph, frontier_detection+answer_detection, frontiers)\n",
    "        if context_graph: context_graph = standardize_graph(context_graph)\n",
    "        #answer_context = False\n",
    "        #print(\"answer_context\",answer_context[0]['rank'])\n",
    "        \n",
    "        answer_context=[]\n",
    "        for ac in answer_context_convex:\n",
    "            answer_context.append(ac[\"answer\"])\n",
    "        answer_context = [answer_context,[]]\n",
    "        \n",
    "        if show_graph: graphqa.plot_graph(context_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "        #if verbose: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "        result = answer_context,context_graph\n",
    "    \n",
    "    else:\n",
    "        result = graphqa.answer_question(\n",
    "            question, verbose=verbose, timer=timer, show_graph=show_graph, cores=cores, banning_str=banning_str,\n",
    "            previous_answer=answer_context, previous_graph=context_graph)\n",
    "                    \n",
    "    if not result:\n",
    "        return (False,False)\n",
    "    #if result == (False,False):\n",
    "    #    return (False,False)\n",
    "\n",
    "    return result\n",
    "\n",
    "conversation_questions = [\n",
    "    \"Which actor voiced the Unicorn in The Last Unicorn?\",\n",
    "    \"And Alan Arkin was behind..\",\n",
    "    \"Who did the score?\",\n",
    "    \"So who performed the songs?\",\n",
    "    \"Genre of this band's music?\",\n",
    "    \"By the way, who was the director?\",\n",
    "    \"Is Alan Arkin in the cast ?\",\n",
    "]\n",
    "\n",
    "\n",
    "#for i_q,question in enumerate(conversation_questions):\n",
    "#    if i_q >= 0:\n",
    "#        if i_q == 0:\n",
    "#            answer_context_1,context_graph_1 = ask_graphqa(question ,answer_context=False, context_graph=False, verbose=True, timer=True, show_graph=True)\n",
    "#            answer_context = answer_context_1.copy()\n",
    "#            context_graph = context_graph_1.copy()\n",
    "#        elif context_graph:\n",
    "#            print(\"Context Question:\",question)\n",
    "#            answer_context,context_graph = ask_graphqa(question,answer_context=answer_context, context_graph=context_graph, verbose=True, timer=True, show_graph=True)\n",
    "#        else:\n",
    "#            print(\"NO CONTEXT ERROR\")\n",
    "#            break\n",
    "#\n",
    "#    if answer_context: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "#    #break\n",
    "\n",
    "#for i_q,question in enumerate(conversation_questions):\n",
    "#    if i_q >= 0:\n",
    "#        if i_q == 0:\n",
    "#            #answer_context_1,context_graph_1 = ask_graphqa(question ,previous_answer=False, previous_graph=False, verbose=True, timer=True, show_graph=True)\n",
    "#            answer_context = answer_context_1.copy()\n",
    "#            context_graph = context_graph_1.copy()\n",
    "#            continue\n",
    "#        elif context_graph:\n",
    "#            print(\"Context Question:\",question)\n",
    "#            answer_context,context_graph = ask_graphqa(question,answer_context=answer_context, context_graph=context_graph, verbose=True, timer=True, show_graph=True,\n",
    "#                                                       use_convex=True, turn=i_q+1)\n",
    "#        else:\n",
    "#            print(\"NO CONTEXT ERROR\")\n",
    "#            break\n",
    "#\n",
    "#    if answer_context: print(\"Answer:\",graphqa.convert_to_literal(graphqa.get_wd_label(answer_context[0][0])), \"(\"+str(answer_context[0][0])+\")\\n\")\n",
    "#    #break\n",
    "\n",
    "#answer_convex,context_graph = answer_conversation(conversation_questions,answer_convex=False,context_graph=False)\n",
    "\n",
    "#answer = ask_graphqa(\"Which actor voiced the Unicorn in The Last Unicorn?\", verbose=True)\n",
    "#answer = ask_graphqa(\"what's akbar tandjung's ethnicity\", verbose=True)\n",
    "#ask_graphqa(\"Which genre of album is harder.....faster?\")\n",
    "#ask_graphqa(\"Which label is somevelvetsidewalk signed to ttle of fort fisher \")\n",
    "#ask_graphqa(\"Who is the wife of Barack Obama\")\n",
    "#print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#answer_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HEADERS = ['conversation_id','turn',\"plus_convex\",\n",
    "           'question', 'answer', 'domain',\n",
    "           'qanswer','qanswer_time', 'qanswer_rr',\n",
    "           'platypus','platypus_time', 'platypus_rr',\n",
    "           'convex','convex_time', 'convex_rr',\n",
    "           'graphqa', \"graphqa_time\", \"graphqa_top2\", \"graphqa_top3\", \"graphqa_top4\", \"graphqa_top5\", \"graphqa_topall\",\"graphqa_rr\"]\n",
    "#df = pd.DataFrame(columns=HEADERS)\n",
    "#LOADING\n",
    "df = pd.read_pickle(\"/data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-37-ic1003-iq0-pcFalse.pickle.bz2\")\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for conversation in conversations:\n",
    "#    questions = [turn['question'] for turn in conversation['questions']]\n",
    "#    print(questions)\n",
    "#    golden_answers = [graphqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "#    print(golden_answers)\n",
    "#    \n",
    "#    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#banning_str = False #[[\"ř\",\"r\"]]\n",
    "#conversations_len = len(conversations)\n",
    "#\n",
    "#last_i = 0\n",
    "#for i, conversation in enumerate(conversations):\n",
    "#    if i >= last_i:\n",
    "#        print(\"\\n-->\",str(i)+\"/\"+str(conversations_len), \"New conversation\")\n",
    "#        questions = [turn['question'] for turn in conversation['questions']]\n",
    "#        questions_len = len(questions)\n",
    "#        for i_q, question in enumerate(questions):\n",
    "#            print(str(i_q)+\"/\"+str(questions_len),question)\n",
    "#            print(tmqa.get_nlp(question,autocorrect=True,banning_str=banning_str))\n",
    "#    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rr(results, answer):\n",
    "    if answer in results:\n",
    "        ans_position = results.index(answer)+1\n",
    "        if ans_position == 1:\n",
    "            return 1.0\n",
    "        return float(ans_position/len(results))\n",
    "    else: return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It is  2020-02-06 15:35:11.985796\n",
      "\t>>> Processing 1004/2240 -> 1/5 -> Convex=False: (Q145) From what country did the singer song writer Adele come from?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Asking Convex\n",
      "df_convex Q40831\n",
      "df_convex_rr 0.0\n",
      "\n",
      "Asking GraphQA\n",
      "User input: From what country did the singer song writer Adele come from?\n",
      "--> Auto correcting question in progress...\n",
      "WARNING:tensorflow:From /data/users/romain.claret/miniconda3/envs/qa/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "-> Auto corrected q_nlp: From what country did the singer song writer Adele come from \n",
      "-> q_themes: ([(Adele, ['Q23215', 'Q354370']), (country, ['Q6256', 'P17']), (Country, ['Q11070708', 'Q1754454'])], [the singer song writer, From country did the singer song writer Adele, The Singer Song Writer, singer song writer, the Singer Song Writer Adele])\n",
      "-> q_themes_enhanced: [('song writer', ['Q753110']), ('singer', ['Q177220']), ('song', ['Q7366']), ('writer', ['Q36180']), ('The Singer', ['Q18086751']), ('The Song', ['Q16254381']), ('The Writer', ['Q26772387']), ('Singer', ['Q1260201']), ('Song', ['Q10397756']), ('Writer', ['Q27067863'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171']), (country, [])]\n",
      "-> q_predicates \tRunning time is 26.72s\n",
      "--> Potential meaningful keywords for the sentence: ['Adele', 'country', 'Country', 'song writer', 'singer', 'song', 'writer', 'The Singer', 'The Song', 'The Writer', 'Singer', 'Song', 'Writer']\n",
      "q_focused_parts: [(country, ['Q83440', 'P3005', 'P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 89.3s\n",
      "-->  185 nodes and 180 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 175 nodes and 170 edges\n",
      "-> predicates_dict: {'P17': 2, 'P1750': 1, 'P1282': 1, 'P1963': 4, 'P156': 2, 'P1346': 1, 'P1686': 2, 'P585': 5, 'P166': 8, 'P805': 3, 'P495': 4, 'P27': 1, 'P1013': 2, 'P175': 1, 'P1441': 1, 'P407': 1, 'P2453': 1, 'P1411': 1, 'P364': 4, 'P571': 1, 'P291': 1, 'P577': 1, 'P1114': 1, 'P1545': 2, 'P1683': 1, 'P793': 1, 'P180': 1, 'P276': 1, 'P800': 1, 'P31': 9, 'P19': 1, 'P1476': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P734': 2, 'P569': 1, 'P155': 3, 'P1559': 1, 'P2452': 2, 'P580': 1, 'P2437': 1, 'P106': 2, 'P264': 1, 'P1080': 1, 'P195': 1, 'P217': 1, 'P373': 1, 'P136': 1, 'P735': 1}\n",
      "-> paths_keywords: (['country', 'come', 'adele', 'country music'], {'stated in': [stated in, ['P248']], 'They Work for You ID': [They Work for You ID, ['P2171']], 'valid in place': [valid in place, ['P3005']], 'country': [country, ['P17']]}, [what])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 416\n",
      "->Computing possible paths \tRunning time is 72.72s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 297\n",
      "->\tRunning time is 4.92s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q258', 33.05961005201645], ['Q211', 15.816351230867431], ['Q30', 6.307574646194619], ['Q145', 2.823633505283558], ['Q18086751', 2.578022665140577], ['Q668', 2.504428689656424], ['Q16254381', 1.3067899658697968], ['Q11424', 0.9190386788159257], ['Q5', 0.8870355060030328], ['Q1860', 0.8183242689381958], ['Q1260201', 0.5076568475250581], ['2013-12-19T00:00:00Z', 0.4032595912485059], ['2009-01-01T00:00:00Z', 0.2715573933933622], ['2012-01-01T00:00:00Z', 0.23214430674213832], ['2011-01-01T00:00:00Z', 0.21837619315944032], ['2016-01-01T00:00:00Z', 0.17280684485200395], ['Q503375', 0.03443736360078391], ['Adele@en', 0.02297897242966235]]\n",
      "->Computing hypothesises \tRunning time is 105.14s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 4\n",
      "->\tRunning time is 19.96s\n",
      "--> len(cleared_golden_paths): 2\n",
      "---> First path: ['Q258', 'P17', 'Q6256', 'P1282', 'Tag:place=country']\n",
      "->\tTotal Running time is 323.17s\n",
      "\n",
      "df_graphqa Q258\n",
      "df_graphqa_rr 0.2222222222222222\n",
      "\n",
      "PARTIAL_CORRECT 1004 - 1 -> graphqa in answers ['Q258', 'Q211', 'Q30', 'Q145', 'Q18086751', 'Q668', 'Q16254381', 'Q11424', 'Q5', 'Q1860', 'Q1260201', '2013-12-19T00:00:00Z', '2009-01-01T00:00:00Z', '2012-01-01T00:00:00Z', '2011-01-01T00:00:00Z', '2016-01-01T00:00:00Z', 'Q503375', 'Adele@en']\n",
      "   conversation_id turn plus_convex  \\\n",
      "37            1003    0       False   \n",
      "\n",
      "                                             question answer domain qanswer  \\\n",
      "37  From what country did the singer song writer A...   Q145  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  ...  convex_time  convex_rr graphqa  \\\n",
      "37          1.28         0.0    False  ...         2.09        0.0    Q258   \n",
      "\n",
      "    graphqa_time  graphqa_top2 graphqa_top3  graphqa_top4 graphqa_top5  \\\n",
      "37        323.39         False        False          True         True   \n",
      "\n",
      "   graphqa_topall graphqa_rr  \n",
      "37           True   0.222222  \n",
      "\n",
      "[1 rows x 23 columns]\n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-38-ic1003-iq0-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 15:40:40.516381\n",
      "\t>>> Processing 1004/2240 -> 2/5 -> Convex=False: (Adele Laurie Blue Adkins) what's her full name?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: what's her full name?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is her full name \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What is comedy full name\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (name, ['Q82799', 'Q503992']), (What, ['Q22073920']), (Name, ['Q13873817', 'Q11236330'])], [her full name, Her Full Name, full name, her full Name])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: full\n",
      "-> q_predicates: [(be, ['P31']), (full, ['P571']), (name, ['P735', 'P1448'])]\n",
      "-> q_predicates \tRunning time is 3.93s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (be, ['P31']), (full, ['P571']), (name, ['P735', 'P1448'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (name, ['Q82799', 'Q503992']), (What, ['Q22073920']), (Name, ['Q13873817', 'Q11236330'])], [her, Her, full])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'name', 'What', 'Name']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'name', 'What', 'Name', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [what, name, What, Name, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(name, ['Q503992']), (What, ['Q22073920', 'Q28036789']), (Name, ['Q13873817', 'Q11236330']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(name, ['Q503992']), (What, ['Q22073920', 'Q28036789']), (Name, ['Q13873817', 'Q11236330']), (comedy, ['Q40831']), (what, ['Q20656446', 'Q28036789'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 12.62s\n",
      "-->  13 nodes and 12 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 13 nodes and 12 edges\n",
      "-> predicates_dict: {'P136': 1, 'P1013': 1, 'P155': 2, 'P156': 2, 'P571': 1, 'P407': 2, 'P577': 2, 'P31': 6, 'P518': 1, 'P186': 1, 'P279': 1, 'P1545': 2, 'P4908': 1, 'P179': 1, 'P195': 1, 'P217': 1, 'P276': 1, 'P495': 1, 'P361': 1, 'P1476': 1, 'P1113': 1, 'P264': 1, 'P910': 2, 'P175': 1}\n",
      "-> paths_keywords: (['name', 'what', 'comedy'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 110\n",
      "->Computing possible paths \tRunning time is 56.59s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 24\n",
      "->\tRunning time is 3.27s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q17036433', 2.1424249639793542], ['Q1860', 1.0691466051156115]]\n",
      "->Computing hypothesises \tRunning time is 4.68s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 0\n",
      "->\tRunning time is 4.27s\n",
      "--> len(cleared_golden_paths): 0\n",
      "->\tTotal Running time is 89.23s\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_convex Q17036433\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: what's her full name?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What is her full name \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What is country full name\n",
      "-> q_themes: ([(what, ['Q20656446', 'Q28036789']), (name, ['Q82799', 'Q503992']), (What, ['Q22073920']), (Name, ['Q13873817', 'Q11236330'])], [her full name, Her Full Name, full name, her full Name])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: full\n",
      "-> q_predicates: [(be, ['P31']), (full, ['P571']), (name, ['P735', 'P1448'])]\n",
      "-> q_predicates \tRunning time is 4.23s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (be, ['P31']), (full, ['P571']), (name, ['P735', 'P1448']), (OSM tag or key, ['P1282'])]\n",
      "----> q_themes in context: ([(what, ['Q20656446', 'Q28036789']), (name, ['Q82799', 'Q503992']), (What, ['Q22073920']), (Name, ['Q13873817', 'Q11236330'])], [her, Her, full])\n",
      "--> Potential meaningful keywords for the sentence: ['what', 'name', 'What', 'Name']\n",
      "---> Meaningful keywords enhanced by previous context: ['what', 'name', 'What', 'Name', 'country', 'South Africa', 'Tag:place=country', 'country']\n",
      "meaningful_names_no_previous_answer [what, name, What, Name, country, South Africa, Tag place country, country]\n",
      "----> Meaningful keywords casted as theme ([(name, ['Q503992']), (What, ['Q22073920', 'Q28036789']), (Name, ['Q13873817', 'Q11236330']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(name, ['Q503992']), (What, ['Q22073920', 'Q28036789']), (Name, ['Q13873817', 'Q11236330']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256']), (what, ['Q20656446', 'Q28036789'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 52.96s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "-> predicates_dict: {'P17': 4, 'P1282': 2, 'P1963': 11, 'P1013': 1, 'P155': 2, 'P156': 2, 'P571': 1, 'P407': 1, 'P577': 3, 'P360': 3, 'P31': 212, 'P361': 2, 'P518': 1, 'P186': 1, 'P279': 1, 'P1545': 2, 'P4908': 1, 'P179': 1, 'P138': 1, 'P527': 1, 'P195': 1, 'P217': 1, 'P276': 1, 'P131': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P921': 1, 'P1424': 1, 'P2452': 2, 'P1476': 2, 'P264': 1, 'P625': 1, 'P910': 1, 'P478': 1}\n",
      "-> paths_keywords: (['name', 'what', 'country', 'south africa'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 154\n",
      "->Computing possible paths \tRunning time is 86.0s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 36\n",
      "->\tRunning time is 3.73s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Tag:place=country', 7.260011402752778]]\n",
      "->Computing hypothesises \tRunning time is 7.42s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 5\n",
      "->\tRunning time is 4.23s\n",
      "--> len(cleared_golden_paths): 4\n",
      "---> First path: ['Tag:place=country', 'P1282', 'Q6256', 'P17', 'Q258', 'P921', 'Q28754705']\n",
      "->\tTotal Running time is 161.48s\n",
      "\n",
      "df_graphqa Tag:place=country\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex               question  \\\n",
      "38            1003    1       False  what's her full name?   \n",
      "\n",
      "                      answer domain qanswer  qanswer_time  qanswer_rr  \\\n",
      "38  Adele Laurie Blue Adkins  music   False          0.38         0.0   \n",
      "\n",
      "   platypus  platypus_time  platypus_rr     convex  convex_time  convex_rr  \\\n",
      "38    False           0.45          0.0  Q17036433        89.46        0.0   \n",
      "\n",
      "              graphqa  graphqa_time graphqa_top2 graphqa_top3 graphqa_top4  \\\n",
      "38  Tag:place=country        161.72        False        False        False   \n",
      "\n",
      "   graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "38        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-39-ic1003-iq1-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1004/2240 -> 2/5 -> Convex=True: (Adele Laurie Blue Adkins) what's her full name?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q1860\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q7275\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex               question  \\\n",
      "39            1003    1        True  what's her full name?   \n",
      "\n",
      "                      answer domain qanswer  qanswer_time  qanswer_rr  \\\n",
      "39  Adele Laurie Blue Adkins  music   False          0.44         0.0   \n",
      "\n",
      "   platypus  platypus_time  platypus_rr convex  convex_time  convex_rr  \\\n",
      "39    False           0.41          0.0  Q1860         0.06        0.0   \n",
      "\n",
      "   graphqa  graphqa_time graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5  \\\n",
      "39   Q7275          1.63        False        False        False        False   \n",
      "\n",
      "   graphqa_topall  graphqa_rr  \n",
      "39          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-40-ic1003-iq1-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 15:44:55.114694\n",
      "\t>>> Processing 1004/2240 -> 3/5 -> Convex=False: (1988-05-05T00:00:00Z) her birth date?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: her birth date?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Her birth date \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What a Country birth date\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her birth date, Her Birth Date, her birth date, -PRON- birth date, her Birth Date])\n",
      "-> q_themes_enhanced: [('birth date', ['P569']), ('birth', ['Q14819852']), ('Birth', ['Q11191282'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: []\n",
      "-> q_predicates \tRunning time is 5.2s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her, her, -PRON-])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date', 'birth date', 'birth', 'Birth']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'birth date', 'birth', 'Birth', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [date, Date, birth date, birth, Birth, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (comedy, ['Q40831'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 20.49s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P136': 2, 'P935': 1, 'P155': 1, 'P156': 1, 'P577': 1, 'P1672': 1, 'P180': 1, 'P186': 1, 'P1013': 3, 'P509': 1, 'P642': 3, 'P279': 5, 'P805': 1, 'P31': 2, 'P407': 1, 'P443': 1, 'P3245': 1, 'P3250': 2, 'P971': 1, 'P910': 1, 'P1709': 1, 'P373': 1}\n",
      "-> paths_keywords: (['date', 'comedy'], {'genre': [genre, ['P136']], 'birth date': [date of birth, ['P569']], 'date of birth': [date of birth, ['P569']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 125.71s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.69s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.08s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: her birth date?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Her birth date \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What a Country birth date\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her birth date, Her Birth Date, her birth date, -PRON- birth date, her Birth Date])\n",
      "-> q_themes_enhanced: [('birth date', ['P569']), ('birth', ['Q14819852']), ('Birth', ['Q11191282'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(birth, ['P569', 'P19']), (date, ['P837'])]\n",
      "-> q_predicates \tRunning time is 4.86s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (birth, ['P569', 'P19']), (date, ['P837'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her, her, -PRON-])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date', 'birth date', 'birth', 'Birth']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'birth date', 'birth', 'Birth', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [date, Date, birth date, birth, Birth, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (comedy, ['Q40831'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 20.53s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P136': 2, 'P935': 1, 'P155': 1, 'P156': 1, 'P1672': 1, 'P180': 1, 'P186': 1, 'P1013': 3, 'P577': 1, 'P509': 1, 'P642': 3, 'P279': 5, 'P361': 3, 'P31': 3, 'P3245': 1, 'P3250': 2, 'P407': 1, 'P443': 1, 'P805': 1, 'P527': 1, 'P910': 1, 'P971': 1, 'P2888': 1, 'P1709': 1, 'P1542': 1, 'P1582': 1, 'P373': 1}\n",
      "-> paths_keywords: (['date', 'comedy'], {'genre': [genre, ['P136']], 'birth date': [date of birth, ['P569']], 'place of birth': [place of birth, ['P19']], 'day in year for periodic occurrence': [day in date for periodic occurrence, ['P837']], 'date of birth': [date of birth, ['P569']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 147.04s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.64s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.11s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 179.97s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: her birth date?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Her birth date \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: country birth date\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her birth date, Her Birth Date, her birth date, -PRON- birth date, her Birth Date])\n",
      "-> q_themes_enhanced: [('birth date', ['P569']), ('birth', ['Q14819852']), ('Birth', ['Q11191282'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: []\n",
      "-> q_predicates \tRunning time is 5.52s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (OSM tag or key, ['P1282']), (main subject, ['P921'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [Her, her, -PRON-])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date', 'birth date', 'birth', 'Birth']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'birth date', 'birth', 'Birth', 'country', 'South Africa', 'Tag:place=country', 'South Africa', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [date, Date, birth date, birth, Birth, country, South Africa, Tag place country, South Africa, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 51.91s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "-> predicates_dict: {'P17': 4, 'P1282': 2, 'P138': 2, 'P921': 6, 'P31': 86, 'P1963': 5, 'P935': 1, 'P155': 1, 'P156': 1, 'P577': 2, 'P910': 1, 'P1672': 1, 'P180': 1, 'P186': 1, 'P1013': 3, 'P509': 1, 'P642': 3, 'P279': 7, 'P805': 1, 'P407': 1, 'P443': 1, 'P361': 2, 'P3245': 1, 'P3250': 2, 'P131': 1, 'P527': 6, 'P1424': 1, 'P5008': 1, 'P2452': 2, 'P813': 1, 'P973': 1, 'P1709': 2, 'P625': 1, 'P136': 2, 'P1476': 1}\n",
      "-> paths_keywords: (['date', 'country', 'south africa'], {'country': [country, ['P17']], 'OSM tag or key': [OSM tag or key, ['P1282']], 'main subject': [main subject, ['P921']], 'birth date': [date of birth, ['P569']], 'date of birth': [date of birth, ['P569']]}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 122\n",
      "->Computing possible paths \tRunning time is 60.31s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 36\n",
      "->\tRunning time is 3.5s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Tag:place=country', 1.1266349473887567]]\n",
      "->Computing hypothesises \tRunning time is 10.43s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 6\n",
      "->\tRunning time is 4.06s\n",
      "--> len(cleared_golden_paths): 4\n",
      "---> First path: ['Tag:place=country', 'P1282', 'Q6256', 'P17', 'Q258', 'P138', 'Q55155433']\n",
      "->\tTotal Running time is 140.6s\n",
      "\n",
      "df_graphqa Tag:place=country\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex         question                answer  \\\n",
      "40            1003    2       False  her birth date?  1988-05-05T00:00:00Z   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "40  music   False          0.51         0.0    False           0.35   \n",
      "\n",
      "    platypus_rr convex  convex_time  convex_rr            graphqa  \\\n",
      "40          0.0  False       335.73        0.0  Tag:place=country   \n",
      "\n",
      "    graphqa_time graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5  \\\n",
      "40         141.0        False        False        False        False   \n",
      "\n",
      "   graphqa_topall  graphqa_rr  \n",
      "40          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-41-ic1003-iq2-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1004/2240 -> 3/5 -> Convex=True: (1988-05-05T00:00:00Z) her birth date?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q30\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q1411042\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex         question                answer  \\\n",
      "41            1003    2        True  her birth date?  1988-05-05T00:00:00Z   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "41  music   False           0.5         0.0    False           0.35   \n",
      "\n",
      "    platypus_rr convex  convex_time  convex_rr   graphqa  graphqa_time  \\\n",
      "41          0.0    Q30         0.03        0.0  Q1411042          7.98   \n",
      "\n",
      "   graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "41        False        False        False        False          False   \n",
      "\n",
      "    graphqa_rr  \n",
      "41         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-42-ic1003-iq2-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 15:53:01.618812\n",
      "\t>>> Processing 1004/2240 -> 4/5 -> Convex=False: (Q503375) what town was she born in?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: what town was she born in?\n",
      "--> Auto correcting question in progress...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> Auto corrected q_nlp: What town was she born in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What town was What a Country born in\n",
      "-> q_themes: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [What town, what Town])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(be, ['P31']), (born, ['P569', 'P19'])]\n",
      "-> q_predicates \tRunning time is 5.28s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (be, ['P31']), (born, ['P569', 'P19'])]\n",
      "----> q_themes in context: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [what])\n",
      "--> Potential meaningful keywords for the sentence: ['Town', 'town']\n",
      "---> Meaningful keywords enhanced by previous context: ['Town', 'town', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [Town, town, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (comedy, ['Q40831'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 12.36s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P136': 0, 'P360': 1, 'P31': 47, 'P2354': 1, 'P910': 1, 'P971': 1, 'P279': 2}\n",
      "-> paths_keywords: (['town', 'comedy', 'born'], {}, [What, What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 122.17s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.01s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.1s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: what town was she born in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What town was she born in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What town What a Country born in\n",
      "-> q_themes: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [What town, what Town])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: town\n",
      "-> q_predicates: [(be, ['P31']), (born, ['P569', 'P19']), (town, ['P131'])]\n",
      "-> q_predicates \tRunning time is 5.1s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (be, ['P31']), (born, ['P569', 'P19']), (town, ['P131'])]\n",
      "----> q_themes in context: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [what])\n",
      "--> Potential meaningful keywords for the sentence: ['Town', 'town']\n",
      "---> Meaningful keywords enhanced by previous context: ['Town', 'town', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [Town, town, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (comedy, ['Q40831'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 12.15s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P136': 0, 'P279': 2, 'P360': 1, 'P31': 47, 'P2354': 1, 'P910': 1, 'P971': 1}\n",
      "-> paths_keywords: (['town', 'comedy', 'born'], {}, [What, What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 123.92s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.99s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.1s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 150.15s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: what town was she born in?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What town was she born in \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What town was country born in\n",
      "-> q_themes: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [What town, what Town])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(be, ['P31']), (born, ['P569', 'P19'])]\n",
      "-> q_predicates \tRunning time is 6.23s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (be, ['P31']), (born, ['P569', 'P19']), (OSM tag or key, ['P1282']), (named after, ['P138'])]\n",
      "----> q_themes in context: ([(Town, ['Q3957', 'Q10740142']), (town, ['Q11881845'])], [What, what])\n",
      "--> Potential meaningful keywords for the sentence: ['Town', 'town']\n",
      "---> Meaningful keywords enhanced by previous context: ['Town', 'town', 'country', 'South Africa', 'Tag:place=country', 'South Africa', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [Town, town, country, South Africa, Tag place country, South Africa, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(Town, ['Q10740142']), (town, ['Q3957', 'Q11881845']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 54.13s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "-> predicates_dict: {'P17': 4, 'P1282': 2, 'P138': 2, 'P921': 2, 'P31': 334, 'P1963': 7, 'P577': 1, 'P361': 4, 'P360': 1, 'P527': 1, 'P2354': 1, 'P910': 1, 'P131': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P1424': 1, 'P2452': 2, 'P625': 1, 'P279': 1, 'P1476': 1}\n",
      "-> paths_keywords: (['town', 'country', 'south africa', 'born'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 122\n",
      "->Computing possible paths \tRunning time is 69.85s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 36\n",
      "->\tRunning time is 3.6s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Tag:place=country', 7.083234077184014]]\n",
      "->Computing hypothesises \tRunning time is 8.94s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 6\n",
      "->\tRunning time is 4.37s\n",
      "--> len(cleared_golden_paths): 4\n",
      "---> First path: ['Tag:place=country', 'P1282', 'Q6256', 'P17', 'Q258', 'P138', 'Q55155433']\n",
      "->\tTotal Running time is 150.8s\n",
      "\n",
      "df_graphqa Tag:place=country\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                    question   answer  \\\n",
      "42            1003    3       False  what town was she born in?  Q503375   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "42  music   False          0.79         0.0    False           0.88   \n",
      "\n",
      "    platypus_rr convex  convex_time  convex_rr            graphqa  \\\n",
      "42          0.0  False       294.84        0.0  Tag:place=country   \n",
      "\n",
      "    graphqa_time graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5  \\\n",
      "42        151.03        False        False        False        False   \n",
      "\n",
      "   graphqa_topall  graphqa_rr  \n",
      "42          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-43-ic1003-iq3-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1004/2240 -> 4/5 -> Convex=True: (Q503375) what town was she born in?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q17036433\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q3957\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                    question   answer  \\\n",
      "43            1003    3        True  what town was she born in?  Q503375   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "43  music   False          0.79         0.0    False           0.87   \n",
      "\n",
      "    platypus_rr     convex  convex_time  convex_rr graphqa  graphqa_time  \\\n",
      "43          0.0  Q17036433         0.03        0.0   Q3957          1.21   \n",
      "\n",
      "   graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "43        False        False        False        False          False   \n",
      "\n",
      "    graphqa_rr  \n",
      "43         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-44-ic1003-iq3-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:00:32.115400\n",
      "\t>>> Processing 1004/2240 -> 5/5 -> Convex=False: (yes) does she have a husband?                                  \n",
      "Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: does she have a husband?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Does she have a husband \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Does What a Country have a husband\n",
      "> Binary question related question detected\n",
      "-> q_themes: ([(husband, ['Q212878', 'Q415546']), (Husband, ['Q16870933', 'Q11863857'])], [a husband, A Husband, a Husband])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: do\n",
      "-> q_predicates: [(Does, []), (have, ['P527'])]\n",
      "-> q_predicates \tRunning time is 4.11s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (Does, []), (have, ['P527'])]\n",
      "----> q_themes in context: ([(husband, ['Q212878', 'Q415546']), (Husband, ['Q16870933', 'Q11863857'])], [A])\n",
      "--> Potential meaningful keywords for the sentence: ['husband', 'Husband']\n",
      "---> Meaningful keywords enhanced by previous context: ['husband', 'Husband', 'What a Country!', 'comedy']\n",
      "meaningful_names_no_previous_answer [husband, Husband, What a Country, comedy]\n",
      "----> Meaningful keywords casted as theme ([(husband, ['Q212878']), (Husband, ['Q16870933', 'Q11863857']), (comedy, ['Q40831'])], [])\n",
      "q_focused_parts: [(husband, ['Q212878']), (Husband, ['Q16870933', 'Q11863857']), (comedy, ['Q40831'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 10.83s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "df_convex no\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: does she have a husband?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Does she have a husband \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Does country have a husband\n",
      "> Binary question related question detected\n",
      "-> q_themes: ([(husband, ['Q212878', 'Q415546']), (Husband, ['Q16870933', 'Q11863857'])], [a husband, A Husband, a Husband])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: do\n",
      "-> q_predicates: [(Does, []), (have, ['P527'])]\n",
      "-> q_predicates \tRunning time is 4.53s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (Does, []), (have, ['P527']), (OSM tag or key, ['P1282']), (named after, ['P138'])]\n",
      "----> q_themes in context: ([(husband, ['Q212878', 'Q415546']), (Husband, ['Q16870933', 'Q11863857'])], [a, A])\n",
      "--> Potential meaningful keywords for the sentence: ['husband', 'Husband']\n",
      "---> Meaningful keywords enhanced by previous context: ['husband', 'Husband', 'country', 'South Africa', 'Tag:place=country', 'South Africa', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [husband, Husband, country, South Africa, Tag place country, South Africa, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(husband, ['Q212878']), (Husband, ['Q16870933', 'Q11863857']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(husband, ['Q212878']), (Husband, ['Q16870933', 'Q11863857']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 45.4s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "df_graphqa no\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                  question answer domain  \\\n",
      "44            1003    4       False  does she have a husband?    yes  music   \n",
      "\n",
      "   qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "44   False          0.25         0.0    False           1.47          0.0   \n",
      "\n",
      "   convex  convex_time  convex_rr graphqa  graphqa_time graphqa_top2  \\\n",
      "44     no        15.59        0.0      no         51.63        False   \n",
      "\n",
      "   graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "44        False        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-45-ic1003-iq4-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1004/2240 -> 5/5 -> Convex=True: (yes) does she have a husband?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex yes\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1004 - 5 -> Convex yes\n",
      "\n",
      "Asking GraphQA\n",
      "User input: does she have a husband?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Does she have a husband \n",
      "> Binary question related question detected\n",
      "-> q_themes: ([(husband, ['Q212878', 'Q415546']), (Husband, ['Q16870933', 'Q11863857'])], [a husband, A Husband, a Husband])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: do\n",
      "-> q_predicates: [(Does, []), (have, ['P527'])]\n",
      "-> q_predicates \tRunning time is 4.57s\n",
      "--> Potential meaningful keywords for the sentence: ['husband', 'Husband']\n",
      "q_focused_parts: []\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 12.5s\n",
      "-->  28 nodes and 26 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 28 nodes and 26 edges\n",
      "---> Rebuilding the graph with k_deep 6 ... Previously: 28 nodes or 26 edges was below the limit of 100\n",
      "->New graph \tRunning time is 12.79s\n",
      "-->  38 nodes and 36 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 38 nodes and 36 edges\n",
      "---> Rebuilding the graph with k_deep 8 ... Previously: 38 nodes or 36 edges was below the limit of 100\n",
      "->New graph \tRunning time is 12.53s\n",
      "-->  44 nodes and 42 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 44 nodes and 42 edges\n",
      "---> Rebuilding the graph with k_deep 10 ... Previously: 44 nodes or 42 edges was below the limit of 100\n",
      "->New graph \tRunning time is 12.91s\n",
      "-->  48 nodes and 46 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 48 nodes and 46 edges\n",
      "---> Rebuilding the graph with k_deep 12 ... Previously: 48 nodes or 46 edges was below the limit of 100\n",
      "->New graph \tRunning time is 12.59s\n",
      "-->  50 nodes and 48 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 50 nodes and 48 edges\n",
      "---> Rebuilding the graph with k_deep 14 ... Previously: 50 nodes or 48 edges was below the limit of 100\n",
      "->New graph \tRunning time is 12.33s\n",
      "-->  50 nodes and 48 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 50 nodes and 48 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "node_name_id ['Q212878', 'Q415546']\n",
      "df_graphqa yes\n",
      "df_graphqa_rr 1.0\n",
      "\n",
      "CORRECT 1004 - 5 -> graphqa yes\n",
      "\n",
      "PARTIAL_CORRECT 1004 - 5 -> graphqa in answers ['yes']\n",
      "   conversation_id turn plus_convex                  question answer domain  \\\n",
      "45            1003    4        True  does she have a husband?    yes  music   \n",
      "\n",
      "   qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "45   False          0.31         0.0    False           1.49          0.0   \n",
      "\n",
      "   convex  convex_time  convex_rr graphqa  graphqa_time graphqa_top2  \\\n",
      "45    yes         0.02        1.0     yes         80.78         True   \n",
      "\n",
      "   graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "45         True         True         True           True         1.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-46-ic1003-iq4-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:03:03.714436\n",
      "\t>>> Processing 1005/2240 -> 1/5 -> Convex=False: (Q145) What country did the band Black Sabbath originally come from?                                  \n",
      "Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Asking Convex\n",
      "df_convex black-sabbath\n",
      "df_convex_rr 0.0\n",
      "\n",
      "Asking GraphQA\n",
      "User input: What country did the band Black Sabbath originally come from?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What country did the band Black Sabbath originally come from \n",
      "-> q_themes: ([(Black Sabbath, ['Q47670', 'Q328482']), (country, ['Q6256', 'P17']), (the band, ['Q600344', 'Q926142']), (Country, ['Q11070708', 'Q1754454'])], [country did the band Black, The Band Black Sabbath, the band black sabbath, the Band Black Sabbath])\n",
      "-> q_themes_enhanced: [('Black Band', ['Q686400']), ('Band', ['Q1096385']), ('black', ['Q17244465']), ('sabbath', ['Q102477']), ('The Band', ['Q1681495']), ('The Black', ['Q12053167'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171']), (country, [])]\n",
      "-> q_predicates \tRunning time is 12.69s\n",
      "--> Potential meaningful keywords for the sentence: ['Black Sabbath', 'country', 'the band', 'Country', 'Black Band', 'Band', 'black', 'sabbath', 'The Band', 'The Black']\n",
      "q_focused_parts: [(country, ['Q83440', 'P3005', 'P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 74.41s\n",
      "-->  199 nodes and 198 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 178 nodes and 176 edges\n",
      "-> predicates_dict: {'P17': 2, 'P1282': 1, 'P1963': 4, 'P1686': 2, 'P585': 5, 'P166': 3, 'P136': 7, 'P767': 2, 'P495': 4, 'P407': 2, 'P793': 1, 'P1013': 1, 'P459': 2, 'P1082': 1, 'P2453': 1, 'P805': 2, 'P1411': 1, 'P1343': 2, 'P131': 1, 'P364': 1, 'P421': 2, 'P571': 3, 'P740': 3, 'P279': 1, 'P31': 12, 'P825': 1, 'P291': 1, 'P577': 2, 'P527': 1, 'P155': 1, 'P813': 1, 'P3744': 1, 'P1709': 1, 'P910': 1, 'P973': 1, 'P2002': 1, 'P734': 1, 'P264': 2, 'P569': 1, 'P1376': 1, 'P19': 3, 'P138': 3, 'P2452': 2, 'P1340': 1, 'P361': 1, 'P304': 1, 'P478': 1, 'P106': 1, 'P921': 2, 'P462': 1}\n",
      "-> paths_keywords: (['country', 'come', 'black sabbath', 'the band', 'country music'], {'stated in': [stated in, ['P248']], 'They Work for You ID': [They Work for You ID, ['P2171']], 'valid in place': [valid in place, ['P3005']], 'country': [country, ['P17']]}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 283\n",
      "->Computing possible paths \tRunning time is 56.32s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 429\n",
      "->\tRunning time is 3.8s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q258', 31.75016337609176], ['Q218', 6.909428072656077], ['Q30', 4.792864868819841], ['Q1096385', 3.706297033529309], ['Q16', 3.3217724997291067], ['Q12053167', 2.621691439977019], ['Q1063385', 1.2786003360520353], ['Q1471926', 1.1562536813538642], ['Q1753252', 1.0399308036501473], ['Q1860', 0.9230433427625958], ['Q482994', 0.7113731862507666], ['1997-01-01T00:00:00Z', 0.5796356353134321], ['1998-01-01T00:00:00Z', 0.5038659356605545], ['2011-10-31T00:00:00Z', 0.5031659944025414], ['1989-01-01T00:00:00Z', 0.4842640541847392]]\n",
      "->Computing hypothesises \tRunning time is 141.41s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 2\n",
      "->\tRunning time is 28.38s\n",
      "--> len(cleared_golden_paths): 1\n",
      "---> First path: ['Q258', 'P17', 'Q6256', 'P1963']\n",
      "->\tTotal Running time is 321.73s\n",
      "\n",
      "df_graphqa Q258\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex  \\\n",
      "46            1004    0       False   \n",
      "\n",
      "                                             question answer domain qanswer  \\\n",
      "46  What country did the band Black Sabbath origin...   Q145  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "46          1.37         0.0    False            2.9          0.0   \n",
      "\n",
      "           convex  convex_time  convex_rr graphqa  graphqa_time graphqa_top2  \\\n",
      "46  black-sabbath         1.55        0.0    Q258        321.98        False   \n",
      "\n",
      "   graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "46        False        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-47-ic1004-iq0-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:08:31.535761\n",
      "\t>>> Processing 1005/2240 -> 2/5 -> Convex=False: (Q56816954) What genre is the band?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: What genre is the band?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What genre is the band \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What genre is the band\n",
      "-> q_themes: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: band\n",
      "-> q_predicates: [(be, ['P31']), (genre, ['P136']), (band, [])]\n",
      "-> q_predicates \tRunning time is 6.15s\n",
      "--> Predicates enhanced by previous context: [(Rock Hall of Fame ID, ['P3162']), (be, ['P31']), (genre, ['P136']), (band, [])]\n",
      "----> q_themes in context: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band']\n",
      "---> Meaningful keywords enhanced by previous context: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band', 'Black Sabbath', 'black-sabbath']\n",
      "meaningful_names_no_previous_answer [genre, the band, band, Band, Genre, The Band, Black Sabbath, black sabbath]\n",
      "----> Meaningful keywords casted as theme ([(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859'])], [])\n",
      "q_focused_parts: [(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859']), (the band, ['Q600344', 'Q926142'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 36.13s\n",
      "-->  29 nodes and 28 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 29 nodes and 28 edges\n",
      "-> predicates_dict: {'P3162': 0, 'P1383': 1, 'P36': 1, 'P131': 2, 'P1376': 1, 'P373': 2, 'P136': 9, 'P1013': 3, 'P1433': 1, 'P31': 9, 'P166': 1, 'P1810': 2, 'P4900': 1, 'P1480': 1, 'P459': 4, 'P585': 3, 'P1082': 2, 'P793': 1, 'P407': 1, 'P571': 1, 'P421': 4, 'P580': 1, 'P582': 1, 'P6': 1, 'P577': 1, 'P19': 2, 'P364': 1, 'P921': 1, 'P1680': 1, 'P2096': 1, 'P734': 1, 'P3744': 1, 'P2002': 1, 'P123': 1, 'P264': 2, 'P161': 3, 'P1705': 5, 'P1476': 1, 'P282': 1, 'P17': 2}\n",
      "-> paths_keywords: (['genre', 'band', 'the band', 'black sabbath'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 110\n",
      "->Computing possible paths \tRunning time is 46.79s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 62\n",
      "->\tRunning time is 4.31s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q38848', 3.056158876838653], ['Q1133657', 2.9535337884443003], ['Q186170', 2.8627471130005064], ['Q617240', 2.709006462166805]]\n",
      "->Computing hypothesises \tRunning time is 18.12s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 6\n",
      "->\tRunning time is 5.68s\n",
      "--> len(cleared_golden_paths): 3\n",
      "---> First path: ['Q38848', 'P136', 'Q328482']\n",
      "->\tTotal Running time is 120.93s\n",
      "\n",
      "df_convex Q38848\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: What genre is the band?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What genre is the band \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: What genre is the band\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> q_themes: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: band\n",
      "-> q_predicates: [(be, ['P31']), (genre, ['P136']), (band, [])]\n",
      "-> q_predicates \tRunning time is 5.6s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (be, ['P31']), (genre, ['P136']), (band, []), (properties for this type, ['P1963'])]\n",
      "----> q_themes in context: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band']\n",
      "---> Meaningful keywords enhanced by previous context: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band', 'country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [genre, the band, band, Band, Genre, The Band, country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256']), (the band, ['Q600344', 'Q926142'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 50.2s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "-> predicates_dict: {'P17': 5, 'P1963': 63, 'P1383': 1, 'P36': 1, 'P131': 3, 'P1376': 1, 'P373': 2, 'P136': 3, 'P1282': 1, 'P1013': 3, 'P1433': 1, 'P1810': 2, 'P4900': 1, 'P31': 212, 'P1480': 1, 'P459': 4, 'P585': 2, 'P1082': 2, 'P527': 1, 'P361': 1, 'P1680': 1, 'P421': 4, 'P580': 2, 'P582': 1, 'P6': 2, 'P19': 1, 'P921': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P2452': 2, 'P1240': 1, 'P138': 1, 'P2096': 1, 'P123': 1, 'P577': 1, 'P1705': 5, 'P1476': 2, 'P282': 1}\n",
      "-> paths_keywords: (['genre', 'band', 'the band', 'country', 'south africa'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 122\n",
      "->Computing possible paths \tRunning time is 80.16s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 36\n",
      "->\tRunning time is 3.84s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 4.99s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: What genre is the band?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: What genre is the band \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: What country genre properties for this type the band\n",
      "-> q_themes: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: band\n",
      "-> q_predicates: [(be, ['P31']), (genre, ['P136']), (band, [])]\n",
      "-> q_predicates \tRunning time is 5.38s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (be, ['P31']), (genre, ['P136']), (band, []), (properties for this type, ['P1963'])]\n",
      "----> q_themes in context: ([(genre, ['Q483394', 'P136']), (the band, ['Q600344', 'Q926142']), (band, ['Q215380', 'Q29289832']), (Band, ['Q1096385', 'Q12080772']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band']\n",
      "---> Meaningful keywords enhanced by previous context: ['genre', 'the band', 'band', 'Band', 'Genre', 'The Band', 'country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [genre, the band, band, Band, Genre, The Band, country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(genre, ['P136']), (band, ['Q215380', 'Q29289832']), (Band, ['Q12080772', 'Q1096385']), (Genre, ['Q37446719', 'Q15758743']), (The Band, ['Q1681495']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256']), (the band, ['Q600344', 'Q926142'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 51.61s\n",
      "-->  10 nodes and 10 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 10 nodes and 10 edges\n",
      "-> predicates_dict: {'P17': 5, 'P1963': 63, 'P1383': 1, 'P36': 1, 'P131': 3, 'P1376': 1, 'P373': 2, 'P1282': 1, 'P136': 3, 'P1013': 3, 'P1433': 1, 'P1810': 2, 'P4900': 1, 'P31': 212, 'P1480': 1, 'P459': 4, 'P585': 2, 'P1082': 2, 'P527': 1, 'P361': 1, 'P1680': 1, 'P421': 4, 'P580': 2, 'P582': 1, 'P6': 2, 'P19': 1, 'P921': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P2452': 2, 'P1240': 1, 'P138': 1, 'P2096': 1, 'P123': 1, 'P577': 1, 'P1705': 5, 'P1476': 2, 'P282': 1}\n",
      "-> paths_keywords: (['genre', 'band', 'the band', 'country', 'south africa', 'properties'], {}, [What])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 126\n",
      "->Computing possible paths \tRunning time is 48.31s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 36\n",
      "->\tRunning time is 4.01s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 6.24s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 120.48s\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex                 question     answer  \\\n",
      "47            1004    1       False  What genre is the band?  Q56816954   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "47  music   False          0.38         0.0    False           0.18   \n",
      "\n",
      "    platypus_rr  convex  convex_time  convex_rr graphqa  graphqa_time  \\\n",
      "47          0.0  Q38848        121.2        0.0   False         266.1   \n",
      "\n",
      "   graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "47        False        False        False        False          False   \n",
      "\n",
      "    graphqa_rr  \n",
      "47         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-48-ic1004-iq1-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1005/2240 -> 2/5 -> Convex=True: (Q56816954) What genre is the band?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q38848\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q19848\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                 question     answer  \\\n",
      "48            1004    1        True  What genre is the band?  Q56816954   \n",
      "\n",
      "   domain qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "48  music   False           0.4         0.0    False           0.19   \n",
      "\n",
      "    platypus_rr  convex  convex_time  convex_rr graphqa  graphqa_time  \\\n",
      "48          0.0  Q38848         0.24        0.0  Q19848          0.77   \n",
      "\n",
      "   graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "48        False        False        False        False          False   \n",
      "\n",
      "    graphqa_rr  \n",
      "48         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-49-ic1004-iq1-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:15:01.034890\n",
      "\t>>> Processing 1005/2240 -> 3/5 -> Convex=False: (Yes) Are they in the Rock and Roll Hall of fame?                                  \n",
      "Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: Are they in the Rock and Roll Hall of fame?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Are they in the Rock and Roll Hall of fame \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Are Heavy metal music and Black Sabbath in the Rock and Roll Hall of fame\n",
      "> Binary question related question detected\n",
      "-> q_themes: ([(fame, ['Q226485', 'Q1351911']), (Fame, ['Q13632838']), (Rock and Roll Hall, ['Q179191']), (Rock and Roll, ['Q7749', 'Q2331674'])], [the Rock and Roll Hall, Are in the Rock, Roll Hall, The Rock And Roll Hall, the rock and roll hall])\n",
      "-> q_themes_enhanced: [('rock', ['Q1404150']), ('In Rock', ['Q1208715']), ('The Rock', ['Q1193362']), ('roll', ['Q23004']), ('hall', ['Q240854']), ('Rock and Roll Hall of Fame', ['P3162']), ('rock and roll', ['P3162'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: fame\n",
      "-> q_predicates: [(be, ['P31']), (fame, [])]\n",
      "-> q_predicates \tRunning time is 9.92s\n",
      "--> Predicates enhanced by previous context: [(genre, ['P136']), (be, ['P31']), (fame, [])]\n",
      "----> q_themes in context: ([(fame, ['Q226485', 'Q1351911']), (Fame, ['Q13632838']), (Rock and Roll Hall, ['Q179191']), (Rock and Roll, ['Q7749', 'Q2331674'])], [the, Are, Roll, The])\n",
      "--> Potential meaningful keywords for the sentence: ['fame', 'Fame', 'Rock and Roll Hall', 'Rock and Roll', 'rock', 'In Rock', 'The Rock', 'roll', 'hall', 'Rock and Roll Hall of Fame', 'rock and roll']\n",
      "---> Meaningful keywords enhanced by previous context: ['fame', 'Fame', 'Rock and Roll Hall', 'Rock and Roll', 'rock', 'In Rock', 'The Rock', 'roll', 'hall', 'Rock and Roll Hall of Fame', 'rock and roll', 'Black Sabbath', 'Black Sabbath', 'Heavy metal music', 'black-sabbath']\n",
      "meaningful_names_no_previous_answer [fame, Fame, Rock and Roll Hall, Rock and Roll, rock, In Rock, The Rock, roll, hall, Rock and Roll Hall of Fame, rock and roll, Black Sabbath, Black Sabbath, Heavy metal music, black sabbath]\n",
      "----> Meaningful keywords casted as theme ([(Fame, ['Q1351911', 'Q13632838']), (Rock and Roll, ['Q2331674']), (Rock and Roll Hall of Fame, ['Q179191']), (rock and roll, ['Q7749']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859'])], [])\n",
      "q_focused_parts: [(Fame, ['Q1351911', 'Q13632838']), (Rock and Roll, ['Q2331674']), (Rock and Roll Hall of Fame, ['Q179191']), (rock and roll, ['Q7749']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859']), (Black Sabbath, ['Q328482', 'Q2270855', 'Q394859']), (fame, ['Q226485', 'Q1351911']), (Rock and Roll Hall, ['Q179191'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 90.81s\n",
      "-->  5 nodes and 4 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 5 nodes and 4 edges\n",
      "df_convex no\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Are they in the Rock and Roll Hall of fame?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Are they in the Rock and Roll Hall of fame \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Are country in the Rock and Roll Hall of fame\n",
      "> Binary question related question detected\n",
      "-> q_themes: ([(fame, ['Q226485', 'Q1351911']), (Fame, ['Q13632838']), (Rock and Roll Hall, ['Q179191']), (Rock and Roll, ['Q7749', 'Q2331674'])], [the Rock and Roll Hall, Are in the Rock, Roll Hall, The Rock And Roll Hall, the rock and roll hall])\n",
      "-> q_themes_enhanced: [('rock', ['Q1404150']), ('In Rock', ['Q1208715']), ('The Rock', ['Q1193362']), ('roll', ['Q23004']), ('hall', ['Q240854']), ('Rock and Roll Hall of Fame', ['P3162']), ('rock and roll', ['P3162'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: fame\n",
      "-> q_predicates: [(be, ['P31']), (fame, [])]\n",
      "-> q_predicates \tRunning time is 8.7s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (be, ['P31']), (fame, [])]\n",
      "----> q_themes in context: ([(fame, ['Q226485', 'Q1351911']), (Fame, ['Q13632838']), (Rock and Roll Hall, ['Q179191']), (Rock and Roll, ['Q7749', 'Q2331674'])], [the, Are, Roll, The])\n",
      "--> Potential meaningful keywords for the sentence: ['fame', 'Fame', 'Rock and Roll Hall', 'Rock and Roll', 'rock', 'In Rock', 'The Rock', 'roll', 'hall', 'Rock and Roll Hall of Fame', 'rock and roll']\n",
      "---> Meaningful keywords enhanced by previous context: ['fame', 'Fame', 'Rock and Roll Hall', 'Rock and Roll', 'rock', 'In Rock', 'The Rock', 'roll', 'hall', 'Rock and Roll Hall of Fame', 'rock and roll', 'country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [fame, Fame, Rock and Roll Hall, Rock and Roll, rock, In Rock, The Rock, roll, hall, Rock and Roll Hall of Fame, rock and roll, country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(Fame, ['Q1351911', 'Q13632838']), (Rock and Roll, ['Q2331674']), (Rock and Roll Hall of Fame, ['Q179191']), (rock and roll, ['Q7749']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(Fame, ['Q1351911', 'Q13632838']), (Rock and Roll, ['Q2331674']), (Rock and Roll Hall of Fame, ['Q179191']), (rock and roll, ['Q7749']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256']), (fame, ['Q226485', 'Q1351911']), (Rock and Roll Hall, ['Q179191'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 120.82s\n",
      "-->  15 nodes and 16 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 15 nodes and 16 edges\n",
      "node_name_id ['Q1351911', 'Q13632838']\n",
      "df_graphqa yes\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex  \\\n",
      "49            1004    2       False   \n",
      "\n",
      "                                       question answer domain qanswer  \\\n",
      "49  Are they in the Rock and Roll Hall of fame?    Yes  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr convex  \\\n",
      "49          0.95         0.0    False            2.1          0.0     no   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "49       102.89        0.0     yes        130.87        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "49        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-50-ic1004-iq2-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1005/2240 -> 3/5 -> Convex=True: (Yes) Are they in the Rock and Roll Hall of fame?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex yes\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa yes\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex  \\\n",
      "50            1004    2        True   \n",
      "\n",
      "                                       question answer domain qanswer  \\\n",
      "50  Are they in the Rock and Roll Hall of fame?    Yes  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr convex  \\\n",
      "50          1.09         0.0    False           1.83          0.0    yes   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "50         1.55        0.0     yes          3.08        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "50        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-51-ic1004-iq2-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:19:05.438715\n",
      "\t>>> Processing 1005/2240 -> 4/5 -> Convex=False: (Q133151) Lead singer?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Asking Convex\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_convex Q2529757\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Lead singer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Lead singer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Lead singer\n",
      "-> q_themes: ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: []\n",
      "-> q_predicates \tRunning time is 2.93s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17'])]\n",
      "----> q_themes in context: ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['singer', 'Singer']\n",
      "---> Meaningful keywords enhanced by previous context: ['singer', 'Singer', 'country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [singer, Singer, country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 45.83s\n",
      "-->  9 nodes and 8 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 9 nodes and 8 edges\n",
      "-> predicates_dict: {'P17': 3, 'P1963': 2, 'P1282': 1, 'P361': 1, 'P1013': 1, 'P527': 1, 'P407': 1, 'P138': 1, 'P921': 1, 'P31': 3, 'P813': 1, 'P973': 1, 'P1709': 1, 'P577': 1, 'P136': 2, 'P2452': 2}\n",
      "-> paths_keywords: (['singer', 'country', 'south africa'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 132.95s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.38s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.04s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: Lead singer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Lead singer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Lead singer\n",
      "-> q_themes: ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: lead\n",
      "behold: get_most_similar started with: singer\n",
      "-> q_predicates: [(Lead, []), (singer, ['P86', 'P4757'])]\n",
      "-> q_predicates \tRunning time is 2.72s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (Lead, []), (singer, ['P86', 'P4757'])]\n",
      "----> q_themes in context: ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['singer', 'Singer']\n",
      "---> Meaningful keywords enhanced by previous context: ['singer', 'Singer', 'country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [singer, Singer, country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(singer, ['Q177220']), (Singer, ['Q20607618', 'Q1260201']), (country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 46.45s\n",
      "-->  9 nodes and 8 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 9 nodes and 8 edges\n",
      "-> predicates_dict: {'P17': 3, 'P1963': 7, 'P1282': 1, 'P1013': 1, 'P136': 5, 'P361': 1, 'P31': 6, 'P407': 1, 'P527': 1, 'P577': 1, 'P138': 1, 'P921': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P131': 1, 'P734': 3, 'P1476': 1, 'P1705': 1, 'P2452': 2}\n",
      "-> paths_keywords: (['singer', 'country', 'south africa'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 139.79s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 5.53s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.09s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 198.8s\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex      question   answer domain qanswer  \\\n",
      "51            1004    3       False  Lead singer?  Q133151  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr    convex  \\\n",
      "51          0.39         0.0    False           0.19          0.0  Q2529757   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "51         0.94        0.0   False        384.74        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "51        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-52-ic1004-iq3-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1005/2240 -> 4/5 -> Convex=True: (Q133151) Lead singer?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q177220\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q177220\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex      question   answer domain qanswer  \\\n",
      "52            1004    3        True  Lead singer?  Q133151  music   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr   convex  \\\n",
      "52           0.3         0.0    False           0.18          0.0  Q177220   \n",
      "\n",
      "    convex_time  convex_rr  graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "52         0.75        0.0  Q177220          5.14        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "52        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-53-ic1004-iq3-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:25:38.119483\n",
      "\t>>> Processing 1005/2240 -> 5/5 -> Convex=False: (2017-01-01T00:00:00Z) When did they break up?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: When did they break up?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: When did they break up \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: When did vocal and Singing break up\n",
      "> Time related question detected\n",
      "-> q_themes: ([], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: break\n",
      "-> q_predicates: [(did, ['P248']), (break, [])]\n",
      "-> q_predicates \tRunning time is 3.83s\n",
      "--> Predicates enhanced by previous context: [(said to be the same as, ['P460']), (did, ['P248']), (break, [])]\n",
      "----> q_themes in context: ([(vocal, ['Q2529757'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['vocal']\n",
      "---> Meaningful keywords enhanced by previous context: ['vocal', 'Singing', 'vocal']\n",
      "meaningful_names_no_previous_answer [vocal, Singing, vocal]\n",
      "----> Meaningful keywords casted as theme ([(vocal, ['Q2529757']), (Singing, ['Q10371908']), (vocal, ['Q2529757'])], [])\n",
      "q_focused_parts: [(vocal, ['Q2529757']), (Singing, ['Q10371908']), (vocal, ['Q2529757'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 19.33s\n",
      "-->  7 nodes and 6 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 7 nodes and 6 edges\n",
      "-> predicates_dict: {'P460': 45, 'P31': 1, 'P279': 1}\n",
      "-> paths_keywords: (['vocal', 'singing', 'break'], {}, [When])\n",
      "-> Computing possible paths... (could be long)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 135.89s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.02s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.06s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: When did they break up?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: When did they break up \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: When did vocal and Singing break up\n",
      "> Time related question detected\n",
      "-> q_themes: ([], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: break\n",
      "-> q_predicates: [(did, ['P248']), (break, [])]\n",
      "-> q_predicates \tRunning time is 3.14s\n",
      "--> Predicates enhanced by previous context: [(said to be the same as, ['P460']), (did, ['P248']), (break, [])]\n",
      "----> q_themes in context: ([(vocal, ['Q2529757'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['vocal']\n",
      "---> Meaningful keywords enhanced by previous context: ['vocal', 'Singing', 'vocal']\n",
      "meaningful_names_no_previous_answer [vocal, Singing, vocal]\n",
      "----> Meaningful keywords casted as theme ([(vocal, ['Q2529757']), (Singing, ['Q10371908']), (vocal, ['Q2529757'])], [])\n",
      "q_focused_parts: [(vocal, ['Q2529757']), (Singing, ['Q10371908']), (vocal, ['Q2529757'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 19.69s\n",
      "-->  7 nodes and 6 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 7 nodes and 6 edges\n",
      "-> predicates_dict: {'P460': 45, 'P31': 1, 'P279': 1}\n",
      "-> paths_keywords: (['vocal', 'singing', 'break'], {}, [When])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 141.47s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.24s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.08s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 172.19s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: When did they break up?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: When did they break up \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: When did country break up\n",
      "> Time related question detected\n",
      "-> q_themes: ([], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: break\n",
      "-> q_predicates: [(did, ['P248']), (break, [])]\n",
      "-> q_predicates \tRunning time is 3.08s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (did, ['P248']), (break, [])]\n",
      "----> q_themes in context: ([(, [])], [])\n",
      "--> Potential meaningful keywords for the sentence: []\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 45.23s\n",
      "-->  9 nodes and 8 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 9 nodes and 8 edges\n",
      "-> predicates_dict: {'P17': 3, 'P1963': 6, 'P1282': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P577': 1, 'P131': 1, 'P361': 1, 'P527': 1, 'P31': 2, 'P138': 1, 'P921': 1, 'P433': 1, 'P2452': 2, 'P1476': 1}\n",
      "-> paths_keywords: (['country', 'south africa', 'break'], {}, [When])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 72\n",
      "->Computing possible paths \tRunning time is 111.38s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 8\n",
      "->\tRunning time is 3.95s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 3.01s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: When did they break up?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: When did they break up \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: When did country break up\n",
      "> Time related question detected\n",
      "-> q_themes: ([], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: break\n",
      "-> q_predicates: [(did, ['P248']), (break, [])]\n",
      "-> q_predicates \tRunning time is 3.17s\n",
      "--> Predicates enhanced by previous context: [(country, ['P17']), (did, ['P248']), (break, [])]\n",
      "----> q_themes in context: ([(, [])], [])\n",
      "--> Potential meaningful keywords for the sentence: []\n",
      "---> Meaningful keywords enhanced by previous context: ['country', 'South Africa', 'country']\n",
      "meaningful_names_no_previous_answer [country, South Africa, country]\n",
      "----> Meaningful keywords casted as theme ([(country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])], [])\n",
      "q_focused_parts: [(country, ['P17', 'Q6256']), (South Africa, ['Q55155433', 'Q28754705', 'Q258']), (country, ['P17', 'Q6256'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 42.03s\n",
      "-->  9 nodes and 8 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 9 nodes and 8 edges\n",
      "-> predicates_dict: {'P17': 3, 'P1963': 6, 'P1282': 1, 'P813': 1, 'P973': 1, 'P1709': 1, 'P577': 1, 'P131': 1, 'P361': 1, 'P527': 1, 'P31': 2, 'P138': 1, 'P921': 1, 'P433': 1, 'P2452': 2, 'P1476': 1}\n",
      "-> paths_keywords: (['country', 'south africa', 'break'], {}, [When])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 72\n",
      "->Computing possible paths \tRunning time is 108.98s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 8\n",
      "->\tRunning time is 3.93s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 2.96s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 164.73s\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex                 question  \\\n",
      "53            1004    4       False  When did they break up?   \n",
      "\n",
      "                  answer domain qanswer  qanswer_time  qanswer_rr platypus  \\\n",
      "53  2017-01-01T00:00:00Z  music   False          0.41         0.0    False   \n",
      "\n",
      "    platypus_time  platypus_rr convex  convex_time  convex_rr graphqa  \\\n",
      "53           1.47          0.0  False       336.35        0.0   False   \n",
      "\n",
      "    graphqa_time graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5  \\\n",
      "53        331.94        False        False        False        False   \n",
      "\n",
      "   graphqa_topall  graphqa_rr  \n",
      "53          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-54-ic1004-iq4-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1005/2240 -> 5/5 -> Convex=True: (2017-01-01T00:00:00Z) When did they break up?                                  \n",
      "Asking qAnswer\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q27939\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa 1986-03-28T00:00:00Z\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                 question  \\\n",
      "54            1004    4        True  When did they break up?   \n",
      "\n",
      "                  answer domain qanswer  qanswer_time  qanswer_rr platypus  \\\n",
      "54  2017-01-01T00:00:00Z  music   False          0.38         0.0    False   \n",
      "\n",
      "    platypus_time  platypus_rr  convex  convex_time  convex_rr  \\\n",
      "54           1.49          0.0  Q27939         0.31        0.0   \n",
      "\n",
      "                 graphqa  graphqa_time graphqa_top2 graphqa_top3 graphqa_top4  \\\n",
      "54  1986-03-28T00:00:00Z         18.07        False        False        False   \n",
      "\n",
      "   graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "54        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-55-ic1004-iq4-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:37:08.628347\n",
      "\t>>> Processing 1006/2240 -> 1/5 -> Convex=False: (Q51511) Who directed Night of the Living Dead?                                  \n",
      "Asking qAnswer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_qanswer Q152309\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Asking Convex\n",
      "df_convex Q51511\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1006 - 1 -> Convex Q51511\n",
      "\n",
      "Asking GraphQA\n",
      "User input: Who directed Night of the Living Dead?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Who directed Night of the Living Dead \n",
      "-> q_themes: ([(Night of the Living Dead, ['Q623051', 'Q545417']), (Night, ['Q575', 'Q592503']), (the Living Dead, ['Q7747842', 'Q27877445']), (Living Dead, ['Q3461339', 'Q7747841']), (the living, ['Q20504012', 'Q7747823'])], [directed Night])\n",
      "-> q_themes_enhanced: [('direct', ['Q15304504']), ('night', ['Q575']), ('Direct', ['Q1187369'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(directed, ['P57'])]\n",
      "-> q_predicates \tRunning time is 6.13s\n",
      "--> Potential meaningful keywords for the sentence: ['Night of the Living Dead', 'Night', 'the Living Dead', 'Living Dead', 'the living', 'direct', 'night', 'Direct']\n",
      "q_focused_parts: [(Night, ['Q13896309', 'Q7754054', 'Q16724838']), (Dead, ['Q2410746', 'Q504434', 'Q15874429', 'Q1180781'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 34.47s\n",
      "-->  122 nodes and 118 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 68 nodes and 68 edges\n",
      "---> Rebuilding the graph with k_deep 5 ... Previously: 68 nodes or 68 edges was below the limit of 100\n",
      "->New graph \tRunning time is 34.62s\n",
      "-->  160 nodes and 162 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 106 nodes and 110 edges\n",
      "-> predicates_dict: {'P57': 3, 'P1431': 1, 'P344': 2, 'P1040': 1, 'P31': 11, 'P136': 7, 'P58': 2, 'P585': 1, 'P166': 1, 'P155': 7, 'P156': 11, 'P50': 2, 'P138': 2, 'P527': 2, 'P364': 1, 'P175': 2, 'P123': 1, 'P407': 1, 'P1191': 1, 'P1113': 1, 'P1476': 1, 'P361': 1, 'P279': 5, 'P1001': 1, 'P3893': 1, 'P2437': 1, 'P577': 1, 'P1995': 2, 'P1545': 1, 'P4908': 1, 'P921': 1, 'P179': 1, 'P17': 1, 'P750': 1, 'P910': 1, 'P4224': 1}\n",
      "-> paths_keywords: (['night', 'dead', 'night of the living dead', 'living dead', 'the living', 'the night', 'the dead', 'per dead ohlin'], {'director': [director, ['P57']]}, [Who])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 280\n",
      "->Computing possible paths \tRunning time is 30.29s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 198\n",
      "->\tRunning time is 3.42s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: [['Q2460466', 2.774774553642436]]\n",
      "->Computing hypothesises \tRunning time is 23.89s\n",
      "-> Computing golden paths...\n",
      "--> len(golden_paths): 0\n",
      "->\tRunning time is 7.66s\n",
      "--> len(cleared_golden_paths): 0\n",
      "->\tTotal Running time is 144.33s\n",
      "\n",
      "df_graphqa Q2460466\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex                                question  \\\n",
      "55            1005    0       False  Who directed Night of the Living Dead?   \n",
      "\n",
      "    answer  domain  qanswer  qanswer_time  qanswer_rr platypus  platypus_time  \\\n",
      "55  Q51511  movies  Q152309          0.63         0.0    False           2.87   \n",
      "\n",
      "    platypus_rr  convex  convex_time  convex_rr   graphqa  graphqa_time  \\\n",
      "55          0.0  Q51511          4.7        1.0  Q2460466        144.57   \n",
      "\n",
      "   graphqa_top2 graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  \\\n",
      "55        False        False        False        False          False   \n",
      "\n",
      "    graphqa_rr  \n",
      "55         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-56-ic1005-iq0-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:39:41.424776\n",
      "\t>>> Processing 1006/2240 -> 2/5 -> Convex=False: (Q3072049) Genre of the movie?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Genre of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(genre, [])]\n",
      "-> q_predicates \tRunning time is 3.75s\n",
      "--> Predicates enhanced by previous context: [(genre, [])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n",
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Tom Savini, ['Q152309']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 11.28s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Genre of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: movie\n",
      "-> q_predicates: [(genre, []), (Genre, ['P136']), (movie, ['P57'])]\n",
      "-> q_predicates \tRunning time is 3.91s\n",
      "--> Predicates enhanced by previous context: [(genre, []), (Genre, ['P136']), (movie, ['P57'])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n",
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Tom Savini, ['Q152309']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 15.1s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Genre of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(genre, [])]\n",
      "-> q_predicates \tRunning time is 4.18s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (genre, [])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.58s\n",
      "-->  6 nodes and 6 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 6 nodes and 6 edges\n",
      "-> predicates_dict: {'P57': 26, 'P136': 5, 'P1013': 1, 'P106': 3, 'P344': 1, 'P31': 5, 'P971': 1, 'P131': 2, 'P19': 1, 'P175': 1, 'P282': 1, 'P734': 1, 'P17': 1, 'P155': 2, 'P585': 1, 'P166': 1, 'P156': 2, 'P1545': 2, 'P735': 2, 'P1001': 1, 'P3893': 1}\n",
      "-> paths_keywords: (['genre', 'movie', 'night of the living dead', 'george a. romero', 'the movie'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 137.43s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.87s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.06s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Genre director of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: movie\n",
      "-> q_predicates: [(genre, []), (Genre, ['P136']), (movie, ['P57'])]\n",
      "-> q_predicates \tRunning time is 4.18s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (genre, []), (Genre, ['P136'])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n",
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.18s\n",
      "-->  6 nodes and 6 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 6 nodes and 6 edges\n",
      "-> predicates_dict: {'P57': 26, 'P136': 5, 'P1013': 1, 'P106': 3, 'P344': 1, 'P31': 5, 'P971': 1, 'P131': 2, 'P19': 1, 'P175': 1, 'P282': 1, 'P734': 1, 'P17': 1, 'P155': 2, 'P156': 2, 'P585': 1, 'P166': 1, 'P1545': 2, 'P735': 2, 'P1001': 1, 'P3893': 1}\n",
      "-> paths_keywords: (['genre', 'movie', 'night of the living dead', 'george a. romero', 'the movie', 'director'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 54\n",
      "->Computing possible paths \tRunning time is 126.99s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 8\n",
      "->\tRunning time is 3.72s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 3.67s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 165.44s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Genre of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(genre, [])]\n",
      "-> q_predicates \tRunning time is 5.83s\n",
      "--> Predicates enhanced by previous context: [(genre, [])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n",
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (John A. Russo, ['Q6217924', 'Q2460466']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 12.72s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Genre of the movie?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Genre of the movie \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Genre of the movie\n",
      "-> q_themes: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: movie\n",
      "-> q_predicates: [(genre, []), (Genre, ['P136']), (movie, ['P57'])]\n",
      "-> q_predicates \tRunning time is 4.86s\n",
      "--> Predicates enhanced by previous context: [(genre, []), (Genre, ['P136']), (movie, ['P57'])]\n",
      "----> q_themes in context: ([(Genre, ['Q483394', 'Q37446719']), (the movie, ['Q7752538', 'Q16955125']), (Movie, ['Q43262595', 'Q2512663']), (genre, ['P136'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Genre', 'the movie', 'Movie', 'genre']\n",
      "---> Meaningful keywords enhanced by previous context: ['Genre', 'the movie', 'Movie', 'genre', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [Genre, the movie, Movie, genre, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(Genre, ['Q37446719']), (Movie, ['Q2512663', 'Q43262595']), (genre, ['P136']), (John A. Russo, ['Q6217924', 'Q2460466']), (the movie, ['Q7752538', 'Q16955125'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 14.68s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex             question    answer  domain  \\\n",
      "56            1005    1       False  Genre of the movie?  Q3072049  movies   \n",
      "\n",
      "   qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "56   False         34.48         0.0    False           0.37          0.0   \n",
      "\n",
      "   convex  convex_time  convex_rr graphqa  graphqa_time graphqa_top2  \\\n",
      "56  False       335.44        0.0   False         38.55        False   \n",
      "\n",
      "   graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "56        False        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-57-ic1005-iq1-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1006/2240 -> 2/5 -> Convex=True: (Q3072049) Genre of the movie?                                  \n",
      "qAnswer extended by Convex\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_qanswer Q200092\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q471839\n",
      "df_convex_rr 0.6666666666666666\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q2526255\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex             question    answer  domain  \\\n",
      "57            1005    1        True  Genre of the movie?  Q3072049  movies   \n",
      "\n",
      "    qanswer  qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  \\\n",
      "57  Q200092          0.19         0.0    False           0.33          0.0   \n",
      "\n",
      "     convex  convex_time  convex_rr   graphqa  graphqa_time graphqa_top2  \\\n",
      "57  Q471839         0.53   0.666667  Q2526255          0.06        False   \n",
      "\n",
      "   graphqa_top3 graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "57        False        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-58-ic1005-iq1-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:46:31.439988\n",
      "\t>>> Processing 1006/2240 -> 3/5 -> Convex=False: (Q1400) Film setting?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840'])]\n",
      "-> q_predicates \tRunning time is 2.7s\n",
      "--> Predicates enhanced by previous context: [(setting, ['P2408', 'P840'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.0s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840']), (Film, ['P57'])]\n",
      "-> q_predicates \tRunning time is 2.59s\n",
      "--> Predicates enhanced by previous context: [(setting, ['P2408', 'P840']), (Film, ['P57'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.07s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840'])]\n",
      "-> q_predicates \tRunning time is 2.49s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (setting, ['P2408', 'P840'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 26.74s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P1013': 1, 'P106': 3, 'P131': 1, 'P19': 1, 'P20': 1, 'P910': 1, 'P155': 3, 'P156': 3, 'P179': 1, 'P407': 1, 'P31': 4, 'P361': 1, 'P495': 1, 'P27': 1, 'P136': 1, 'P1196': 1, 'P175': 2, 'P1545': 2, 'P735': 2}\n",
      "-> paths_keywords: (['film', 'night of the living dead', 'george a. romero'], {'director': [director, ['P57']], 'set in period': [set in period, ['P2408']], 'narrative location': [narrative location, ['P840']]}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 146.78s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.68s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.1s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840']), (Film, ['P57'])]\n",
      "-> q_predicates \tRunning time is 2.62s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (setting, ['P2408', 'P840'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 27.54s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P1013': 1, 'P106': 3, 'P131': 1, 'P19': 1, 'P20': 1, 'P910': 1, 'P155': 3, 'P156': 3, 'P179': 1, 'P407': 1, 'P31': 4, 'P361': 1, 'P495': 1, 'P27': 1, 'P136': 1, 'P1196': 1, 'P175': 2, 'P1545': 2, 'P735': 2}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-> paths_keywords: (['film', 'night of the living dead', 'george a. romero'], {'director': [director, ['P57']], 'set in period': [set in period, ['P2408']], 'narrative location': [narrative location, ['P840']]}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 143.19s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.93s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.13s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 181.61s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840'])]\n",
      "-> q_predicates \tRunning time is 2.6s\n",
      "--> Predicates enhanced by previous context: [(setting, ['P2408', 'P840'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (John A. Russo, ['Q6217924', 'Q2460466'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.69s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Film setting?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Film setting \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Film setting\n",
      "-> q_themes: ([(film, ['Q11424'])], [Film Setting])\n",
      "-> q_themes_enhanced: [('Film', ['Q11332514']), ('Setting', ['Q351657']), ('setting', ['Q11631156'])]\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(setting, ['P2408', 'P840']), (Film, ['P57'])]\n",
      "-> q_predicates \tRunning time is 2.72s\n",
      "--> Predicates enhanced by previous context: [(setting, ['P2408', 'P840']), (Film, ['P57'])]\n",
      "----> q_themes in context: ([(film, ['Q11424'])], [Film])\n",
      "--> Potential meaningful keywords for the sentence: ['film', 'Film', 'Setting', 'setting']\n",
      "---> Meaningful keywords enhanced by previous context: ['film', 'Film', 'Setting', 'setting', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [film, Film, Setting, setting, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(film, ['Q11424']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(film, ['Q11424']), (John A. Russo, ['Q6217924', 'Q2460466'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 24.85s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex       question answer  domain qanswer  \\\n",
      "58            1005    2       False  Film setting?  Q1400  movies   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr convex  \\\n",
      "58          51.8         0.0    False           0.24          0.0  False   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "58       363.07        0.0   False         54.32        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "58        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-59-ic1005-iq2-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1006/2240 -> 3/5 -> Convex=True: (Q1400) Film setting?                                  \n",
      "qAnswer extended by Convex\n",
      "df_qanswer Q10800557\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q11424\n",
      "df_convex_rr 0.0\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q11424\n",
      "df_graphqa_rr 0.0\n",
      "   conversation_id turn plus_convex       question answer  domain    qanswer  \\\n",
      "59            1005    2        True  Film setting?  Q1400  movies  Q10800557   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr  convex  \\\n",
      "59          0.11         0.0    False           0.28          0.0  Q11424   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "59         2.18        0.0  Q11424          0.18        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "59        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-60-ic1005-iq2-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 16:54:23.649424\n",
      "\t>>> Processing 1006/2240 -> 4/5 -> Convex=False: (Q3568798) Composer?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, [])]\n",
      "-> q_predicates \tRunning time is 2.14s\n",
      "--> Predicates enhanced by previous context: [(composer, [])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [Composer, composer, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 13.89s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, []), (Composer, ['P86'])]\n",
      "-> q_predicates \tRunning time is 2.57s\n",
      "--> Predicates enhanced by previous context: [(composer, []), (Composer, ['P86'])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [Composer, composer, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "->New graph \tRunning time is 17.92s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, [])]\n",
      "-> q_predicates \tRunning time is 2.37s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (composer, [])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [Composer, composer, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 20.38s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P106': 4, 'P175': 1, 'P58': 3, 'P31': 2, 'P136': 2, 'P3442': 1, 'P3454': 1, 'P131': 1, 'P19': 1, 'P856': 1, 'P571': 1, 'P155': 2, 'P548': 1, 'P577': 3, 'P348': 3, 'P156': 2, 'P910': 1, 'P277': 1, 'P1545': 2, 'P735': 2}\n",
      "-> paths_keywords: (['composer', 'night of the living dead', 'george a. romero'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 136.97s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.69s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.06s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, []), (Composer, ['P86'])]\n",
      "-> q_predicates \tRunning time is 1.97s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (composer, []), (Composer, ['P86'])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [Composer, composer, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 23.32s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P106': 4, 'P3442': 1, 'P3454': 1, 'P175': 1, 'P58': 3, 'P31': 2, 'P136': 2, 'P131': 1, 'P19': 1, 'P856': 1, 'P571': 1, 'P155': 2, 'P548': 1, 'P577': 3, 'P348': 3, 'P156': 2, 'P910': 1, 'P277': 1, 'P1545': 2, 'P735': 2}\n",
      "-> paths_keywords: (['composer', 'night of the living dead', 'george a. romero'], {}, [])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 143.28s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 3.58s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.06s\n",
      "--> End of loop\n",
      "->\tTotal Running time is 175.61s\n",
      "\n",
      "df_convex False\n",
      "df_convex_rr 0\n",
      "\n",
      "GraphQA extended by GraphQA\n",
      "User input: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, [])]\n",
      "-> q_predicates \tRunning time is 1.96s\n",
      "--> Predicates enhanced by previous context: [(composer, [])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [Composer, composer, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (John A. Russo, ['Q6217924', 'Q2460466'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 15.48s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: Composer?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: Composer \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: Composer\n",
      "-> q_themes: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "-> q_predicates: [(composer, []), (Composer, ['P86'])]\n",
      "-> q_predicates \tRunning time is 2.8s\n",
      "--> Predicates enhanced by previous context: [(composer, []), (Composer, ['P86'])]\n",
      "----> q_themes in context: ([(Composer, ['Q36834', 'Q15252222']), (composer, ['P86'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['Composer', 'composer']\n",
      "---> Meaningful keywords enhanced by previous context: ['Composer', 'composer', 'John A. Russo']\n",
      "meaningful_names_no_previous_answer [Composer, composer, John A. Russo]\n",
      "----> Meaningful keywords casted as theme ([(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (John A. Russo, ['Q6217924', 'Q2460466'])], [])\n",
      "q_focused_parts: [(Composer, ['Q15252222']), (composer, ['P86', 'Q36834']), (John A. Russo, ['Q6217924', 'Q2460466'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 18.87s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_graphqa False\n",
      "df_graphqa_rr 0\n",
      "   conversation_id turn plus_convex   question    answer  domain qanswer  \\\n",
      "60            1005    3       False  Composer?  Q3568798  movies   False   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr convex  \\\n",
      "60         36.96         0.0    False           0.18          0.0  False   \n",
      "\n",
      "    convex_time  convex_rr graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "60       339.63        0.0   False         39.89        False        False   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "60        False        False          False         0.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-61-ic1005-iq3-pcFalse.pickle.bz2\n",
      "\n",
      "\n",
      "\t>>> Processing 1006/2240 -> 4/5 -> Convex=True: (Q3568798) Composer?                                  \n",
      "qAnswer extended by Convex\n",
      "df_qanswer Q152309\n",
      "df_qanswer_rr 0.0\n",
      "\n",
      "Asking Platypus\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by Convex\n",
      "df_convex Q3568798\n",
      "df_convex_rr 1.0\n",
      "\n",
      "CORRECT 1006 - 4 -> Convex Q3568798\n",
      "\n",
      "GraphQA extended by Convex\n",
      "df_graphqa Q3568798\n",
      "df_graphqa_rr 1.0\n",
      "\n",
      "CORRECT 1006 - 4 -> graphqa Q3568798\n",
      "\n",
      "PARTIAL_CORRECT 1006 - 4 -> graphqa in answers ['Q3568798']\n",
      "   conversation_id turn plus_convex   question    answer  domain  qanswer  \\\n",
      "61            1005    3        True  Composer?  Q3568798  movies  Q152309   \n",
      "\n",
      "    qanswer_time  qanswer_rr platypus  platypus_time  platypus_rr    convex  \\\n",
      "61          0.16         0.0    False           0.18          0.0  Q3568798   \n",
      "\n",
      "    convex_time  convex_rr   graphqa  graphqa_time graphqa_top2 graphqa_top3  \\\n",
      "61         1.04        1.0  Q3568798          0.32         True         True   \n",
      "\n",
      "   graphqa_top4 graphqa_top5 graphqa_topall  graphqa_rr  \n",
      "61         True         True           True         1.0  \n",
      "Saving Dataframe Done! /data/users/romain.claret/tm/mse.tm.chatbot.base/benchmark_pickles/convex-conversations-testset-para-2/benchmarking-qanswer-platypus-convex-qagraph-62-ic1005-iq3-pcTrue.pickle.bz2\n",
      "\n",
      "\n",
      "It is  2020-02-06 17:01:22.045822\n",
      "\t>>> Processing 1006/2240 -> 5/5 -> Convex=False: (1968-01-01T00:00:00Z) And what year did it come out?                                  \n",
      "qAnswer extended by GraphQA\n",
      "User input: And what year did it come out?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: And what date did it come out \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: And what date did Tom Savini come out\n",
      "> Time related question detected\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171'])]\n",
      "-> q_predicates \tRunning time is 5.54s\n",
      "--> Predicates enhanced by previous context: [(did, ['P248']), (come, ['P2171'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [date, Date, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 15.8s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "Looped in aggressive mode with: And what year did it come out?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: And what date did it come out \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: And what date did Tom Savini come out\n",
      "> Time related question detected\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171']), (date, ['P837'])]\n",
      "-> q_predicates \tRunning time is 5.12s\n",
      "--> Predicates enhanced by previous context: [(did, ['P248']), (come, ['P2171']), (date, ['P837'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'Tom Savini']\n",
      "meaningful_names_no_previous_answer [date, Date, Tom Savini]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Tom Savini, ['Q152309'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Tom Savini, ['Q152309'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 17.26s\n",
      "-->  0 nodes and 0 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 0 nodes and 0 edges\n",
      "---> Loop detected, returning the graph in the current state\n",
      "---> No nodes, cancelling this run\n",
      "\n",
      "df_qanswer False\n",
      "df_qanswer_rr 0\n",
      "\n",
      "Asking Platypus\n",
      "df_platypus False\n",
      "df_platypus_rr 0.0\n",
      "\n",
      "Convex extended by GraphQA\n",
      "User input: And what year did it come out?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: And what date did it come out \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> New q_nlp: And what date did Night of the Living Dead come out\n",
      "> Time related question detected\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171'])]\n",
      "-> q_predicates \tRunning time is 5.36s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (did, ['P248']), (come, ['P2171'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [date, Date, Night of the Living Dead, George A. Romero]\n",
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 20.67s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P577': 1, 'P569': 1, 'P570': 1, 'P1013': 3, 'P3245': 1, 'P3250': 2, 'P407': 2, 'P443': 1, 'P131': 1, 'P19': 1, 'P156': 2, 'P155': 2, 'P910': 1, 'P20': 1, 'P1196': 1, 'P31': 2, 'P179': 1, 'P527': 3, 'P279': 5, 'P642': 2, 'P509': 1, 'P495': 1, 'P1672': 1, 'P1709': 1, 'P180': 1, 'P186': 1, 'P175': 1, 'P1545': 2, 'P735': 2}\n",
      "-> paths_keywords: (['date', 'night of the living dead', 'george a. romero', 'night', 'come'], {}, [what])\n",
      "-> Computing possible paths... (could be long)\n",
      "--> len(path_nodes): 0\n",
      "->Computing possible paths \tRunning time is 167.69s\n",
      "-> Filtering paths... (could be long)\n",
      "--> len(paths_nodes_filtered): 0\n",
      "->\tRunning time is 4.34s\n",
      "-> Computing hypothesises...\n",
      "--> hypothesises: []\n",
      "->Computing hypothesises \tRunning time is 0.07s\n",
      "-> Looping on aggressive mode...\n",
      "\n",
      "Looped in aggressive mode with: And what year did it come out?\n",
      "--> Auto correcting question in progress...\n",
      "-> Auto corrected q_nlp: And what date did it come out \n",
      "> Processing in conversational context..\n",
      "-> Replacing pronouns from context..\n",
      "-> Replacing verbs in context..\n",
      "-> New q_nlp: And what date did Night of the Living Dead come out\n",
      "> Time related question detected\n",
      "-> q_themes: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "-> q_themes_enhanced: []\n",
      "--> Calculating predicates... (could be long.. depends on uncached unpure predicates)\n",
      "behold: get_most_similar started with: come\n",
      "-> q_predicates: [(did, ['P248']), (come, ['P2171']), (date, ['P837'])]\n",
      "-> q_predicates \tRunning time is 5.58s\n",
      "--> Predicates enhanced by previous context: [(director, ['P57']), (did, ['P248']), (come, ['P2171']), (date, ['P837'])]\n",
      "----> q_themes in context: ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q36603893', 'Q16997'])], [])\n",
      "--> Potential meaningful keywords for the sentence: ['date', 'Date']\n",
      "---> Meaningful keywords enhanced by previous context: ['date', 'Date', 'Night of the Living Dead', 'George A. Romero']\n",
      "meaningful_names_no_previous_answer [date, Date, Night of the Living Dead, George A. Romero]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----> Meaningful keywords casted as theme ([(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])], [])\n",
      "q_focused_parts: [(date, ['Q1652093', 'Q3016931']), (Date, ['Q16997']), (Night of the Living Dead, ['Q3876764', 'Q404038', 'Q2652860']), (George A. Romero, ['Q51511'])]\n",
      "-> Building the graph with k_deep 3 ... (could be long)\n",
      "->New graph \tRunning time is 22.78s\n",
      "-->  3 nodes and 2 edges\n",
      "--> Removing meaningless subgraphs\n",
      "--> New graph of: 3 nodes and 2 edges\n",
      "-> predicates_dict: {'P57': 26, 'P577': 1, 'P569': 1, 'P570': 1, 'P1672': 1, 'P180': 1, 'P186': 1, 'P1013': 3, 'P3245': 1, 'P3250': 2, 'P407': 2, 'P443': 1, 'P131': 1, 'P19': 1, 'P156': 2, 'P155': 2, 'P31': 2, 'P910': 1, 'P20': 1, 'P279': 5, 'P1196': 1, 'P179': 1, 'P527': 3, 'P642': 2, 'P509': 1, 'P495': 1, 'P1709': 1, 'P175': 1, 'P1545': 2, 'P735': 2}\n",
      "-> paths_keywords: (['date', 'night of the living dead', 'george a. romero', 'night', 'come'], {}, [what])\n",
      "-> Computing possible paths... (could be long)\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "[Errno 12] Cannot allocate memory",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-1515f4ad4221>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    156\u001b[0m                 convex_answer, convex_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n\u001b[1;32m    157\u001b[0m                                          \u001b[0manswer_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvex_answer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvex_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m                                          use_convex=use_convex)\n\u001b[0m\u001b[1;32m    159\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mconvex_answer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m                     \u001b[0mdf_convex_rr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_rr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvex_answer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-b91dccfc5f02>\u001b[0m in \u001b[0;36mask_graphqa\u001b[0;34m(question, verbose, timer, show_graph, cores, banning_str, answer_context, context_graph, use_convex, turn)\u001b[0m\n\u001b[1;32m     30\u001b[0m         result = graphqa.answer_question(\n\u001b[1;32m     31\u001b[0m             \u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshow_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbanning_str\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbanning_str\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             previous_answer=answer_context, previous_graph=context_graph)\n\u001b[0m\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36manswer_question\u001b[0;34m(question, verbose, aggressive, looped, deep_k, deep_k_step, deep_k_max, graph_size_min, graph_size_target, graph_size_max, paths_filter_max, paths_max, timer, g_paths, show_graph, cores, banning_str, reload_cache, answer_sentence, previous_answer, previous_graph, graph_size_target_context, deep_match, k_context, in_context, k_deep_followup, k_deep_context_graph, context_themes, previous_answers, max_deepness, g_autocorrect)\u001b[0m\n\u001b[1;32m   5628\u001b[0m                     \u001b[0mdeep_match\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeep_match\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mk_context\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0min_context\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk_deep_followup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mk_deep_followup\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5629\u001b[0m                     \u001b[0mk_deep_context_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mk_deep_context_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_themes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontext_themes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprevious_answers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprevious_answers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5630\u001b[0;31m                     \u001b[0mmax_deepness\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_deepness\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg_autocorrect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mg_autocorrect\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5631\u001b[0m                     )\n\u001b[1;32m   5632\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36manswer_question\u001b[0;34m(question, verbose, aggressive, looped, deep_k, deep_k_step, deep_k_max, graph_size_min, graph_size_target, graph_size_max, paths_filter_max, paths_max, timer, g_paths, show_graph, cores, banning_str, reload_cache, answer_sentence, previous_answer, previous_graph, graph_size_target_context, deep_match, k_context, in_context, k_deep_followup, k_deep_context_graph, context_themes, previous_answers, max_deepness, g_autocorrect)\u001b[0m\n\u001b[1;32m   5579\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5580\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"-> Computing possible paths... (could be long)\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5581\u001b[0;31m     \u001b[0mpath_nodes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_path_nodes_from_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq_nlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpaths_keywords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mspecial_pred_theshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthres_inter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtop_performance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_paths\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_paths\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpaths_max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim_paths_threshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"--> len(path_nodes):\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5583\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36mfind_path_nodes_from_graph\u001b[0;34m(nlp_question, graph, predicates_dict, keywords, threshold, special_pred_theshold, thres_inter, top_performance, min_paths, max_paths, sim_paths_threshold, sim_looped, cores)\u001b[0m\n\u001b[1;32m   3000\u001b[0m     \u001b[0;31m#print(\"filtered all_keywords\",all_keywords)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3001\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3002\u001b[0;31m     \u001b[0mmain_keyword_paths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_paths_keywords_nodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_keywords\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtop_performance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtop_performance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmax_paths\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_paths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3003\u001b[0m     \u001b[0malternative_keyword_paths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tm/mse.tm.chatbot.base/tmqa35.py\u001b[0m in \u001b[0;36mget_paths_keywords_nodes\u001b[0;34m(graph, keywords, threshold, top_performance, per_limit, max_paths, max_paths_proc_killer, cores)\u001b[0m\n\u001b[1;32m   2873\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mproc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprocs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2874\u001b[0m         \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdaemon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2875\u001b[0;31m         \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2876\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mproc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprocs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2877\u001b[0m         \u001b[0min_mp_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/process.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    110\u001b[0m                \u001b[0;34m'daemonic processes are not allowed to have children'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0m_cleanup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sentinel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0;31m# Avoid a refcycle if the target function holds an indirect\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/context.py\u001b[0m in \u001b[0;36m_Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProcess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mDefaultContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/context.py\u001b[0m in \u001b[0;36m_Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    275\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m             \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mpopen_fork\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mclass\u001b[0m \u001b[0mSpawnProcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBaseProcess\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinalizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_launch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mduplicate_for_child\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/qa/lib/python3.7/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36m_launch\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0mcode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0mparent_r\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchild_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [Errno 12] Cannot allocate memory"
     ]
    }
   ],
   "source": [
    "### Evaluate\n",
    "banning_str = False\n",
    "show_graph = False\n",
    "\n",
    "start_time = time.time()\n",
    "conversations_len = len(conversations)\n",
    "\n",
    "for i_c, conversation in enumerate(conversations):\n",
    "    questions = [turn['question'] for turn in conversation['questions']]\n",
    "    answers = [graphqa.wikidata_url_to_wikidata_id(turn['answer']) for turn in conversation['questions']]\n",
    "    domain = conversation['domain']\n",
    "    questions_len = len(questions)\n",
    "    \n",
    "    qanswer_answer, qanswer_graph = False,False\n",
    "    platypus_answer, platypus_graph = False,False\n",
    "    convex_answer, convex_graph = False,False\n",
    "    graphqa_answer, graphqa_graph = False,False\n",
    "\n",
    "    qanswer_answer_convex, qanswer_graph_convex = False,False\n",
    "    platypus_answer_convex, platypus_graph_convex = False,False\n",
    "    convex_answer_convex, convex_graph_convex = False,False\n",
    "    graphqa_answer_convex, graphqa_graph_convex = False,False\n",
    "    \n",
    "    #break\n",
    "    if i_c+1 < 1004:\n",
    "        continue\n",
    "        \n",
    "    for i_q,question in enumerate(questions):\n",
    "        if i_q+1 <0:\n",
    "            continue\n",
    "            \n",
    "        print(\"It is \", datetime.now())\n",
    "                \n",
    "        for use_convex in [False,True]:\n",
    "\n",
    "            answer = answers[i_q]\n",
    "            print(\"\\r\\t>>> Processing {}/{} -> {}/{} -> Convex={}: ({}) {}\".format(i_c+1,conversations_len,i_q+1,questions_len,use_convex,answer,question), \n",
    "                  end='                                  ')\n",
    "            #time.sleep(1)\n",
    "\n",
    "            #ASK QANSWER\n",
    "            start_time = time.time()            \n",
    "            if qanswer_graph and not use_convex:\n",
    "                print(\"\\nqAnswer extended by GraphQA\")\n",
    "                qanswer_answer, qanswer_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=qanswer_answer, context_graph=qanswer_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if qanswer_answer: \n",
    "                    df_qanswer_rr = get_rr(qanswer_answer[0], answer)\n",
    "                    if qanswer_answer[0]: df_qanswer = qanswer_answer[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "\n",
    "            elif qanswer_graph_convex and use_convex:\n",
    "                print(\"\\nqAnswer extended by Convex\")\n",
    "                qanswer_answer_convex, qanswer_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=qanswer_answer_convex, context_graph=qanswer_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if qanswer_answer_convex:\n",
    "                    df_qanswer_rr = get_rr(qanswer_answer_convex[0], answer)\n",
    "                    if qanswer_answer_convex[0]: df_qanswer = qanswer_answer_convex[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "\n",
    "            else:\n",
    "                print(\"\\nAsking qAnswer\")\n",
    "                qanswer_answer, qanswer_graph = ask_qanswer(question)\n",
    "                if qanswer_answer: \n",
    "                    qanswer_answer=[[qanswer_answer],[]]\n",
    "                    qanswer_graph=nx.Graph()\n",
    "                    qanswer_graph.add_node(qanswer_answer[0][0], name=graphqa.get_wd_label(qanswer_answer[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                else: \n",
    "                    qanswer_answer=[[],[]]\n",
    "                    qanswer_graph=nx.Graph()\n",
    "                if qanswer_answer:\n",
    "                    df_qanswer_rr = get_rr(qanswer_answer[0], answer)\n",
    "                    if qanswer_answer[0]: df_qanswer = qanswer_answer[0][0]\n",
    "                    else: df_qanswer = False\n",
    "                else: \n",
    "                    df_qanswer = False\n",
    "                    df_qanswer_rr = 0\n",
    "                if show_graph: graphqa.plot_graph(qanswer_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                \n",
    "                qanswer_answer_convex = qanswer_answer.copy()\n",
    "                qanswer_graph_convex = qanswer_graph.copy()\n",
    "            \n",
    "            print(\"df_qanswer\",df_qanswer) \n",
    "            print(\"df_qanswer_rr\",df_qanswer_rr)\n",
    "            \n",
    "            df_qanswer_time = round(time.time()-start_time,2)\n",
    "            \n",
    "            ## ASK PLATYPUS\n",
    "            start_time = time.time()\n",
    "            if platypus_graph and not use_convex:\n",
    "                print(\"\\nPlatypus extended by GraphQA\")\n",
    "                platypus_answer, platypus_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=platypus_answer, context_graph=platypus_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if platypus_answer: \n",
    "                    df_platypus_rr = get_rr(platypus_answer[0], answer)\n",
    "                    if platypus_answer[0]: platypus_answer[0][0]: df_platypus = platypus_answer[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "\n",
    "            elif platypus_graph_convex and use_convex:\n",
    "                print(\"\\nPlatypus extended by Convex\")\n",
    "                platypus_answer_convex, platypus_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=platypus_answer_convex, context_graph=platypus_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if platypus_answer_convex: \n",
    "                    df_platypus_rr = get_rr(platypus_answer_convex[0], answer)\n",
    "                    if platypus_answer_convex[0]: df_platypus = platypus_answer_convex[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "\n",
    "            else:\n",
    "                print(\"\\nAsking Platypus\")\n",
    "                platypus_answer, platypus_graph = ask_platypus(question)\n",
    "                if platypus_answer: \n",
    "                    platypus_answer=[[platypus_answer],[]]\n",
    "                    platypus_graph=nx.Graph()\n",
    "                    platypus_graph.add_node(platypus_answer[0][0], name=graphqa.get_wd_label(platypus_answer[0][0]), type='entity', turn=i_q+1, weight=1, qa=True)\n",
    "                else: \n",
    "                    platypus_answer=[[],[]]\n",
    "                    platypus_graph=nx.Graph()\n",
    "                if platypus_answer:\n",
    "                    df_platypus_rr = get_rr(platypus_answer[0], answer)\n",
    "                    if platypus_answer[0]: df_platypus = platypus_answer[0][0]\n",
    "                    else: df_platypus = False\n",
    "                else: \n",
    "                    df_platypus_rr = 0\n",
    "                    df_platypus = False\n",
    "                if show_graph: graphqa.plot_graph(platypus_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                \n",
    "                platypus_answer_convex = platypus_answer.copy()\n",
    "                platypus_graph_convex = platypus_graph.copy()\n",
    "            \n",
    "            print(\"df_platypus\",df_platypus) \n",
    "            print(\"df_platypus_rr\",df_platypus_rr)\n",
    "            \n",
    "            df_platypus_time = round(time.time()-start_time,2)\n",
    "            \n",
    "            \n",
    "            ## ASK CONVEX\n",
    "            start_time = time.time()\n",
    "            if convex_graph and not use_convex:\n",
    "                print(\"\\nConvex extended by GraphQA\")\n",
    "                convex_answer, convex_graph = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=convex_answer, context_graph=convex_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if convex_answer:\n",
    "                    df_convex_rr = get_rr(convex_answer[0], answer)\n",
    "                    if convex_answer[0]: df_convex = convex_answer[0][0]\n",
    "                    else: df_convex = False\n",
    "                else: \n",
    "                    df_convex_rr = 0\n",
    "                    df_convex = False\n",
    "                    \n",
    "            elif convex_graph_convex and use_convex:\n",
    "                print(\"\\nConvex extended by Convex\")\n",
    "                convex_answer_convex, convex_graph_convex = ask_graphqa(question, verbose=True, timer=True, show_graph=show_graph, banning_str=banning_str,\n",
    "                                         answer_context=convex_answer_convex, context_graph=convex_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if convex_answer_convex:\n",
    "                    df_convex_rr = get_rr(convex_answer_convex[0], answer)\n",
    "                    if convex_answer_convex[0]: df_convex = convex_answer_convex[0][0]\n",
    "                    else: df_convex = False\n",
    "                else: \n",
    "                    df_convex = False\n",
    "                    df_convex_rr = 0\n",
    "                    \n",
    "            else:\n",
    "                print(\"\\nAsking Convex\")\n",
    "                convex_answer, convex_graph = ask_convex(question)\n",
    "                #([['Q766106'], ['Q76', 'P25', 'Q766106']],<networkx.classes.graph.Graph at 0x7f94423d1f90>)\n",
    "                if not convex_answer: \n",
    "                    convex_answer=[[],[]]\n",
    "                    convex_graph=nx.Graph()\n",
    "                if convex_answer:\n",
    "                    df_convex_rr = get_rr(convex_answer[0], answer)\n",
    "                    if convex_answer[0]: df_convex = convex_answer[0][0]\n",
    "                    else: ddf_convex = False\n",
    "                else: \n",
    "                    df_convex = False\n",
    "                    df_convex_rr = 0\n",
    "                if show_graph: graphqa.plot_graph(convex_graph, \"file_name_context_graph\", \"Context_Graph_title\")\n",
    "                convex_answer_convex = convex_answer.copy()\n",
    "                convex_graph_convex = convex_graph.copy()\n",
    "                \n",
    "            print(\"df_convex\",df_convex) \n",
    "            print(\"df_convex_rr\",df_convex_rr)\n",
    "            \n",
    "            df_convex_time = round(time.time()-start_time,2)\n",
    "                \n",
    "                \n",
    "            \n",
    "\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> qAnswer\", df_qanswer) if df_qanswer == answer else False\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> Platypus\", df_platypus) if df_platypus == answer else False\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> Convex\", df_convex) if df_convex == answer else False\n",
    "\n",
    "            df_graphqa = False\n",
    "            df_graphqa_top2 = False\n",
    "            df_graphqa_top3 = False\n",
    "            df_graphqa_top4 = False\n",
    "            df_graphqa_top5 = False\n",
    "            df_graphqa_topall = False\n",
    "            df_graphqa_rr = 0\n",
    "            \n",
    "            start_time = time.time()\n",
    "            if graphqa_graph and not use_convex:\n",
    "                print(\"\\nGraphQA extended by GraphQA\")\n",
    "                graphqa_answer, graphqa_graph = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer, context_graph=graphqa_graph,\n",
    "                                         use_convex=use_convex)\n",
    "                if graphqa_answer:\n",
    "                    if graphqa_answer[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer[0], answer)\n",
    "                        if graphqa_answer[0][0]: df_graphqa = graphqa_answer[0][0]\n",
    "                        if answer in graphqa_answer[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer[0]: df_graphqa_topall = True\n",
    "                \n",
    "            elif graphqa_graph and use_convex:\n",
    "                print(\"\\nGraphQA extended by Convex\")\n",
    "                graphqa_answer_convex, graphqa_graph_convex = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer_convex, context_graph=graphqa_graph_convex,\n",
    "                                         use_convex=use_convex, turn=i_q+1)\n",
    "                if graphqa_answer_convex:\n",
    "                    if graphqa_answer_convex[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer_convex[0], answer)\n",
    "                        if graphqa_answer_convex[0][0]: df_graphqa = graphqa_answer_convex[0][0]\n",
    "                        if answer in graphqa_answer_convex[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer_convex[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer_convex[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer_convex[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer_convex[0]: df_graphqa_topall = True\n",
    "                \n",
    "            else:\n",
    "                print(\"\\nAsking GraphQA\")\n",
    "                graphqa_answer, graphqa_graph = ask_graphqa(question, verbose=True, timer=True, banning_str=banning_str,\n",
    "                                         answer_context=graphqa_answer, context_graph=graphqa_graph,\n",
    "                                         use_convex=False)\n",
    "                if not graphqa_answer: \n",
    "                    graphqa_answer=[[],[]]\n",
    "                    graphqa_graph=nx.Graph()\n",
    "                else:\n",
    "                    graphqa_answer_convex = graphqa_answer.copy()\n",
    "                    graphqa_graph_convex = graphqa_graph.copy()\n",
    "                \n",
    "                if graphqa_answer:\n",
    "                    if graphqa_answer[0]:\n",
    "                        df_graphqa_rr = get_rr(graphqa_answer[0], answer)\n",
    "                        if graphqa_answer[0][0]: df_graphqa = graphqa_answer[0][0]\n",
    "                        if answer in graphqa_answer[0][:2]: df_graphqa_top2 = True\n",
    "                        if answer in graphqa_answer[0][:3]: df_graphqa_top3 = True\n",
    "                        if answer in graphqa_answer[0][:4]: df_graphqa_top4 = True\n",
    "                        if answer in graphqa_answer[0][:5]: df_graphqa_top5 = True\n",
    "                        if answer in graphqa_answer[0]: df_graphqa_topall = True\n",
    "                            \n",
    "            print(\"df_graphqa\",df_graphqa) \n",
    "            print(\"df_graphqa_rr\",df_graphqa_rr)\n",
    "                \n",
    "\n",
    "            df_graphqa_time = round(time.time()-start_time,2)\n",
    "\n",
    "            df = df.append({\n",
    "                'conversation_id':i_c,'turn':i_q,\"plus_convex\":use_convex,\n",
    "                'question':question, 'answer':answer,'domain':domain,\n",
    "                'qanswer':df_qanswer,'qanswer_time':df_qanswer_time, 'qanswer_rr':df_qanswer_rr,\n",
    "                'platypus':df_platypus,'platypus_time':df_platypus_time, 'platypus_rr':df_platypus_rr,\n",
    "                'convex':df_convex,'convex_time':df_convex_time, 'convex_rr':df_convex_rr,\n",
    "                'graphqa':df_graphqa, 'graphqa_time':df_graphqa_time, 'graphqa_top2':df_graphqa_top2,\n",
    "                \"graphqa_top3\":df_graphqa_top3,\"graphqa_top4\":df_graphqa_top4, \"graphqa_top5\":df_graphqa_top5,\n",
    "                \"graphqa_topall\":df_graphqa_topall, \"graphqa_rr\":df_graphqa_rr},\n",
    "               ignore_index=True)\n",
    "\n",
    "            print(\"\\nCORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa\", df_graphqa) if str(df_graphqa) == str(answer) else False\n",
    "            if use_convex: print(\"\\nPARTIAL_CORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa in answers\", graphqa_answer_convex[0]) if df_graphqa_topall == True else False\n",
    "            else: print(\"\\nPARTIAL_CORRECT\",i_c+1,\"-\",i_q+1, \"-> graphqa in answers\", graphqa_answer[0]) if df_graphqa_topall == True else False\n",
    "\n",
    "            print(df.tail(1))\n",
    "\n",
    "            pickle_data(df, \"benchmarking-qanswer-platypus-convex-qagraph-\"+str(len(df))+\"-ic\"+str(i_c)+\"-iq\"+str(i_q)+\"-pc\"+str(use_convex))\n",
    "\n",
    "            print(\"\\n\")\n",
    "            \n",
    "            if i_q == 0: \n",
    "                break\n",
    "\n",
    "        #if i_q >= 1:      \n",
    "            #break\n",
    "    \n",
    "    #break\n",
    "\n",
    "print(\"->\\tRunning time is {}s\".format(round(time.time()-start_time,2)))\n",
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result_convex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.drop(df.tail(1).index,inplace=True) # drop last n rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "\n",
    "print(df_copy[df_copy.graphqa == df_copy.answer][\"graphqa_rr\"]) \n",
    "df_copy.loc[df_copy[\"graphqa\"] == df_copy[\"answer\"], 'graphqa_rr'] = 1\n",
    "print(df_copy[df_copy.graphqa == df_copy.answer][\"graphqa_rr\"]) \n",
    "\n",
    "print(df_copy[df_copy.convex == df_copy.answer][\"convex_rr\"]) \n",
    "df_copy.loc[df_copy[\"convex\"] == df_copy[\"answer\"], 'convex_rr'] = 1  \n",
    "print(df_copy[df_copy.convex == df_copy.answer][\"convex_rr\"]) \n",
    "\n",
    "print(df_copy)  \n",
    "df = df_copy # made at from 0 to 278, len of 279"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SAVING\n",
    "#pickle_data(df_loaded, \"benchmarking-qanswer-platypus-convex-tm1-from-0-to-\"+str(len(df_loaded)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOADING\n",
    "#df_loaded = pd.read_pickle(\"/data/users/romain.claret/tm/wikidata-simplequestions/benchmark_pickles/benchmarking-qanswer-platypus-convex-tm1-from-0-to-9961.pickle.bz2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['qanswer'][34] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del df_loaded['tm2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.rename({'mine':'tm1'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded['tm1_top4'] = \"False\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded = df_loaded[['question','source','qanswer','platypus','convex','tm1','tm1_time','tm1_top2','tm1_top3','tm1_top4','tm1_top5','tm1_topall']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded_len = len(df_loaded)\n",
    "#for i, question in enumerate(df_loaded['question']):\n",
    "#    if i >= 0:\n",
    "#    #if i >= 497:\n",
    "#        source = str(df_loaded['source'][i])\n",
    "#        print(str(i)+\"/\"+str(df_loaded_len),question,\"-> source:\",source)\n",
    "#        \n",
    "#        start_time = time.time()\n",
    "#        result_tmqa_1 = ask_tmqa_1(question, verbose=True)\n",
    "#        \n",
    "#        if result_tmqa_1:\n",
    "#            df_loaded['tm1'][i] = result_tmqa_1[0]\n",
    "#            if source in result_tmqa_1[:2]:\n",
    "#                df_loaded['tm1_top2'][i] = True\n",
    "#            if source in result_tmqa_1[:3]:\n",
    "#                df_loaded['tm1_top3'][i] = True\n",
    "#            if source in result_tmqa_1[:4]:\n",
    "#                df_loaded['tm1_top4'][i] = True\n",
    "#            if source in result_tmqa_1[:5]:\n",
    "#                df_loaded['tm1_top5'][i] = True\n",
    "#            if source in result_tmqa_1:\n",
    "#                df_loaded['tm1_topall'][i] = True\n",
    "#        else:\n",
    "#            df_loaded['tm1'][i] = False\n",
    "#        end_time = time.time()\n",
    "#        df_loaded['tm1_time'][i] = round(end_time-start_time,2)\n",
    "#        print(\"->\\tRunning time is {}s\".format(round(end_time-start_time,2)))\n",
    "#        print(str(str(df_loaded['tm1'][i])==str(source)),\"---> result_tmqa_1:\",str(result_tmqa_1)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_loaded.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = df_loaded.copy()\n",
    "#df = df.replace(\"\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_backup = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_row = 496\n",
    "#df_len = len(df)\n",
    "#df_qanswer_max = df[(df.index<=max_row) & (df.qanswer == df.source)]\n",
    "#df_qanswer_max_len = len(df_qanswer_max)\n",
    "#\n",
    "#df_platypus_max = df[(df.index<=max_row) & (df.platypus == df.source)]\n",
    "#df_platypus_max_len = len(df_platypus_max)\n",
    "#\n",
    "#df_convex_max = df[(df.index<=max_row) & (df.convex == df.source)]\n",
    "#df_convex_max_len = len(df_convex_max)\n",
    "#\n",
    "#df_tm1_max = df[(df.index<=max_row) & (df.tm1 == df.source)]\n",
    "#df_tm1_max_len = len(df_tm1_max)\n",
    "#\n",
    "#df_tm1_max_top2 = df[(df.index<=max_row) & (df.tm1_top2 == True)]\n",
    "#df_tm1_max_top2_len = len(df_tm1_max_top2)\n",
    "#\n",
    "#df_tm1_max_top3 = df[(df.index<=max_row) & (df.tm1_top3 == True)]\n",
    "#df_tm1_max_top3_len = len(df_tm1_max_top3)\n",
    "#\n",
    "#df_tm1_max_top4 = df[(df.index<=max_row) & (df.tm1_top4 == True)]\n",
    "#df_tm1_max_top4_len = len(df_tm1_max_top4)\n",
    "#\n",
    "#df_tm1_max_top5 = df[(df.index<=max_row) & (df.tm1_top5 == True)]\n",
    "#df_tm1_max_top5_len = len(df_tm1_max_top5)\n",
    "#\n",
    "#df_tm1_max_topall = df[(df.index<=max_row) & (df.tm1_topall == True)]\n",
    "#df_tm1_max_topall_len = len(df_tm1_max_topall)\n",
    "#\n",
    "#print(\"qanswer:\", df_qanswer_max_len,df_qanswer_max_len/max_row)\n",
    "#print(\"platypus:\", df_platypus_max_len, df_platypus_max_len/max_row)\n",
    "#print(\"convex:\", df_convex_max_len, df_convex_max_len/max_row)\n",
    "#print(\"tm1:\", df_tm1_max_len, df_tm1_max_len/max_row)\n",
    "#print(\"tm1_top2:\", df_tm1_max_top2_len, df_tm1_max_top2_len/max_row)\n",
    "#print(\"tm1_top3:\", df_tm1_max_top3_len, df_tm1_max_top3_len/max_row)\n",
    "#print(\"tm1_top4:\", df_tm1_max_top4_len, df_tm1_max_top4_len/max_row)\n",
    "#print(\"tm1_top5:\", df_tm1_max_top5_len, df_tm1_max_top5_len/max_row)\n",
    "#print(\"tm1_topall:\", df_tm1_max_topall_len, df_tm1_max_topall_len/max_row)\n",
    "#\n",
    "#df[ & (df.qanswer == df.source)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"qanswer:\",len(df[df.qanswer == df.source]),len(df[df.qanswer == df.source])/len(df))\n",
    "#print(\"platypus:\",len(df[df.platypus == df.source]),len(df[df.platypus == df.source])/len(df))\n",
    "#print(\"convex:\",len(df[df.convex == df.source]),len(df[df.convex == df.source])/len(df))\n",
    "#print(\"tm1:\",len(df[df.tm1 == df.source]),len(df[df.tm1 == df.source])/len(df))\n",
    "#print(\"tm1_top2:\",len(df[df.tm1_top2 == True]),len(df[df.tm1_top2 == True])/len(df))\n",
    "#print(\"tm1_top3:\",len(df[df.tm1_top3 == True]),len(df[df.tm1_top3 == True])/len(df))\n",
    "#print(\"tm1_top4:\",len(df[df.tm1_top4 == True]),len(df[df.tm1_top4 == True])/len(df))\n",
    "#print(\"tm1_top5:\",len(df[df.tm1_top5 == True]),len(df[df.tm1_top5 == True])/len(df))\n",
    "#print(\"tm1_topall:\",len(df[df.tm1_topall == True]),len(df[df.tm1_topall == True])/len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.tm1_top2 = df.tm1_top3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:qa]",
   "language": "python",
   "name": "conda-env-qa-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}