scicite_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import json\n",
    "from copy import deepcopy as dc\n",
    "import torch\n",
    "\n",
    "import joblib\n",
    "import re\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from data.utils import votes_filter, plot_overlap_conflict\n",
    "from data.scicite.utils import load_jsonl, extract_item, get_inst, textsplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "scicite_loc = 'data/scicite/{:s}.jsonl'\n",
    "\n",
    "train = load_jsonl(dc(scicite_loc).format('train'))\n",
    "dev = load_jsonl(dc(scicite_loc).format('dev'))\n",
    "test = load_jsonl(dc(scicite_loc).format('test'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nplm.plf import WeakRule, BinaryRERules, regex_decision\n",
    "\n",
    "train_instances, _ = get_inst(train, keep_instance=True)\n",
    "dev_instances, dev_labels = get_inst(dev, keep_instance=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7843137254901961 0.7843137254901961 0.08733624454148471\n",
      "[0.0, 1.0, 1.0]\n",
      "0.8837535014005602 0.8760244445497181 0.6888646288209607\n",
      "[0.9088607594936708, 0.7942583732057417, 0.9636363636363636]\n",
      "0.8973799126637555 0.7853329989610576 0.8973799126637555\n",
      "[0.9405204460966543, 0.9803921568627451, 0.5365853658536586]\n",
      "0.8918918918918919 0.8918918918918919 0.036026200873362446\n",
      "[1.0, 1.0, 0.0]\n",
      "1.0 1.0 0.001091703056768559\n",
      "[1.0, nan, nan]\n",
      "0.8002183406113537 0.757343631299063 0.8002183406113537\n",
      "[0.895910780669145, 0.5294117647058824, 0.943089430894309]\n",
      "0.8378378378378378 0.8378378378378378 0.06768558951965066\n",
      "[1.0, 1.0, 0.0]\n",
      "0.7297297297297297 0.7297297297297297 0.05895196506550218\n",
      "[0.0, 0.0, 1.0]\n",
      "0.7857142857142857 0.7857142857142857 0.012008733624454149\n",
      "[0.0, 0.0, 1.0]\n",
      "0.7142857142857143 0.7142857142857143 0.027292576419213975\n",
      "[0.0, 1.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def df1(instance):\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    new_sen = re.sub(r'\\([^)]*\\)', '', p+m+s).lower()\n",
    "    if ' we ' in new_sen or ' our ' in new_sen or ' by us ' in new_sen:\n",
    "        return 1\n",
    "    return -1\n",
    "\n",
    "\n",
    "firstperson_rule = WeakRule(exec_module=df1, label_maps={0:[0], 1:[1,2]})\n",
    "\n",
    "v1 = firstperson_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = firstperson_rule.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "re_patt_results = 'result|discussion|conclusion|observation'\n",
    "re_patt_method = 'method|approach|experiment|evaluation'\n",
    "re_patt_intro = 'introduction|background'\n",
    "\n",
    "def sectionTitleRule(instance):\n",
    "    t = str(instance['sectionName']).lower()\n",
    "    if regex_decision(re_patt_method, t):\n",
    "        return 1\n",
    "    elif regex_decision(re_patt_intro, t):\n",
    "        return 0\n",
    "    elif regex_decision(re_patt_results, t):\n",
    "        return 2\n",
    "    return -1\n",
    "\n",
    "sectionTitle_rule = WeakRule(exec_module=sectionTitleRule, label_maps={0:[0], 1:[1], 2:[0, 2]})\n",
    "v2 = sectionTitle_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = sectionTitle_rule.eval(dev_labels, class_wise_acc=True)\n",
    "\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "def df2(instance):\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    new_sen = re.sub(r'\\([^)]*\\)', '', p+s).lower()\n",
    "    if ' result ' in new_sen or ' results ' in new_sen:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "result_rule = WeakRule(exec_module=df2, label_maps={0:[0,1], 1:[2]})\n",
    "v3 = result_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = result_rule.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "def length_citation(instance):\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    if len(m.split(';')) > 2:\n",
    "        return 1\n",
    "    return -1\n",
    "\n",
    "simple_re_rule_0 = WeakRule(exec_module=length_citation, label_maps={0:[2], 1:[0,1]})\n",
    "v4 = simple_re_rule_0.execute(dev_instances)\n",
    "acc, p, r, cwacc = simple_re_rule_0.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "def df3(instance):\n",
    "    def result_related(inst):\n",
    "        patterns = ['equivocal result', 'similar result', 'same result', 'different result', 'expected result']\n",
    "        for pattern in patterns:\n",
    "            if pattern in inst:\n",
    "                return 1\n",
    "        return -1\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    new_sen = re.sub(r'\\([^)]*\\)', '', p+s).lower()\n",
    "    return result_related(new_sen)\n",
    "\n",
    "\n",
    "res_rule = WeakRule(exec_module=df3, label_maps={0:[1], 1:[0,2]})\n",
    "v7 = res_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = res_rule.eval(dev_labels, class_wise_acc=True, average='macro')\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "re_patt = 'using|measuring|used|the method|data|state-of-art|calculated|applied|according to|approach'\n",
    "\n",
    "simple_re_rule_1 = BinaryRERules(re_pattern=re_patt, preproc=lambda inst:inst['string'], label_maps={0:[0,2], 1:[1]}, unipolar=False)\n",
    "v5 = simple_re_rule_1.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = simple_re_rule_1.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "def df5(instance):    \n",
    "    def match(inst):\n",
    "        patterns = [\n",
    "            'has been', 'in order to', 'considered to', \n",
    "            'initially', 'even if','have shown', 'has shown'\n",
    "        ]\n",
    "        for pattern in patterns:\n",
    "            if pattern in inst:\n",
    "                return 1\n",
    "        return -1\n",
    "\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    new_sen = re.sub(r'\\([^)]*\\)', '', p+s).lower()\n",
    "    return match(new_sen)\n",
    "\n",
    "\n",
    "add_rule = WeakRule(exec_module=df5, label_maps={0:[2], 1:[0,1]})\n",
    "v9 = add_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = add_rule.eval(dev_labels, class_wise_acc=True, average='macro')\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "def df4(instance):\n",
    "    \n",
    "    def comparison(inst):\n",
    "        patterns = [\n",
    "            'in line with', 'discordant with', 'consistent with', 'keeping with', 'accordance with', 'agreement with', 'similar with',\n",
    "            'compared to', 'contrast to', 'contrary to', 'comparable to', 'contradict to', 'affirmed by', 'supported by', 'in support of', \n",
    "        ]\n",
    "        for pattern in patterns:\n",
    "            if pattern in inst:\n",
    "                return 1\n",
    "        return -1\n",
    "    p, m, s = textsplit(instance, p=False)\n",
    "    new_sen = re.sub(r'\\([^)]*\\)', '', p+s).lower()\n",
    "    return comparison(new_sen)\n",
    "\n",
    "\n",
    "resultp_rule = WeakRule(exec_module=df4, label_maps={0:[0,1], 1:[2]})\n",
    "v8 = resultp_rule.execute(dev_instances)\n",
    "acc, p, r, cwacc = resultp_rule.eval(dev_labels, class_wise_acc=True, average='macro')\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "re_patt = 'our result|this is in keeping|with previous|this study differ|this is in agreement with|this conclusion|this finding|similar result.*(found|observed|obtained)'\n",
    "\n",
    "re_tb = BinaryRERules(re_pattern=re_patt, preproc=lambda inst:inst['string'], label_maps={0:[0,1], 1:[2]}, unipolar=True)\n",
    "v5 = re_tb.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = re_tb.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "re_patt = 'we (employ|utilize)|iap-as|metaanalyses|temporal transition|was estimated|quantitative (analyses|analysis)|this procedure|implementation|sequence analysis|regularization method|were analysed|we adopt|bayesian|were sampled|quantitative method|fracture|simulating|this design|algorithm|developed|model performance|(was|were) evaluated|as control|scheme|control management|(was|were|is|are) measured|the rats|the pigs|we appl'\n",
    "\n",
    "re_tm = BinaryRERules(re_pattern=re_patt, preproc=lambda inst:inst['string'], label_maps={0:[0,2], 1:[1]}, unipolar=True)\n",
    "v5 = re_tm.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = re_tm.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: [0], 1: [1, 2]}\n",
      "{0: [0], 1: [1], 2: [0, 2]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [2], 1: [0, 1]}\n",
      "{0: [1], 1: [0, 2]}\n",
      "{0: [0, 2], 1: [1]}\n",
      "{0: [2], 1: [0, 1]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [0, 2], 1: [1]}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAEYCAYAAADlIcXmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAogklEQVR4nO3de5xdVX338c83IRBAINxFQMASUbAFIQW8iyg3qaFqKagYKTX4iNZ7C33qC+ulSm1V7KPUVKLgBaSxFmoRyAOI9akgV1FAmogihDsJIEKAJN/nj73GHIaZOXtmn03mnPm+X6/9mn3W3nvtNZPMb9baa+21ZJuIiJiYaeu6ABER/SxBNCKigQTRiIgGEkQjIhpIEI2IaCBBNCKigQTRqE2SJe26rssRMZkkiPY5SW+T9FNJj0i6S9Jpkmat63JFTBUJon1M0geAU4APAZsB+wM7AYslrd/D+6zXq7wiBk2CaJ+StCnwt8C7bV9g+wnbvwKOBHYGPijpUUlbdFzzQkn3SZpRPv+ZpJskrZB0oaSdOs61pBMkLQGWjHD/10q6VtJDkm6T9JGOYzuX6+dLukPSnZI+2HF8X0lXlWvvlvSZnv+AIp4mCaL968XATODfOhNtPwycD/w+8CPgDR2H3wQssv2EpLnAXwOvB7YG/gs4a9g9jgD2A3Yf4f6/Bd4KzAJeC/wvSUcMO+cAYDZwEPBXkl5d0k8FTrW9KfB7wDl1vuGIyShBtH9tBdxne9UIx+4sx78JHA0gScBRJQ3gHcAnbd9U8vg7YK/O2mg5vtz2o8NvYPv7tn9qe43t66kC8CuGnfa3tn9r+6fAV4bKAjwB7CppK9sP2758At9/xKSQINq/7gO2GuV55Xbl+LeBF0naDng5sIaqxgnVs9NTJT0g6QFgOSBg+458bhvt5pL2k3SppHslPUgVlLcadlrn9bcCzyr7xwHPBX4u6UpJh3f9biMmqQTR/vUj4DGq5vjvSHoGcChwse0VwEXAn1I15c/22mm7bgOOtz2rY9vQ9n93ZDfWFF/fBM4DdrS9GfDPVEG4044d+88G7gCwvcT20cA2VB1jiyRtXPcbj5hMEkT7lO0HqTqW/knSIZJmSNqZ6vni7cDXyqnfpHp2+UbWNuWhCnonSdoDQNJmkv5kHEXYBFhue6WkfamC9HAflrRRucexwLfKvd4iaWvba4AHyrlrxnHviEkjQ1f6mO2/l3Q/8A9UHTQPAf8OvNn2Y+W084AvA7+2/ZOOa79Taq1nl+egDwKLgX+teft3Av8o6f8Al1EF71nDzrkMWEr1x/ofbF9U0g8BPiNpI6pm/lEjPXeN6AfKpMzRa6VG/EtgxigdXxEDI835iIgGEkQjIhpIcz4iooHURCMiGmild3796Rt5wxmbtpE1tFlz1vBhjr3lxx5vNf+2aIOezWXyFP36MwGeOiq2D6z0b3ncj/W05AcfsLHvX7661rlXX//YhbYP6eX917VWguiGMzblRTvNayNrtLK9XzrPbC9YAKxeckur+bdl+k7PaS3v1Ut/2VreqN2Glqb1XxS9fNWFPc/z/uWr+fGFz6517vTtlgx/q63vZZxoRDRiYM0UflciQTQiGjKrPXWDaDqWIqKRqibqWlsdkt4n6QZJP5N0lqSZknaRdIWkpZK+NTTpuKQNyuel5fjOHfmcVNJvlnRwK988CaIR0ZAxT3h1ra0bSdsDfwHMsf0CYDrVFI6nAJ+1vSuwgmomMMrXFSX9s+U8JO1ertuD6jXjL0qa3tNvvEgQjYjGelkTpXrMuGGZ5nEjqvlxXwUsKsfPoJowHGBu+Uw5fmCZO3cu1axlj9n+JdUcDvs2/T5HkiAaEY0YWI1rbVRz4F7Vsc1/Ul72MqoJdX5NFTwfBK4GHuiYh+F21s57uz1l3tpy/EFgy870Ea7pqXQsRURj46hl3md7zmgHJW1OVYvchWqaxH+lao5PWrVqomW+ypvLQ9oT2y5URPQPA6vtWlsNrwZ+afte209QrSH2EmBWxyoOOwDLyv4yyuTf5fhmwP2d6SNc01Ndg2h5GPsFqtnSdweOLg9tIyKAakbtOlsNvwb2L5N5CzgQuBG4lGpicYB5wLll/7zymXL8krJ6w3nAUaX3fheqBRN/PPHvcHR1mvP7Aktt3wIg6Wyq6vaNbRQoIvqL1z7vbJ6XfYWkRcA1wCrgWmAB8J9UE4h/vKSdXi45HfiapKVU64QdVfK5QdI5VHFqFXCCXWN4wATUCaIjPaDdb/hJ5QHxfICZ67X03nxETD6G1T2c0sL2ycDJw5JvYYTeddsrgRGXtbH9CeATvSvZyHrWO297ge05tuesP33DXmUbEZNcNdi+Z835vlOnJvq0PaCNiP5jxBPuv8lYeqVOEL0SmF0ezi6jeuYw0sqOETFFre7HeQF7pGsQtb1K0ruAC6lewVpo+4bWSxYRfaEabJ8gOibb5wPnt1yWiOhTa9Kcj4iYmNREIyIaMGL1FJ6GI0E0IhpLcz4iYoLSnI+IaESsdprzPeXHHm9tZcs/uKZ//+L97KUbtZb3mkcfbS3vPb7V3oqcNxzcv4s/rnnoodbynrbBBq3kq4d7H+yqN5YSRCMiJsQWj7uVlTf6QoJoRDS2Js9EIyImpupYSnM+ImKC0rEUETFh6ViKiGhodQbbR0RMTF77jIhoaM0UfiZaZ7XPhZLukfSzp6NAEdFfhnrn62zdSNpN0nUd20OS3itpC0mLJS0pXzcv50vS58ty7tdL2rsjr3nl/CWS5o1+12bq/Pn4KnBIWwWIiP5WLQ8yvdbWNS/7Ztt72d4L2Ad4BPgOcCJwse3ZwMXlM1RLuc8u23zgNABJW1Atdrcf1QJ3Jw8F3l7rGkRt/4BqKdKIiKewYbWn1drG6UDgF7ZvpVqm/YySfgZwRNmfC5zpyuXALEnbAQcDi20vt70CWExLlcGePRN90pLJtPeOeERMNhrPG0tbSbqq4/MC2wtGOfco4Kyyv63tO8v+XcC2ZX+kJd23HyO953oWRMsPYgHAptqih6tQR8RkZhhPLfM+23O6nSRpfeB1wElPuZ9tSZMmxkzdLrWI6JledSx1OBS4xvbd5fPdpZlO+XpPSR9tSfenban3BNGIaMSINa63jcPRrG3KA5wHDPWwzwPO7Uh/a+ml3x94sDT7LwQOkrR56VA6qKT1XNfmvKSzgFdSPcu4HTjZ9ultFCYi+lMvB9tL2hh4DXB8R/KngHMkHQfcChxZ0s8HDgOWUvXkHwtge7mkjwFXlvM+aruVDvI6684f3caNI2IwmN4Otrf9W2DLYWn3U/XWDz/XwAmj5LMQWNizgo0ibyxFREPKGksRERPV65pov0kQjYhGbNV6G2lQJYhGRGOZlDkiYoKqSZnzTLRvXL93ey8qXHjHda3lDXDo9Je1mn9bPv3Ma1vL+7XrtTi3jdt9qUVqMXCsP6OdfFspc5YHiYiYsKpjKTXRiIgJy8z2ERETNPTa51SVIBoRjWW1z4iICaomZU5NNCJiwtKcj4iYoKE1lqaqBNGIaGSqD3Gqs2TyjpIulXSjpBskvefpKFhE9AuxxtNqbYOoTk10FfAB29dI2gS4WtJi2ze2XLaI6BN57XMMZar9O8v+byTdRLVqXoJoRKR3fjwnS9oZeCFwxQjHsmRyxBQ1qE31OmoHUUnPAL4NvNf2Q8OPZ8nkiKlpUN5YkvQQIKq+st8lU8XJDeyR/1LU+vMhaQZVAP2G7X9rWNaIGDBrUK2tDkmzJC2S9HNJN0l6kaQtJC2WtKR83bycK0mfl7RU0vWS9u7IZ145f4mkeaPfsWJ7U9ublK+bAs8CPg7cAfzjaNfV6Z0XcDpwk+3PdP8RRMRUMjTEqYdLJp8KXGD7ecCewE3AicDFtmcDF5fPUK1PP7ts84HTACRtAZwM7AfsC5w8FHi7KcssfwT4KbAJ8Ie2PzTa+XVqoi8BjgFeJem6sh1WpzARMQVYrFozvdbWjaTNgJdTVdyw/bjtB4C5wBnltDOAI8r+XOBMVy4HZknaDjgYWGx7ue0VwGJgzMlrJW0j6RTgGqpRSX9g+2+6LbVcp3f+hzCFxy9ExJjGObP9VpKu6vi8oPSnDNkFuBf4iqQ9gauB9wDblpFCAHcB25b97YHbOq6/vaSNlj6WW4D7qQL4o8D8zom3bY/YpM8bSxHR2Dia6vfZnjPG8fWAvYF3275C0qmsbboD1VrzktrovD6FtRXGjetelCAaEY30+LXP24HbbQ8No1xEFUTvlrSd7TtLc/2ecnwZsGPH9TuUtGXAK4elf3+sG9v+2EQKnCAaEY31KojavkvSbZJ2s30zcCDViz03AvOAT5Wv55ZLzgPeJelsqk6kB0ugvRD4u47OpIOAk8a6t6RtgLcDO9MRG20fO9Z1CaIR0UgL40TfDXxD0vpUzymPpeoEP0fSccCtwJHl3POBw4ClwCPlXGwvl/Qx4Mpy3ke7dRBRBebLgAuBNXULmyAaEY318t1529cBIz03PXCEcw2cMEo+C4GF47j1TNsndj/tyabuu1oR0Rvu+TjRdeVCSUdonGthpyba4eBn7dVq/tM2aTHzFtdYP3S3l7WW95qH724t77bXnW/VypWtZGuv7n2eDMx8ou8EPgQ8IenxkibbY/7mJohGRGODEETLq57jliAaEY0YsXpN/z8ZlPSKkdJtXzbWdQmiEdHYgEzK/IGO/Q2APwSu58njTZ8iQTQiGrEHpjn/us7PZdzoglFO/50E0YhozAMQREfwAPCCbicliEZEQ30xfKkrSZew9t35acDvAV/rdl2CaEQ0NiA10Q927G9A9arokm4XdQ2ikmYCPyiZrgcssn3yBAsZEQNmUMaJ2r5mWNKPJF0BfHOs6+rURB8DXmX74bJMyA8lfa9MgBoRU537+72GIcOGOE0Dfh+Y2e26OpMyG3i4fJxRtgH4kUVErwzgEKdVwK9YO4P+qGo9E5U0nWqG6V2BL3TM9RcRU9ygDLYfPsSprlrfue3Vtveimth0X0lP6faXNF/SVZKueoLHJlKWiOhTdr1tMpP0RxO5blx/PsqCUZcywoJPthfYnmN7zgw2mEhZIqJP2aq1TXL7TOSiOksmby1pVtnfEHgN8POJ3CwiBk9Vy+z/IGr7IxO5rs4z0e2AM8pz0WnAOba/O5GbRcRgGoQhThNVp3f+euCFT0NZIqJPTfbnnW3KG0sR0dhkb6q3qf/HJUTEOmXqPQ+tG2gl/UrSTyVdJ+mqkraFpMWSlpSvm5d0Sfq8pKWSrpe0d0c+88r5SyTNq3nvP5b0mbK9vs41CaIR0ZhrbuNwgO29bA8tWHcicLHt2cDF5TPAocDsss0HToMq6AInUy2jvC9wcsfyySOSdErJ4/qyHS/p090KmiAaEc08Pb3zc4Ezyv4ZrH2TaC5wpiuXA7MkbQccDCy2vdz2CmAxIwzNHOYw4LW2v1q2g2tckyAaEc15jWptdbMDLpJ0taT5JW1b23eW/buAbcv+9sBtHdfeXtJGSx/LGmDLoQ9lUuau68+nYykiGhtH7/xWQ885iwW2h88e/1Lby0oQWyzpSePSbVtSG+MBPgb8WNJ/UQXyV/Lk6fFGlCD6NFrzm9+0l/n4lsoelzUPP9z9pImaymNjBoQZV+/8fR3POUfOz15Wvt4j6TtUzzTvlrSd7TtLc/2ecvoyYMeOy3coact48tpIOwDf73LfRZJ+UO4H8Je2u67pneZ8RDRjwKq3dSFpY0mbDO1TTYz8M+A8YKiHfR5wbtk/D3hr6aXfH3iwNPsvBA6StHnpUDqopI39rdj3lJeJFgMvknRWt2tSE42IxnrYoNgW+I6qltV6wDdtXyDpSuAcSccBtwJHlvPPp+oQWgo8AhxblcfLJX0MuLKc91Hby8e6saT1qILtkVS12O8DZ3YrcIJoRDTXoyBq+xZgzxHS7wcOHCHdwAmj5LUQWDiO299FtYrHV4C3236izkUJohHR0OSfXKSmf6eaYOlRqnH836sTSPNMNCKaa2G0/dPN9p9TrfB5FvAnwFJJX+l2XWqiEdGMB+fdedurgO8C35W0AfDabtckiEZEc5O8llmHqt6sP6d628nARcDp3a5Lcz4imuvREKd17ONUPf1folrpc2vgU90uql0TLZMyXwUss334BAsZEYNoAGqiwB8Be9teJelR239X1p0f03ia8+8BbgI2nWgJI2IADQ22738qz0SrD9L60H3BuFrNeUk7UD1g/fKEixcRA2sQVvsE7pE0u+xvCvw/4AvdLqpbE/0c8JfAJqOdUGZbmQ8wk41qZhsRA2HyB8g6jgBWl/3jgSW2b+12UdcgKulw4B7bV0t65WjnlZlYFgBsqi0G40caEfUMRnN+b4DyyukTwM6SdrZ92VgX1amJvgR4naTDgJnAppK+bvstDQscEQOilYnpnn4f6NjfmGo2p2uBl491UZ3VPk8CTgIoNdEPJoBGxO/0wdtIddh+XednSc8G/qnbdRlsHxEN9cUY0HGz/WtJu0mabnv1aOeNK4ja/j5dJjaNiCmo6yIa/cn287qdk5poRDQ3AM15SQ8B4snfzdBn2R5xdFKCaEQ0MyCD7W1P6EWiBNGIaGwQeuclbQm8GXgQ+AbVn4eZtn871nWZgCQimhuA+USB/6CaT/QQ4LPAhqxdy2lUqYlGRFQ2tv0eSdOAa20/LGlWt4sSRAdFiy8mX3jHda3l/dp9Dmkt77Zf1l7zwIOt5a1nbNxOviumt5Pv5K9l1nGVpANsXyppTWnez+h2UZrzEdFcj+cTlTRd0rWSvls+7yLpCklLJX2rzLCEpA3K56Xl+M4deZxU0m+WdHCN2+4PXCzpl8BzgcuBD3e7KEE0Ipqp+zx0fLXVoak3h5wCfNb2rsAK4LiSfhywoqR/tpyHpN2Bo4A9qJ5xfrHMiTyWQ4GdgVcAu9iebfu8bgVNEI2I5noYRIdPvVmW7XgVsKiccgbVjEsAc8tnyvEDy/lzgbNtP2b7l1Tr0u87yv12krQT1ZjQoW3DjvQx5ZloRDSm+m8sbSXpqo7PC8oMcJ0+x5On3twSeKBjwuTbge3L/vbAbVAtMifpwXL+9lTNcUa4Zrj/oAqc6wG7Ab8u6c8GbgaeP9Y3lCAaEc3Vb6rfZ3vOaAfrTr3ZS7b/oNz7TOBttq8on/cH3tXt+gTRiGhE7mnv/FOm3gROBWZJWq/URncAlpXzlwE7ArdLWg/YDLi/I31I5zWjmTMUQAFsXy5pYbcC55loRDTXo9552yfZ3sH2zlQdQ5fYfjNwKfDGcto81g6CP698phy/xLZL+lGl934XYDbw4y63v0HSv0g6oGynAz/rVubURCOiufbHif4VcLakj1NNlDy0HvzpwNckLQWWUwVebN8g6RzgRmAVcMJY09kVxwDvoGrCC/gh8MVuBasVRCX9CvgN1fojq8Z6phERU08bg+07p960fQsj9K7bXgn8ySjXfwL4xDjut5KqU+tz4ynneGqiB9i+bzyZR8QU0cdvLEm6harmOSLbu4x1fZrzEdFMbzuW1oWhlvWHqAbyf4sqqB4FbN7t4rpB1MBFkgx8aYRxXVkyOWIq6+Mgans5gKTDbO/ZcehTkq6neh47qrpB9KW2l0naBlgs6ee2fzCsIFkyOWKKGsdg+8lspaS3AGeXz0cDj3S7qNYQJ9vLytd7gO8wyutTERF97GiqTqo7gDuBI4E3dbuoa01U0sbANNu/KfsHAR9tVtaIGCgD0PYsIwDmjve6Os35bYHvVO/0sx7wTdsXjPdGETGg+r9jCQBJJ4+UbvtvJR1v+0sjHe8aREt03rPbeRExhQ1AEKUaCz+aUddZyhCniGhuAIKo7c+Mcezrox1LEI2IRsTANOcvYYRB97YPGOu6BNGIaG4AgijwwY79DYDXU73qPqYE0YhoZkA6lmxfMyzpR5KuGPHkDgmiEdHcAATRsrrnkOnAPlTzk44pQTS6OvhZe7WW93rbtZY1q+66u73M27ZyZSvZdp8NbmIG5I2lKzv2VwG3snZBvFEliEZEcwNQE7X9nIlclyAaEc2MfznkSUXSM6jekZ8JvJNqiRIokzLbfnSs67M8SEQ0NrTOUrdtkroUmAF8FXgOaydm3rWkjSk10YhobvIGyDpm2H5M0u62X9CRfpmkrmsspSYaEY31eU0USc8Bfi5p9460PYD/6XZtgmhENOeaWxeSZkr6saSfSLpB0t+W9F0kXSFpqaRvSVq/pG9QPi8tx3fuyOukkn6zpIPHuO1fAoupmvLXSrpa0lVUC+I9q1uZ05yPiGZ627H0GPAq2w9LmgH8UNL3gPcDn7V9tqR/php6dFr5usL2rpKOAk4B/rTUKI8C9qAKhP9X0nNHWvHT9kWSdgN2osa40OFSE42IRjSOrRtXHi4fZ5TNwKuARSX9DOCIsj+3fKYcP1DVvJ1zgbNtP2b7l8BSxphM3vYq27+wfc3wrVuZawVRSbMkLZL0c0k3SXpRnesiYoqo35zfStJVHdv84VlJmi7pOuAeqmb2L4AHbK8qp9wObF/2twdugyoQAg8CW3amj3BNT9Vtzp8KXGD7jeVZRFaii4jfGccbS/fZnjPWCaXJvZekWVTLET2vUeFa1rUmKmkz4OXA6QC2H7f9QMvlioh+0qOOpSdlWcWZS4EXAbMkDVX6dgCWlf1lwI4A5fhmwP2d6SNcU0upEY/VIQXUa87vAtwLfEXStZK+XNZaGn7D+UNV9Cd4bDxljYh+VnN4U50hTpK2LjVQJG0IvAa4iSqYvrGcNg84t+yfVz5Tjl9i2yX9qNJ7vwswG/hxjftPk3RA6bz6BTXena8TRNcD9gZOs/1CqmnyTxx+ku0FtufYnjODDWpkGxEDo3c10e2AS8t671cCi21/l2rt9/dLWkr1zPP0cv7pwJYl/f2U2GT7BuAc4EbgAuCEkXrmO0n6J6rA+S7gEuD5to/sVuA6z0RvB263PTSv3iJGCKIRMXX1aiC97euBF46Qfgsj9K7bXkm1zPFIeX0C+MQ4bj+PKuD+C1WNttaUV11rorbvAm4r46gADqSK7hERlRaeia4D21DVXo8HbpH0RUmv6HZR3d75dwPfKD3ztwDHTriYETFwJvMrnXWVWu0iYFHp93kd1ZIhl411Xa0gavs6YMxhCRExRfVHLbMWSc9k7WOD79s+q9s1eWMpIpobgOa8pD8FfgS8gWqRusslHd3turw7HxGNiIFZHuR/A/vYXg4gaQvg+8CYtdEE0YhoTJ7k1cz6HuzYf6DOBQmiEdFMHzTVa/oucIGkoZrnm4H/7HZRgmhENDYgvfN/Lelw4JUl6VTb53W7LkE01qlVd97VWt7rPXPb1vIGWH3f/a3l7VWrup80mQxAEAUob0d9d+izpE/aPmmsa9I7HxGN9fvyIFAFTEn3S/pN2R4CPlT2Rw2kqYlGRHOTPEDWNBfYpvN1T0nX2N57rIsSRCOimT6oZdZ09Qjvy9/Q7aIE0YhobgCCqO1jJD2fan4QAxfbPqbbdXkmGhGNiIF5Jnok8O9U0/GdBPy9pDd3uy410YhoTGsmeYSs5yTgpbbvlXQo8MfAfwPfGOuiBNGIaGZwBttPs31v2Zft1WXZ5jEliEZEYwPy7vzjkja3vQKYKekLwBXdLqqzUN1ukq7r2B6S9N4eFDgiBsUAzOIEnABsUvbPoprR6Z3dLupaE7V9M7AXVKvfUa2Y952JljIiBs9k7zSqaSXVGnIAnweeQ9VvNuZ3N97e+QOBX9i+ddzFi4jBZMCut3UhaUdJl0q6UdINkt5T0reQtFjSkvJ185IuSZ+XtFTS9ZL27shrXjl/iaR5o92zw1eomvQbUS2S9ynWLog3qvEG0aMYZW69LJkcMXX1cIjTKuADtncH9gdOkLQ71eKYF9ueDVzM2sUyD6VaDnk2MB84DX43F+jJwH5UM9WfPBR4xzDN9m+Ag4HzbR8E7NOtwLWDaFlf6XXAv450PEsmR0xhPXomavtO29eU/d9QrTm/PdUrmWeU084Ajij7c4EzXbkcmCVpO6pAuNj28tJRtBg4pNvtJe0BHAOc3/GdjWk8NdFDgWts3z2OayJiwI1zsP1WQy3Wss0fNV9pZ6rlk68AtrV9Zzl0FzA0Rdf2wG0dl91e0kZLH8uJwNeAFcBFkjYFFnb7/sczxOloukyTHxFTkD2ewfb32e666KWkZwDfBt5r+yFJHbezpd53Zdm+CLioI+kh4NRu19WqiZblQ18D/NuEShcRg62HQ5zKAPdvA9+wPRRz7i7NdMrXe0r6MmDHjst3KGmjpfdcrSBq+7e2t7T9YPezI2Kq6VXHkqoq5+nATbY/03HoPGCoh30ecG5H+ltLL/3+wIOl2X8hcJCkzUuH0kElrefyxlJENGOgd+/Ov4SqY+enkq4raX9NNdzoHEnHAbcCR5Zj5wOHAUuBR4BjAWwvl/QxqqFKAB8dWsWz1xJEI6K5HsVQ2z+k6qsayYEjnG+qN41GymshNTqGmkoQjYjGBuSNpQlJEI2I5gZn3flxSxCNiMZSE42ImKj+mKGpNQmiMbBW3dXuy3XTZs5sLe9+Wne+emNp6kbRBNGIaEyrE0QjIiYmzfmIiCbqzRU6qBJEI6Kx9M5HRDSRmmhExAR5YFb7nJAE0YhobgrXROvOJ/q+smjUzySdJam9AXIR0X8GY8nkCamz7vz2wF8Ac2y/AJhOtWBdRARQDbavsw2ius359YANJT0BbATc0V6RIqKvGJjCg+271kRtLwP+Afg1cCfVzNEXDT8vSyZHTE2iXi10UGuidZrzm1MtS7oL8CxgY0lvGX5elkyOmMLsetsAqtOx9Grgl7bvtf0E1WJ1L263WBHRV6ZwEK3zTPTXwP6SNgIepZqi/6pWSxUR/cPAFB4nWueZ6BXAIuAa4KflmgUtlysi+kivnolKWijpHkk/60jbQtJiSUvK181LuiR9XtJSSddL2rvjmnnl/CWS5o10r16pu2TyybafZ/sFto+xnZ6jiFird835rwKHDEs7EbjY9mzg4vIZ4FBgdtnmA6dBFXSBk4H9gH2Bk4cCbxtqBdGIiNHVDKA1gqjtHwDDlzaeC5xR9s8AjuhIP9OVy4FZkrYDDgYW215uewWwmKcG5p7Ja58R0YwZT6fRVpI6+1QW2O72eHBb23eW/buAbcv+9sBtHefdXtJGS29FgmhENFe/Y+k+23MmehvblibXxHtpzkdEY1qzptY2QXeXZjrl6z0lfRmwY8d5O5S00dJbkSAaEc0YWON628ScBwz1sM8Dzu1If2vppd+f6m3KO4ELgYMkbV46lA4qaa1Icz4iGurdQHpJZwGvpHp2ejtVL/ungHMkHQfcChxZTj8fOAxYCjwCHAtge7mkjwFXlvM+ant4Z1XPJIhGTNCalStby3v6Vlu2kq9WTG8l314FUdtHj3LowBHONXDCKPksBBb2pFBdJIhGRHMD+kpnHQmiEdHM0DPRKSpBNCIaMnjqvjyfIBoRzaU5HxExQWnOR0Q0lJpoRMREGSb+NlLfq7tk8nvKcsk3SHpvy2WKiH5iqiBaZxtAddZYegHwdqp5+fYEDpe0a9sFi4g+MoWXB6lTE30+cIXtR2yvAi4DXt9usSKirySIjulnwMskbVnWWTqMJ8+QEhFTWs3JRwa0B79rx5LtmySdAlwE/Ba4Dlg9/DxJ86mm6GcmG/W2lBExeRk8hQfb111j6XTb+9h+ObAC+J8Rzsm68xFTVWqiY5O0je17JD2b6nno/u0WKyL6yoA+76yj7jjRb0vaEngCOMH2A+0VKSL6iqf2ONFaQdT2y9ouSET0L69+SjfJlJE3liKiocEdvlRHgmhENJMJSCIiGsoQp4iIiTHgNa611SHpEEk3S1oq6cR2S99cgmhENOMys32drQtJ04EvAIcCuwNHS9q95e+gkQTRiGishzXRfYGltm+x/ThwNjC31cI3JLfQqybpXqr1oevYCriv54VoP++280/eg5N32/mPJ++dbG/dy5tLuqCUoY6ZQOda0wtsL+jI643AIbb/vHw+BtjP9rt6Vd5ea6VjaTz/SJKusj2njXK0mXfb+Sfvwcm77fzbLns3tg9ZV/eeDNKcj4jJZBlPniVuh5I2aSWIRsRkciUwW9IuktYHjgLOW8dlGtNkGCe6oPspkzLvtvNP3oOTd9v5t132p43tVZLeBVwITAcW2r5hHRdrTK10LEVETBVpzkdENJAgGhHRQILoFCVJ67oM4yFp4xbzfma//Txi8lhnQVTSbpJeJGlGedWrjXv0PF9Ju0qaI6mVNVAk7SHpFWUS7F7n/dIyeBnb7mXgkPRHkt7Tq/yG5T0XOEXSNi3kfTDwHVpYfFHS/pKOKV/XbyH/2eX/4rS2foeiu3USRCW9HjgX+DhwOnCCpE17mP9zAWyv7uV/LkmHA/8GfBr46tB9epj/ocBZwPuAMyU9s0f5TpP0DOBLwEmS3gG/C6SN/w9IOgj4GHBj07xGyPsVwCnAubbv6XHeB5W8twM+0OO8X0fVa/5q4IPATj3O/whgEXAS8Bng+DZr6zG6pz2ISpoB/ClwnO0DqYLpjsBf9SKQlkB3naRvQu8CqaQXUwXPebYPoFqwr2czzEh6JXAq8Oe2jwAeB17Qi7xtr7H9MHAG1R+tF0t639CxJnmXn8vXgPm2F0vaTNJOZXntXtgH+HLJ+1mSXiNpP0mbNclU0quBLwJvBmYDz5f08h6Ul9KKOAF4k+15wEPAXpK2kTSzR/kfDxxt+w3A9cCxwPslbdI0/xifddWc35TqPy5UTanvAjOANzVpYpa/xO8C3gs8Lunr0NMa6Sm2ry37JwNb9LBZfzdwvO0flxrofsC7JH1J0ht71PReRfUH6wxgX0mfkfRJVSb6f+F+qrW3tiu/3P8OnEZVU+9FuVd17C8C/ozq3/gLkjZvkO904K1lDOLGwM3AHtCT58WrgA2B55WKwSuBtwKfA/6mBzXGVcAzgGcC2F4I/Irq/fXDG+Yd42X7ad+A11C9hfCy8nk68Cbg65Sxqw3yfhbVf7CtqH7pvt6jMk8HNu3Y3wG4Fti6pG3Zw5/P/wb+puy/jWomm617kO/vASeW/Q8AjwBf6EG+ewK3ALcDb6f64/xnVI8mtmiY9+9TBbizgWNL2nOAfwYO7kHZp5WvhwB3Ab/fo3/DNwJXA5cDHy5prwK+CuzZg/zfUX5fjgE+UfaPB07v1f/DbPW2dVUT/S/gIuAYSS+3vdr2N6kC4J5NMrZ9h+2Hbd9H9Z9qw6EaqaS9JT1vgvmutv1Q+SjgAWC57XslvRn4uKQNm5S9416fsP3xsv9Vqpp7Lzo+HgV2k/R2ql/CTwHPlnR8k0xt/4SqBvQp2//i6vHBQmBz4NkN8/4p1TPF/YBdStotVH/IGs9G5PI4w/YFVM8wD29YMx/KdxHV89D/ovpji+1LgE3ozfPRs4DvAQcAG9p+i+0vAdv2sn8hulsnr33aXinpG1STYp9UAttjwLbAnT28z/0lQHxa0s+pfvEO6EG+q4CHJd0m6ZPAQcDbbD/aNG9JcqlqlM9voPq53NE0b9t3SLoN+DDV0tf/IekAYGkP8r6Rjo6lUu6t6c2/5/eoHp98RNLQFIsvpPoj0Es/oerU+3vbjZevtL1C0iXAkZIep5oGbheqZ5hN834Q+Iaks4b+EEh6K7AFMHWX3lwX1mU1GFifKqidTdXMeWFL93kfvW2qqZT9F8CvgdktlHkD4DjgBuAFPcx3R2Cfjs/TelxuUTXlbwT26HHeewN/B/xjr/4tR7jHOcDOPcxvFvAXwGVU74Pv2VK5h37mrfxcso2+TYp350unj92wp3iUvDen+sX4gO3GNYBheb8NuNItTJBQRjG8BviF7ZtbyP9JNd5e5gu8ArjL9s97nX9b2vp5dOS/CdXz/oe6njyx/HcCZthu3KqI8ZkUQbRtkmbaXtn9zHHn2+ovXkRMflMiiEZEtCXvzkdENJAgGhHRQIJoREQDCaIREQ0kiEZENJAgGhHRwP8HhS6CijlLr+YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAEYCAYAAADlIcXmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkRUlEQVR4nO3de7hdVX3u8e+bEBMuco0gJCgoOVZAUUwBL1UQ5WKp4VhUUBGQCj7Fioq1oPZgVaq2VaHnoDUVBJSClGpBiwLFKxaQEBRFRCMKJAYh3BGBZOc9f8y5ZSXsvdbca86Z7LX2+3me+ew1L2OssVayf3uMOcYcQ7aJiIj+TFvfBYiIGGQJohERNSSIRkTUkCAaEVFDgmhERA0JohERNSSIxpgkbSjpq5Lul/Tvko6UdGXH+YckPWN9ljFiMkgQHQKS3iBpURnYlkv6uqSX1Mz2EGAbYCvbr137pO1NbN/So1w7SLKkDWqWJWLSShAdcJLeDZwK/D1F0Hsa8GlgQc2snw783PaqmvlEDLUE0QEmaTPgQ8Bxtr9s+3e2V9r+qu2/ljRT0qmSflNup0qaWabdW9JSSSdIurOswR5Vnvs74P8Ary9rt0eP8d6WtFP5ekNJn5B0a9n8v1LShsB3y8vvK/N5oaSdJH2nvG6FpC+ti+8qoi1pZg22FwKzgK+Mc/79wF7A8wADFwEfAP62PP9UYDNgDvBK4EJJ/2n7ZEkGdrL9JgBJR3Ypxz8BuwAvAu4A9gRWAy8FfgVsPlqjlXQecBmwD/AkYP5EP3TEZJKa6GDbCljRpcn9RuBDtu+0fRfwd8DhHedXludX2r4EeAh41kQKIGka8BbgeNvLbI/Y/h/bj46TZCXFrYLtbD9i+8pxrosYCAmig+1uYHaXjpvtgFs79m8tj/0h/VoB+GFgkwmWYTZFbfiXFa9/LyDgB5JulPSWCb5fxKSSIDrYrgIeBQ4e5/xvKGp9o55WHmvSCuAR4JljnHvCFGG277D9VtvbAccCnx69txoxiBJEB5jt+yk6gE6XdLCkjSTNkHSgpH8AzgM+IOkpkmaX136x4TKsBs4EPilpO0nTyw6kmcBdFPdG/zCeVNJrJc0td++lCLSrmyxTxLqUjqUBZ/sTku6g6DA6F3gQuA44BVgMbArcUF7+78BHWijGe4CPAtdS3A74EbC/7YclnQJ8X9IM4ADgj4FTy5EFv6W4l9p1vGnEZKZMyhwR0b805yMiakgQjYioIUE0IqKGBNGIiBpa6Z1/kmZ6Fhu3kTWa3l7c90hG2sRwe4Tf8ZgfVZN57r/Pxr77npFK1153w6OX2j6gyfdf31oJorPYmD21bxtZM32TTVvJF2DkgQdayztiMrjGVzSe5933jPCDS59W6drp2/5iduMFWM8yTjQiaimelpi6rbgE0YioyYw4QTQioi9FTXTqPrSTIBoRtRiz0tU6loZRgmhE1JaaaEREnwyMJIhGRPRvKtdEK41cl3SApJslLZF0YtuFiojBYWDErrQNo55BVNJ04HTgQGBn4DBJO7ddsIgYHKsrbsOoSk10D2CJ7VtsPwacT/01zSNiSBgzUnEbRlXuic4Bbu/YX0qxJO4aJB0DHAMwi40aKVxEDADDyHDGx0oa61iyvRBYCLCptpzCX2nE1DLVF8mqEkSXAdt37M8tj0VEYMRKNzox1ECpEkSvBeZJ2pEieB4KvKHVUkXEQBkhQXRctldJejtwKTAdONP2ja2XLCIGQjHYPkG0K9uXAJe0XJaIGFCr05yPiOhPaqIRETUYMTKFl2tLEI2I2tKcj4jo01Rvzk/dOnhENESMeFqlrWdO0pmS7pT0kzHOnSDJkmaX+5L0z+XESDdI2r3j2iMk/aLcjmj0465l4Gqiba7IOX2rLVvLG2Dk7ntazb8tl/7mh63lvf92z2st7+mbtrcyLMDIQ79rLe8N5mzbSr66Y0bjeRZPLDVWHzsL+H/AOZ0HJW0P7Afc1nH4QGBeue0JfAbYU9KWwMnA/LJ410m62Pa9TRWyU2qiEVGLLR7z9Epb77z8XWCs2sangPfCGrOYLADOceFqYHNJ2wL7A5fbvqcMnJcDra11P3A10YiYfFZXvyc6W9Kijv2F5bwb45K0AFhm+0fSGu8z1uRIc7ocb0WCaETUUnQsVW7UrrA9v+rFkjYC3kfRlJ+U0pyPiJqa61gawzOBHYEfSfo1xQRIiyU9lfEnR1qnkyYliEZELaMdS1W2Cedt/9j21rZ3sL0DRdN8d9t3ABcDby576fcC7re9nGKej/0kbSFpC4pa7KVNfd61pTkfEbWNNDTYXtJ5wN4U906XAifbPmOcyy8BXgUsAR4GjgKwfY+kD1PMQAfwIdutDY1JEI2IWpp87NP2YT3O79Dx2sBx41x3JnBmI4XqIUE0Impb3d/9zqFQZbXPcZ8giIgY7Z2vsg2jKp/qLFocqBoRg61YHmR6pW0YVZnZ/ruSdlgHZYmIAWTT7/ClodDYPdEsmRwxVWkiTywNnSyZHBG1mNREIyJqGdZOoyoSRCOiFqMpPbN9lSFO5wFXAc+StFTS0e0XKyIGyVQe4lSld77rEwQRMbWZqT3YPs35iKhJU3qNpQTRiKglNdGIiBpsDe3TSFUkiEZEbRknGhHRp2JS5twTDdpf0lgbtPd1e9Wq1vL+0z0Pai3vabNWtJb3yuc/s7W8AWbceFvvi/o0svyOVvL1qpUt5KrURCMi+lV0LKUmGhHRt2EdSF9FgmhE1DLVH/tMEI2I2vpZyXNYTN1PHhGNKCZlVqWtl7GWI5L0j5J+JukGSV+RtHnHuZMkLZF0s6T9O44fUB5bIunEpj9zpwTRiKhttVVpq+Asnrgc0eXArrafC/wcOAlA0s7AocAuZZpPS5ouaTpwOnAgsDNwWHltK9Kcj4haRtdYaiSvMZYjsn1Zx+7VwCHl6wXA+bYfBX4laQmwR3luie1bACSdX17700YKuZYE0YioZYJDnGZLWtSxv7BcFaOqtwBfKl/PoQiqo5aWxwBuX+v4nhN4jwnpGUQlbQ+cA2xD8X0ttH1aWwWKiEGjiUxAssL2/L7eRXo/sAo4t5/0balSE10FnGB7saQnA9dJutx2K1XjiBg8bT/2KelI4CBgX9uja7gtA7bvuGxueYwuxxvX88+H7eW2F5evHwRu4vEqc0RMcU32zo9F0gHAe4FX236449TFwKGSZkraEZgH/AC4FpgnaUdJT6LofLq41ofsYkL3RMsbvs8HrhnjXJZMjpiimppPtFyOaG+Ke6dLgZMpeuNnApdLArja9tts3yjpAooOo1XAcbZHynzeDlwKTAfOtH1jIwUcQ+UgKmkT4D+Ad9p+YO3zWTI5Ympq8omlcZYjOqPL9acAp4xx/BLgkkYK1UOlICppBkUAPdf2l9stUkQMmkyF14WK+vMZwE22P9l+kSJikGQWp95eDBwO/FjSD8tj7yuryxEx1VmsWp3lQcZl+0qYwnX1iOhqWGa2l/QARazr7NMRRZycaY/de5YnliKitmFoztvetHO/7Ew/jmLU0bh9QQmiEVHLsN0TlbQFcDxwBMXTUX9se9y1gxJEI6K2YQiikrYGTgBeR9GZ/tzyAaOuEkQjopYhmtn+FuBuigD6e+CYcnA/ALY/MVaiBNGIqG0YOpaAj/N4J/rGVRMliEZEPR6O5rztD/eTLkF0HWpzbfg2rbp96fouQl+mf//HreY/MjLSXuZu6cnpFrIdlo6l8p7oW4Ed6IiNto/qli5BNCJqG4YgClwEfIdi4pLVVRMliEZELUaMrB6K5dpm2Z7wonZD8ckjYv1ajSptk9ylkg5WZ5d8BamJRkQtHpKOJeAvgb8GVkp6rDwm20/ulihBNCJq8xAE0bUf+6wqQTQiahqOwfaSXjbWcdvf6ZYuQTQiahuGmijFI5+jZgJ/DNxAsVzJuKpMyjwL+G6Z6QbAhbZP7ruYETFUhmWcqO1Xd+6X40YX9kpXpSb6KPBy2w+Vy4RcKenrtq/ur6gRMVTc3rMB69l9wK69LqoyKbOBh8rdGeU2nF9ZRPRlAIYv9STpmzz+7Pw04JnAF3qlq7pQ3XTgOmAn4HTbT1gyOSKmpiYH20s6EzgIuNP2ruWxLYEvUTyO+WvgdbbvLcdznga8CngYONL24jLNEcAHymw/YvvsCm//no7XM4H9gF/0SlTpk9sesf08YC6wh6QnVHElHSNpkaRFK3m0SrYRMSTsalsFZwEHrHXsROAK2/OAK8p9gAOBeeV2DPAZ+EPQPRnYE9gDOLmcaLnHZ/Diju0q239HMTlzVxPqnbd9n6RvUXzIn6x1LuvOR0xRTfXO2/6upB3WOryAx3vIzwa+DfxNefyc8pbj1ZI2l7Rtee3lo7PRS7qcImad1+291xriNA14DjCrV5mr9M4/BVhZBtANgVdSzLsXEVHWMisH0dmSFnXsLywrYN1sY3t5+foOYJvy9Rzg9o7rlpbHxjveS+cQp1UUtw4O7pWoSk10W+Ds8r7oNOAC21+rkC4ipogJDHFaYXt+v+9j25JaaemuPcSpqiq98zcAz+8n84iYGloe4vRbSdvaXl421+8sjy8Dtu+4bm55bBlrDpCfS3ELoCtJf2b7qxMtXGZxiojabFXa+nQxxcqblD8v6jj+ZhX2Au4vm/2XAvtJ2qLsUNqvPNbLC/opXB77jIhaTK0AuQZJ51HUImdLWkrRy/4x4AJJRwO3UqzGCXAJxfCmJRRDnI4CsH2PpA8D15bXfajbksd/+Bz2B/spc4JoRNTWVGve9mHjnNp3jGsNHDdOPmcCZzZUrK4SRCOinon1zg+dBNGIqM2rE0QjIvo2LBOQSNqfYiw8wH/b/kavNAmi0dvElpyZmBZ/+wZ1iepBY4ajOS/pvcCrgc+Xhz4gaTfbXR8uShCNiHoMDEEQpRg+Nd/27wEkfZGihz9BNCLaNSTN+UdHAyiA7UcljfRKlMH2EVGfK26T2yWdsz1J2hz4eq9EqYlGRE3NDbZfn2x/AEDSpuX+fcD7eqVLTTQi6huCmqik3SRdT7E43Q2SfiTpeb3SpSYaEfUMz2D7fwHebvv7AJJeQjHR8wu7JUoQjYj6Jnkts6INRwMogO0ryzmUu0oQjYj6hqMmeoukk3l8cbojgF/2SlT5nqik6ZKul5QJmSNiTUNwT5RiFqhNgQvK7cnAW3olmkhN9HjgpvJNIiIKQzLY3vb9rLlESCVVl0yeC/wpcArw7om+SUQMt2EYbL/WuvN/YHufbumq1kRPBd5LUb0drwDHUCxbyiw2qphtRAyFIQiiPHHd+dcAPZ9YqrLa50HAnbavk7T3eNdlyeSIKWw4mvOL1zp0laRreqWrUhN9MfBqSa+iWIN5U0lftP2mPsoZEUOonfU31y1JW3XsTqdYc2mzXumqrPZ5EnBS+SZ7A+9JAI2IPxiMnvcqrqW4J2qKdedvBY7ulSjjRCOiJg1Lc/4Z/aSbUBC1/W0qrN8cEVPM6vVdgPokPX2MwxcD+9u+Y7x0qYlGRH0NNuclvQv4izLXH1MMgt8WOB/YCrgOONz2Y5JmAudQ3L+8G3i97V/3+dZf5fHm/KhnAt+TtGi8lUgzi1NE1DM62L7K1oOkOcA7KGaY35Wig+dQitnlP2V7J+BeHr9XeTRwb3n8U/SYhb7rx7Cfa/s55c/n2n4ucLPtecCzx0uXIBoRtcnVtoo2ADaUtAGwEbAceDlwYXn+bODg8vWCcp/y/L5So4uCvaP8+fnxLkhzPiLqqx4gZ0ta1LG/sBxjXmRjL5P0T8BtwO+Byyia7/fZHl15cCkwp3w9B7i9TLtK0v0UTf4VE/0Iks5kjCeWJHXtoU8QjYh1aYXt+eOdLJfnWADsCNwH/DtwwLopGn1NrpQgGr0N6IPR0zfvOU66ltUP/a61vD1/53YyvuF/Wsm2wcH2rwB+ZfsuAElfpnjgZ3NJG5S10bnAsvL6ZcD2wNKy+b8ZRQfThNn+cj/pEkQjor7mxoneBuwlaSOK5vy+wCLgW8AhFD30RwAXlddfXO5fVZ7/pt3fX/1uE5BI+lfbbx0rXYJoRNTT4BNLtq+RdCGwmOKpoesp5uT4L+B8SR8pj51RJjkD+IKkJcA9FD35/XpPl3OfGO9EgmhE1NfgHR/bJwMnr3X4FmCPMa59BHhtQ++7WNJsYK/y0NW2V5TnfjZeugxxiojatLraNplJ2p9iJMChwOuBxZJ6dmqlJhoR9Q1m3+PaPgq81PatAJKeBnwF+Ea3RAmiEVHLBAfST2YaDaAAtm+T1LO1nuZ8RNTX0GOf69md5ThVACRtToVB+6mJRkR9Q1ATtb3/Wvv3SfqLXumqLlT3a+BBivVGVnV74iAipp5haM5Lmgf8GWuuJfc2Sf8CfNv2d8ZKN5Ga6D6j3f0REWsYgiBKMYHJV4AHOo6tAh4CHhsvUZrzEVHP8HQsjdj+YOcBSW+yPe5Ae6jesWTgMknXlUsjP4GkYyQtkrRoJY9WzDYihoIrbpPbX1U8toaqNdGXlFNUbQ1cLulntr/beUGWTI6Yuib7QPqK9pX0ijGOf1/SsbY/O1aiSjVR28vKn3dS3DN4wuNXERED7iGKDvS1N4Bxp+zqWROVtDEwzfaD5ev9gA/VLm5EDI8haHva/mSXc18c71yV5vw2wFfKGfc3AP7NdtfHoCJiChmSjqVuU+F1S9cziNq+Bdit/6JFxNAbgiDKmlPhzQReQzE2vqsMcYqI+oYgiNpevNahqyRd0ytdgmhE1CKGpjm/VcfudIq17HuuMZMgGhH1DUEQBa7teL0KuJXH17cfV4JoRNQzJB1Ltp/RT7oE0Yiob4CDqKRNgIeBWcBfUqwuCnAl8Gnbv++WPkE0hlabSxoDeKRnx23/rr6hnXy7x4O+DfgTS98CXgKcRTF/6Knl8UPLY6/vljhBNCLqa7AmWk6G/Dlg1zLntwA3A18CdgB+DbzO9r0qBrCfBryKojZ55Bi97L3MsP2opJ1t79px/DuSftIrcWa2j4h6qk4+Uj3QngZ8w/YfUYxRvwk4EbjC9jzginIf4EBgXrkdA3ymn48g6RnAzyTt3HFsF+DnvdImiEZEbaPrLPXaeuYjbQa8lHJdeduP2b4PWACcXV52NnBw+XoBcI4LVwObS9p2gsV/L3A58Azg+nK2ukUU69tv1ytxmvMRUV/1WubsMkCNWljOADdqR+Au4POSdqNYwvh4YBvby8tr7qB4HB1gDnB7R/ql5bHlVGT7MknPAp5OhXGha0sQjYjaJjDEaUWP5YU2AHYH/sr2NZJO4/GmOwC2LTU7qMr2KuCX/aRNcz4i6mvunuhSYKnt0cctL6QIqr8dbaaXP+8szy8Dtu9IP7c8ts4kiEZEPQ12LNm+A7i9bF4D7Av8FLgYOKI8dgRwUfn6YuDNKuwF3N/R7F8n0pyPiFrEGPPH1fNXwLmSngTcAhxFUeG7QNLRFI9jvq689hKK4U1LKIY4HdVUISRNB15h+9Ju11VdMnlz1hq3ZfuquoWMiCHR4B1K2z8Exrpvuu8Y1xo4rqn3ljQNeBnFAPsDgB8A9YMoj4/bOqT867BRnYJGxHAZ8CeWAJD0f4GDgMUUA/vf1euRT6i2PMjouK0joRi3RZc1mCNiChrgZ+c7HAF8A/hX4Ju2Kz3XW6VjqXPc1vWSPleutbSGLJkcMUVVHGg/ADM9bQ1cABwL3CLp05Je1itRlSA6Om7rM7afT7Hq3YlrX2R7oe35tufPYObEih4Rg20I1p23/YjtC20fAuwMfI81lwwZU5V7omON23pCEI2IqWsAapmVSHoqjy8J/23b5/VK07Mm2mXcVkREYQhqopJeD1wF/DnFInVXSzqsV7qqvfNjjduKiACGpib6fuAFtu8BkLQl8G2ga220UhDtMm4rIqa6AahlTsD9Ha/vq5IgTyxFRH3DEUS/BnxD0mjN843Af/VKlCAaEbWI4Rhsb/t9kg4C9i4PnWb74l7pEkQjojZ5OKqitr9GUSMFQNJHbZ/ULU1mcYqIeppfHmS9kPRRSXdLerDcHgD+unw9biBNTTQiahuS3vkFwNadj3tKWmx7926JEkRjaHnVqvVdhP6p4cnlRrUV7IYjiF43xvPyN/ZKlCAaEbUNQ03U9uGSnk3xQJEpVhc9vFe63BONiPqG457o64D/BLYFTgL+QdIbe6VLTTQi6hmMGZqqOAl4ie27JB0I/G/gf4BzuyVKEI2I+oYjiE6zfVf5WrZHJM3omajlQkXEkBNDM5/oY5K2KF/PknQ6cE23BJCaaEQ0QKsnf4Ss4DjgycC9FJOO3EKPpjwkiEZEXQPQaVTRIxSTzgP8M/AMiop210+X5nxE1KbV1bbK+UnTy+WIvlbu7yjpGklLJH2pnJYTSTPL/SXl+R1qfIzPUzTpNwKuBT4GnNErUc8gKulZkn7YsT0g6Z01ChoRw6b5IU7HAzd17H8c+JTtnSia20eXx48G7i2Pf6q8rl/TbD8I7A9cYns/4AU9E/W6wPbNtp9n+3llhg8DX6lR0IgYMk12LEmaC/wp8LlyX8DLKZYmAjgbOLh8vaDcpzy/b3l9PyxpF+Bw4JLRY70STbQ5vy/wS9u3TjBdRAwrA3a1DWaPrgpcbseMkeOpwHuB0RsAWwH32R59jncpMKd8PQe4HaA8f395fT9OBL5AUdO9TNKmwJm9Ek20Y+lQxpkqv/wyjgGYxUYTzDYiBtkEhi+tsD3uKhnlfJ532r5O0t71S1ad7cuAyzoOPQCc1itd5SBa3sh9NcWo/rEKsBBYCLCpthyOvrqIqKa53/gXA6+W9CpgFrApRSDbXNIGZW1zLrCsvH4ZsD2wVNIGwGbA3Y2VpoKJNOcPBBbb/m1bhYmIwdPkYHvbJ9mea3sHipbvN22/EfgWcEh52RHAReXri8t9yvPftNftDNETCaKH0WPVu4iYgmy0utpWw98A75a0hOKe5+jQozOArcrj76a4r7lOVWrOS9oYeCVwbLvFiYiB1ELdz/a3KZYsxvYtwB5jXPMI8Nrm3726qksm/47+e7wiYsgNwHPxrcljnxFRj4HheHa+LwmiEVHf1I2hCaIRUV+a8xERdQzJuvP9SBCNiNpSE42I6NfwzCfalwTRiMlogJrHxRNLg1PepiWIRkRtGkkQjYjoT5rzERF1eKBuPzQtQTQiakvvfEREHamJRkT0yRNbyXPYJIhGRH1TuCZaaVJmSe+SdKOkn0g6T9KstgsWEQOk+SWTB0aVdefnAO8A5tveFZhOMW1/RARQDLavsg2jqs35DYANJa0ENgJ+016RImKgGJjCg+171kRtLwP+CbgNWA7cXy4tugZJx4yuJb2SR5svaURMSqJaLXRYa6JVmvNbAAuAHYHtgI0lvWnt62wvtD3f9vwZzGy+pBExednVtiFUpWPpFcCvbN9leyXwZeBF7RYrIgZKQ0FU0vaSviXpp2Vn9vHl8S0lXS7pF+XPLcrjkvTPkpZIukHS7i1/0ieoEkRvA/aStJEkAfsCN7VbrIgYGAZWV9x6WwWcYHtnYC/gOEk7UyyFfIXtecAVPL408oHAvHI7BvhMI59pAqrcE70GuBBYDPy4TLOw5XJFxABp6p6o7eW2F5evH6SosM2huKV4dnnZ2cDB5esFwDkuXA1sLmnbhj9eV1WXTD4ZOLnlskTEoGrhfqekHYDnA9cA29heXp66A9imfD0HuL0j2dLy2HLWkTyxFBE1TajTaLakRR37C20/oWUraRPgP4B32n6guJNYvpttafJMeZIgGhH1mIkE0RW253e7QNIMigB6ru0vl4d/K2lb28vL5vqd5fFlwPYdyeeWx9aZSo99RkR01VDHUtl5fQZwk+1Pdpy6GDiifH0EcFHH8TeXvfR7UYxjX2dNeUhNNCIaoNWNTeP0YuBw4MeSflgeex/wMeACSUcDtwKvK89dArwKWAI8DBzVVEGqShCNiHoMrG7mFqXtKynWvhvLvmNcb+C4Rt68TwmiEVHT8D6NVEWCaETUlyAaEVFDgmhERJ8avCc6iBJEI6Img6fuIksJohFRX5rzERF9SnM+IqKm1EQjIvplaO6JpYFTdcnk48vlkm+U9M6WyxQRg8QUQbTKNoSqrLG0K/BWYA9gN+AgSTu1XbCIGCBZY6mrZwPX2H7Y9irgO8Br2i1WRAyUBNGufgL8iaStJG1EMWPK9j3SRMSU4aJ3vso2hHp2LNm+SdLHgcuA3wE/BEbWvk7SMRQLRTGLjZotZURMXgZP4cH2lTqWbJ9h+wW2XwrcC/x8jGuy7nzEVJWaaHeStrZ9p6SnUdwP3avdYkXEQBnS+51VVB0n+h+StgJWAsfZvq+9IkXEQPHUHidadcnkP2m7IBExuDzyhG6SKSNPLEVETcM7fKmKBNGIqCcTkERE1JQhThER/THg1a60VSHpAEk3S1oi6cR2S19fgmhE1ONyZvsqWw+SpgOnAwcCOwOHSdq55U9QS4JoRNTWYE10D2CJ7VtsPwacDyxotfA1tXJP9EHuXfHfvvDWipfPBla0UY6W8247/+Q9PHm3nf9E8n5602/+IPde+t+rL5hd8fJZkhZ17C+0vbBjfw5we8f+UmDPumVsUytB1PZTql4raZHt+W2Uo828284/eQ9P3m3n33bZe7F9wPp678kgzfmImEyWseYscXPLY5NWgmhETCbXAvMk7SjpScChwMXruUxdTYZxogt7XzIp8247/+Q9PHm3nX/bZV9nbK+S9HbgUmA6cKbtG9dzsbqSp/DjWhERdaU5HxFRQ4JoREQNCaJTlCSt7zJMhKSNW8z7qYP2fcTksd6CqKRnSXqhpBnlo15tvEfj+UraSdJ8Sa2sgSJpF0kvKyfBbjrvl0g6HMC2mwwckv5M0vFN5bdW3guAj0vauoW89we+QguLL0raS9Lh5c8ntZD/vPL/4rS2foeit/USRCW9BrgI+AhwBnCcpE0bzP9/AdgeafI/l6SDgC8D/wicNfo+DeZ/IHAe8C7gHElPbSjfaZI2AT4LnCTpbfCHQFr7/4Ck/YAPAz+tm9cYeb8M+Dhwke07G857vzLvbYETGs771RS95q8A3kPDTwpJOhi4EDgJ+CRwbJu19RjfOg+ikmYArweOtr0vRTDdHvibJgJpGeh+KOnfoLlAKulFFMHzCNv7UCzY19gMM5L2Bk4D/sL2wcBjwK5N5G17te2HgLMp/mi9SNK7Rs/Vybv8Xr4AHGP7ckmbSXp6ubx2E14AfK7MeztJr5S0p6TN6mQq6RXAp4E3AvOAZ0t6aQPlpWxFHAe8wfYRwAPA8yRtLWlWQ/kfCxxm+8+BG4CjgHdLenLd/GNi1ldzflOK/7hQNKW+BswA3lCniVn+JX478E7gMUlfhEZrpB+3fX35+mRgywab9b8FjrX9g7IGuifwdkmflXRIQ03vVRR/sM4G9pD0SUkfVaHf/wt3U6y9tW35y/2fwGcoaupNlHtVx+sLgbdQ/BufLmmLGvlOB95cjkHcGLgZ2AUauV+8CtgQ+KOyYrA38GbgVOADDdQYVwGbAE8FsH0m8GuKZ+gPqpl3TJTtdb4Br6R4CuFPyv3pwBuAL1KOXa2R93YU/8FmU/zSfbGhMk8HNu14PRe4HnhKeWyrBr+f9wMfKF8fSTGTzVMayPeZwInl6xOAh4HTG8h3N+AWiski3krxx/ktFLcmtqyZ93MoAtz5wFHlsWcA/wLs30DZp5U/DwDuAJ7T0L/hIcB1wNXA35bHXg6cBezWQP5vK39fDgdOKV8fC5zR1P/DbNW29VUT/R5wGXC4pJfaHrH9bxQBcLc6Gdv+je2HbK+g+E+14WiNVNLukv6oz3xHbD9Q7gq4D7jH9l2S3gh8RNKGdcre8V6n2P5I+fosipp7Ex0fvweeJemtFL+EHwOeJunYOpna/hFFDehjtv/Vxe2DM4EtgKfVzPvHFPcU9wR2LI/dQvGHrPJEN13yX13+/AbFPcyDatbMR/O9kOJ+6Pco/thi+5vAk2nm/uh5wNeBfYANbb/J9meBbZrsX4je1stjn7YfkXQuxaTYJ5WB7VFgG2B5g+9zdxkg/lHSzyh+8fZpIN9VwEOSbpf0UWA/4Ejbv6+btyS5rGqU+39O8b38pm7etn8j6XbgbymWvv6qpH2AJQ3k/VM6OpbKcj+FZv49v05x++SDkkanWHw+xR+BJv2IolPvH2zXXr7S9r2Svgm8TtJjwCyKPwQ3NJD3/cC5ks4b/UMg6c3AlsDUXXpzfVif1WDgSRRB7XyKZs7zW3qfd9FsU01l2X8J3AbMa6HMM4GjgRuBXRvMd3vgBR370xoutyia8j8Fdmk4792Bvwc+0dS/5RjvcQGwQ4P5bQ68A/gOxfPgu7VU7tHvvJXvJdv426R4dr7s9LFr9hSPk/cWFL8YJ9iuXQNYK+8jgWvdwgQJ5SiGVwK/tH1zC/mvUeNtMl/gZcAdtn/WdP5taev76Mj/yRT3+x/oeXF/+T8dmGG7dqsiJmZSBNG2SZpl+5EW8m31Fy8iJr8pEUQjItqSZ+cjImpIEI2IqCFBNCKihgTRiIgaEkQjImpIEI2IqOH/A8Fo8f16NmKdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from nplm.plf import executor, evaluator\n",
    "\n",
    "train_instances, _ = get_inst(train, keep_instance=True)\n",
    "\n",
    "plfs = [firstperson_rule, sectionTitle_rule, result_rule, \n",
    "              simple_re_rule_0,  res_rule, simple_re_rule_1, \n",
    "              add_rule, resultp_rule ,re_tb, \n",
    "              re_tm]\n",
    "votes, fid2clusters = executor(plfs, train_instances, one_indexed=False)\n",
    "\n",
    "votes, _, train_idx = votes_filter(votes, return_idx=True)\n",
    "plot_overlap_conflict(votes, fid2clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "epoch   1:   0%|          | 0/2 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup:  0.1452467441558838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch   1: 100%|██████████| 2/2 [00:00<00:00,  9.89it/s, Epoch Loss: =9.2] \n",
      "epoch   2: 100%|██████████| 2/2 [00:00<00:00, 10.29it/s, Epoch Loss: =9.08]\n",
      "epoch   3: 100%|██████████| 2/2 [00:00<00:00, 10.45it/s, Epoch Loss: =8.96]\n",
      "epoch   4: 100%|██████████| 2/2 [00:00<00:00, 10.53it/s, Epoch Loss: =8.85]\n",
      "epoch   5: 100%|██████████| 2/2 [00:00<00:00, 10.41it/s, Epoch Loss: =8.73]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup:  1.0310964584350586\n",
      "Parallel Estimation:  0.008614778518676758\n"
     ]
    }
   ],
   "source": [
    "from nplm import PartialLabelModel\n",
    "\n",
    "lm_cfg={'lr': 0.01,\n",
    " 'epoch': 5,\n",
    " 'seed': 0,\n",
    " 'batch_size': 8192,\n",
    " 'momentum': 0.9,\n",
    " 'step_schedule': 'p',\n",
    " 'step_multiplier': 0.1}\n",
    "\n",
    "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "labelmodel = PartialLabelModel(num_classes=3,\n",
    "                        fid2clusters=fid2clusters,\n",
    "                        opt_cfg=lm_cfg,\n",
    "                        device=device, verbose=True)\n",
    "\n",
    "labelmodel.optimize(votes)\n",
    "label_estim = labelmodel.weak_label(votes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('scicite_soft_labels', {'sl': label_estim, 'idx': train_idx})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: [0], 1: [1, 2]}\n",
      "{0: [0], 1: [1], 2: [0, 2]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [2], 1: [0, 1]}\n",
      "{0: [1], 1: [0, 2]}\n",
      "{0: [0, 2], 1: [1]}\n",
      "{0: [2], 1: [0, 1]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [0, 1], 1: [2]}\n",
      "{0: [0, 2], 1: [1]}\n",
      "Name\tAcc\tP\tR\n",
      "Rule 0 \t0.8304\t0.8304\t0.0999\n",
      "Rule 1 \t0.9326\t0.9348\t0.8259\n",
      "Rule 2 \t0.8651\t0.7221\t0.8651\n",
      "Rule 3 \t0.8140\t0.8140\t0.0188\n",
      "Rule 4 \t1.0000\t1.0000\t0.0113\n",
      "Rule 5 \t0.7684\t0.7418\t0.7684\n",
      "Rule 6 \t0.9139\t0.9139\t0.0742\n",
      "Rule 7 \t0.7460\t0.7460\t0.0505\n",
      "Rule 8 \t0.9245\t0.9245\t0.0263\n",
      "Rule 9 \t0.7143\t0.7143\t0.0322\n",
      "Setup:  0.264676570892334\n",
      "Parallel Estimation:  0.010888814926147461\n",
      "(0.7152068780225685, 0.6912292100997061, 0.7156869632478277, 0.6940463570460244)\n",
      "[0.71520688 0.9586244 ]\n"
     ]
    }
   ],
   "source": [
    "from eval import gen_stats, topk_results\n",
    "\n",
    "data_instance_test, data_label_test = get_inst(test, keep_instance=True)\n",
    "\n",
    "votes_test, _ = executor(plfs, data_instance_test, one_indexed=False)\n",
    "accs, ps, rs = evaluator(plfs, data_label_test, pprint=True)\n",
    "\n",
    "wlabels_test = labelmodel.weak_label(votes_test)\n",
    "\n",
    "test_label_test = [elem+1 for elem in data_label_test]\n",
    "\n",
    "_, stats = gen_stats(wlabels_test, test_label_test)\n",
    "print(stats)\n",
    "print(topk_results(wlabels_test, test_label_test, [1,2]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}