trec_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import json\n",
    "from copy import deepcopy as dc\n",
    "import torch\n",
    "\n",
    "import joblib\n",
    "import re\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from data.utils import votes_filter, plot_overlap_conflict\n",
    "from data.trec.utils import load_data, get_inst, get_subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = load_data('train')\n",
    "dev = load_data('dev')\n",
    "test = load_data('test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from annotators import WeakRule, BinaryRERules, regex_decision\n",
    "\n",
    "train_instances, _ = get_inst(train)\n",
    "dev_instances, dev_labels = get_inst(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 1.0 0.3213367609254499\n",
      "[nan, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
      "1.0 1.0 0.017994858611825194\n",
      "[nan, nan, 1.0, 1.0, 1.0, nan]\n",
      "0.8571428571428571 0.8571428571428571 0.015424164524421594\n",
      "[nan, 1.0, nan, 0.0, nan, nan]\n",
      "1.0 1.0 0.012853470437017995\n",
      "[1.0, nan, nan, nan, nan, nan]\n",
      "1.0 1.0 0.007712082262210797\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "0.8888888888888888 0.8888888888888888 0.02056555269922879\n",
      "[nan, nan, 1.0, nan, nan, 0.0]\n",
      "1.0 1.0 0.03598971722365039\n",
      "[nan, nan, nan, nan, 1.0, nan]\n",
      "1.0 1.0 0.010282776349614395\n",
      "[nan, nan, nan, nan, nan, 1.0]\n",
      "0.9629629629629629 0.9629629629629629 0.06683804627249357\n",
      "[nan, nan, nan, 0.0, nan, 1.0]\n",
      "1.0 1.0 0.017994858611825194\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "1.0 1.0 0.02056555269922879\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "0.96 0.96 0.061696658097686374\n",
      "[nan, nan, nan, nan, 0.0, 1.0]\n",
      "1.0 1.0 0.038560411311053984\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "1.0 1.0 0.005141388174807198\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "1.0 1.0 0.038560411311053984\n",
      "[nan, 1.0, nan, nan, nan, nan]\n",
      "1.0 1.0 0.012853470437017995\n",
      "[1.0, nan, nan, nan, nan, nan]\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "0 - ABBR - Abbreviation\n",
    "\n",
    "1 - DESC - Description and abstract concepts\n",
    "\n",
    "2 - ENTY - Entities\n",
    "\n",
    "3 - HUM - Human beings\n",
    "\n",
    "4 - LOC - Locations\n",
    "\n",
    "5 - NUM - Numeric values\n",
    "\n",
    "'''\n",
    "\n",
    "\n",
    "\n",
    "def first_word(instance):\n",
    "    st = instance['string']\n",
    "    word = st.split()[0].lower()\n",
    "    if word == \"who\":\n",
    "        return 0\n",
    "    elif word == \"where\":\n",
    "        return 1\n",
    "    elif word == \"when\":\n",
    "        return 2\n",
    "    elif word == \"why\":\n",
    "        return 3\n",
    "    elif word == \"how\":\n",
    "        return 4\n",
    "    elif word == \"name\":\n",
    "        return 5\n",
    "\n",
    "   # elif word == 'what':\n",
    "     #   return 6\n",
    "    else:\n",
    "        return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [3],\n",
    "    1: [4],\n",
    "    2: [5],\n",
    "    3: [1],\n",
    "    4: [1, 5],\n",
    "    5: [2, 3],\n",
    "    6: [0]\n",
    "}\n",
    "\n",
    "r0 = WeakRule(exec_module=first_word, label_maps=label_maps)\n",
    "v3 = r0.execute(dev_instances)\n",
    "acc, p, r, cwacc = r0.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "def called(instance):\n",
    "    st = instance['string'].lower().split()\n",
    "    if \"called\" in st:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 1, 5],\n",
    "    1: [2, 3, 4]\n",
    "}\n",
    "\n",
    "r1 = WeakRule(exec_module=called, label_maps=label_maps)\n",
    "v3 = r1.execute(dev_instances)\n",
    "acc, p, r, cwacc = r1.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "def mean(instance):\n",
    "    st = instance['string'].lower().split()\n",
    "    if \"mean\" in st or \"meaning\" in st:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [2, 3, 4, 5],\n",
    "    1: [0, 1]\n",
    "}\n",
    "\n",
    "r2 = WeakRule(exec_module=mean, label_maps=label_maps)\n",
    "v3 = r2.execute(dev_instances)\n",
    "acc, p, r, cwacc = r2.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "def abbre(instance):\n",
    "    st = instance['string'].lower()\n",
    "    if \"stand for\" in st or \"abbreviat\" in st:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [1, 2, 3, 4, 5],\n",
    "    1: [0]\n",
    "}\n",
    "\n",
    "r3 = WeakRule(exec_module=abbre, label_maps=label_maps)\n",
    "v3 = r3.execute(dev_instances)\n",
    "acc, p, r, cwacc = r3.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "# DESC\n",
    "def desc(instance):\n",
    "    st = instance['string'].lower()\n",
    "    tokens = st.split()\n",
    "    if \"definition\" in tokens or \\\n",
    "            \"come from\" in st or \\\n",
    "            \"origin\" in tokens:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 2, 3, 4, 5],\n",
    "    1: [1]\n",
    "}\n",
    "\n",
    "\n",
    "r4 = WeakRule(exec_module=desc, label_maps=label_maps)\n",
    "v3 = r4.execute(dev_instances)\n",
    "acc, p, r, cwacc = r4.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "# ENTY\n",
    "def enty(instance):\n",
    "    subject = get_subject(instance['string'])\n",
    "    if subject in (\"animal\", \"body\", \"color\", \"creative\",\n",
    "                   \"currency\", \"disease\", \"event\", \"food\", \"instrument\",\n",
    "                   \"language\", \"letter\", \"plant\", \"product\", \"religion\",\n",
    "                   \"sport\", \"substance\", \"symbol\", \"technique\", \"term\",\n",
    "                   \"vehicle\", \"word\"):\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 1, 3, 4, 5],\n",
    "    1: [2]\n",
    "}\n",
    "\n",
    "r5 = WeakRule(exec_module=enty, label_maps=label_maps)\n",
    "v3 = r5.execute(dev_instances)\n",
    "acc, p, r, cwacc = r5.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "# LOC\n",
    "def loc(instance):\n",
    "    subject = get_subject(instance['string'])\n",
    "    if subject in (\"city\", \"country\", \"mountain\", \"state\", \"capital\"):\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 1, 2, 3, 5],\n",
    "    1: [4]\n",
    "}\n",
    "\n",
    "\n",
    "r7 = WeakRule(exec_module=loc, label_maps=label_maps)\n",
    "v3 = r7.execute(dev_instances)\n",
    "acc, p, r, cwacc = r7.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "# NUM\n",
    "def num(instance):\n",
    "    st = instance['string'].lower()\n",
    "    if \"what year\" in st:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 1, 2, 3, 4],\n",
    "    1: [5]\n",
    "}\n",
    "\n",
    "r8 = WeakRule(exec_module=num, label_maps=label_maps)\n",
    "v3 = r8.execute(dev_instances)\n",
    "acc, p, r, cwacc = r8.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "# NUM\n",
    "def num1(instance):\n",
    "    st = instance['string'].lower()\n",
    "    if \"how many\" in st or 'how much' in st or 'how old' in st:\n",
    "        return 1\n",
    "    \n",
    "    return -1\n",
    "    \n",
    "label_maps = {\n",
    "    0: [0, 1, 2, 3, 4],\n",
    "    1: [5]\n",
    "}\n",
    "\n",
    "r9 = WeakRule(exec_module=num1, label_maps=label_maps)\n",
    "v3 = r9.execute(dev_instances)\n",
    "acc, p, r, cwacc = r9.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "test_patt = 'what.*mean'\n",
    "whatmean = BinaryRERules(name='what_*_mean',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0, 2, 3, 4,5], 1:[1]}, unipolar=True)\n",
    "v5 = whatmean.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = whatmean.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "test_patt = 'what.*use of|what.*origin of|why do'\n",
    "desc_r1 = BinaryRERules(name='what_*_',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0, 2, 3, 4,5], 1:[1]}, unipolar=True)\n",
    "v5 = desc_r1.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = desc_r1.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "test_patt = 'how far|what.*birthday|how long|how deep|when did|when was|how tall|what month|population|toll|how big|how long|what year'\n",
    "numpattern1 = BinaryRERules(name='num_patt',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0,1, 2, 3, 4], 1:[5]}, unipolar=True)\n",
    "v5 = numpattern1.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = numpattern1.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "test_patt = 'what is the origin|what is the history|what.*mean|how do you buy|what is the difference|how can I|how do I|what effect'\n",
    "descpatt_1 = BinaryRERules(name='num_patt',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0, 2, 3, 4,5], 1:[1]}, unipolar=True)\n",
    "v5 = descpatt_1.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = descpatt_1.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "test_patt = 'how.*tell|how d.*affect|how do.*work|how do you fix|how do you get|how do you find|how do I find|how.*made'\n",
    "descpatt_2 = BinaryRERules(name='num_patt',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0, 2, 3, 4,5], 1:[1]}, unipolar=True)\n",
    "v5 = descpatt_2.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = descpatt_2.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "test_patt = 'how do|how was|how are|how is|how was|how could|how can'\n",
    "newhow = BinaryRERules(name='num_patt',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[0, 2, 3, 4,5], 1:[1]}, unipolar=True)\n",
    "v5 = newhow.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = newhow.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)\n",
    "\n",
    "\n",
    "\n",
    "test_patt = 'what.*stand for'\n",
    "wsf = BinaryRERules(name='what_*_sf',re_pattern=test_patt, \n",
    "                                 preproc=lambda inst:inst['string'].lower(), \n",
    "                                 label_maps={0:[1, 2, 3, 4,5], 1:[0]}, unipolar=True)\n",
    "v5 = wsf.execute(dev_instances)\n",
    "\n",
    "acc, p, r, cwacc = wsf.eval(dev_labels, class_wise_acc=True)\n",
    "print(acc, p, r)\n",
    "print(cwacc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: [3], 1: [4], 2: [5], 3: [1], 4: [1, 5], 5: [2, 3], 6: [0]}\n",
      "{0: [0, 1, 5], 1: [2, 3, 4]}\n",
      "{0: [2, 3, 4, 5], 1: [0, 1]}\n",
      "{0: [1, 2, 3, 4, 5], 1: [0]}\n",
      "{0: [0, 1, 2, 3, 5], 1: [4]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 1, 3, 4, 5], 1: [2]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [1, 2, 3, 4, 5], 1: [0]}\n"
     ]
    }
   ],
   "source": [
    "from annotators import executor, evaluator\n",
    "\n",
    "train_instances, _ = get_inst(train)\n",
    "\n",
    "plfs = [r0,r1,r2,r3,r7,r8,r9, r4, r5,\n",
    "              whatmean, \n",
    "              desc_r1, \n",
    "              numpattern1, \n",
    "              descpatt_1, \n",
    "              descpatt_2, \n",
    "              newhow,\n",
    "              wsf]\n",
    "\n",
    "votes, fid2clusters = executor(plfs, train_instances, one_indexed=False)\n",
    "votes, _, train_idx = votes_filter(votes, return_idx=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch   1: 100%|██████████| 1/1 [00:00<00:00, 15.41it/s, Epoch Loss: =13.4]\n",
      "epoch   2: 100%|██████████| 1/1 [00:00<00:00, 17.44it/s, Epoch Loss: =13.3]\n",
      "epoch   3: 100%|██████████| 1/1 [00:00<00:00, 17.56it/s, Epoch Loss: =13.3]\n",
      "epoch   4:   0%|          | 0/1 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup:  0.00919961929321289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch   4: 100%|██████████| 1/1 [00:00<00:00, 17.01it/s, Epoch Loss: =13.2]\n",
      "epoch   5: 100%|██████████| 1/1 [00:00<00:00, 17.14it/s, Epoch Loss: =13.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup:  0.011838912963867188\n",
      "Parallel Estimation:  0.007220745086669922\n"
     ]
    }
   ],
   "source": [
    "from nplm import Abstention\n",
    "\n",
    "lm_cfg={'lr': 0.01,\n",
    " 'epoch': 5,\n",
    " 'seed': 0,\n",
    " 'batch_size': 8192,\n",
    " 'momentum': 0.9,\n",
    " 'step_schedule': 'p',\n",
    " 'step_multiplier': 0.1}\n",
    "\n",
    "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "labelmodel = Abstention(num_classes=6,\n",
    "                        fid2clusters=fid2clusters,\n",
    "                        opt_cfg=lm_cfg,\n",
    "                        preset_cb=torch.ones([6]).to(device)/ 6,\n",
    "                        device=device, verbose=True)\n",
    "\n",
    "labelmodel.optimize(votes)\n",
    "label_estim = labelmodel.weak_label(votes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('trec6_soft_labels', {'sl': label_estim, 'idx': train_idx})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: [3], 1: [4], 2: [5], 3: [1], 4: [1, 5], 5: [2, 3], 6: [0]}\n",
      "{0: [0, 1, 5], 1: [2, 3, 4]}\n",
      "{0: [2, 3, 4, 5], 1: [0, 1]}\n",
      "{0: [1, 2, 3, 4, 5], 1: [0]}\n",
      "{0: [0, 1, 2, 3, 5], 1: [4]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 1, 3, 4, 5], 1: [2]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 1, 2, 3, 4], 1: [5]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [0, 2, 3, 4, 5], 1: [1]}\n",
      "{0: [1, 2, 3, 4, 5], 1: [0]}\n",
      "Setup:  0.0019774436950683594\n",
      "Parallel Estimation:  0.00121307373046875\n",
      "(0.382, 0.7317976025615662, 0.4775683434404692, 0.42981563493713243)\n",
      "[0.446 0.534]\n"
     ]
    }
   ],
   "source": [
    "from eval import gen_stats, topk_results\n",
    "\n",
    "data_instance_test, data_label_test = get_inst(test)\n",
    "\n",
    "votes_test, _ = executor(plfs, data_instance_test, one_indexed=False)\n",
    "\n",
    "\n",
    "votes_test_filtered, label_filtered_test = votes_filter(votes_test, labels=data_label_test)\n",
    "\n",
    "\n",
    "wlabels_test = labelmodel.weak_label(votes_test)\n",
    "\n",
    "test_label_test = [elem+1 for elem in data_label_test]\n",
    "\n",
    "_, stats = gen_stats(wlabels_test, test_label_test)\n",
    "print(stats)\n",
    "print(topk_results(wlabels_test, test_label_test, [1,2]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}