Linear-Multi-Frontier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "import re\n",
    "import glob\n",
    "import io\n",
    "import time\n",
    "from statistics import mean bert\n",
    "import os\n",
    "import json\n",
    "import gensim\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import mpld3\n",
    "\n",
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "from torchtext import data\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchtext.vocab import Vectors\n",
    "\n",
    "%matplotlib inline\n",
    "mpld3.enable_notebook()\n",
    "\n",
    "#torch.cuda.set_device(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 77\n",
    "BATCH_SIZE = 64\n",
    "MAX_SEQUENCE = 2048\n",
    "MAX_VOCAB_SIZE = 25000\n",
    "HIDDEN_DIM = 100\n",
    "OUTPUT_DIM = 5\n",
    "EMBEDDING_DIM = 100\n",
    "N_EPOCHS = 500\n",
    "TRAIN_RATIO = 0.8\n",
    "POS_WEIGHT = torch.tensor([1, 7, 8, 4, 9])\n",
    "MICRO = 'micro'\n",
    "MACRO = 'macro'\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "DATA_FOLDER=\"FastText-Multi\"\n",
    "\n",
    "DROPOUT=0.5\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f_measure(predictions, labels):\n",
    "    diagnoses = {}\n",
    "    MICRO = 'micro'\n",
    "    MACRO = 'macro'\n",
    "    predicts = []\n",
    "    diagnoses[MICRO] = {}\n",
    "    \n",
    "    rounded_preds = torch.round(torch.sigmoid(predictions))\n",
    "    predicts.extend(rounded_preds.data.tolist())\n",
    "    \n",
    "    for index, value in enumerate(rounded_preds):\n",
    "        for did, dvalue in enumerate(rounded_preds[index]):\n",
    "            v = dvalue.item()                    \n",
    "            if v == 1:\n",
    "                if dvalue == labels[index, did]:\n",
    "                    if did not in diagnoses:\n",
    "                        diagnoses[did] = {}                                \n",
    "                    diagnoses[did]['tp'] = diagnoses[did].get('tp', 0) + 1\n",
    "                    diagnoses[MICRO]['tp'] = diagnoses[MICRO].get('tp', 0) + 1\n",
    "                else:\n",
    "                    if did not in diagnoses:\n",
    "                        diagnoses[did] = {}\n",
    "                    diagnoses[did]['fp'] = diagnoses[did].get('fp', 0) + 1\n",
    "                    diagnoses[MICRO]['fp'] = diagnoses[MICRO].get('fp', 0) + 1\n",
    "            elif v == 0:\n",
    "                if 1 == labels[index, did].item():\n",
    "                    if did not in diagnoses:\n",
    "                        diagnoses[did] = {}\n",
    "                    diagnoses[did]['fn'] = diagnoses[did].get('fn', 0) + 1\n",
    "                    diagnoses[MICRO]['fn'] = diagnoses[MICRO].get('fn', 0) + 1\n",
    "    diagnoses[MACRO] = {}\n",
    "    for d in diagnoses:        \n",
    "        if d is MACRO:\n",
    "            continue\n",
    "        try:\n",
    "            diagnoses[d]['p']=diagnoses[d].get('tp', 0)/(diagnoses[d].get('tp', 0)+diagnoses[d].get('fp', 0))            \n",
    "        except:            \n",
    "            diagnoses[d]['p']=0.0\n",
    "        if d is not MICRO:\n",
    "                diagnoses[MACRO]['p']=diagnoses[MACRO].get('p', 0.0)+diagnoses[d]['p']                \n",
    "            \n",
    "        try:\n",
    "            diagnoses[d]['r']=diagnoses[d].get('tp', 0)/(diagnoses[d].get('tp', 0)+diagnoses[d].get('fn', 0))            \n",
    "        except:\n",
    "            diagnoses[d]['r']=0.0\n",
    "        if d is not MICRO:\n",
    "            diagnoses[MACRO]['r']=diagnoses[MACRO].get('r', 0.0)+diagnoses[d]['r']\n",
    "        \n",
    "        try:\n",
    "            diagnoses[d]['f']=2/(1/diagnoses[d]['p']+1/diagnoses[d]['r'])            \n",
    "        except:\n",
    "            diagnoses[d]['f']=0.0\n",
    "        if d is not MICRO:\n",
    "                diagnoses[MACRO]['f']=diagnoses[MACRO].get('f', 0.0)+diagnoses[d]['f']\n",
    "    if len(diagnoses)-2 > 0:\n",
    "        diagnoses[MACRO]['f']=diagnoses[MACRO].get('f', 0.0)/float(len(diagnoses)-2)\n",
    "        diagnoses[MACRO]['p']=diagnoses[MACRO].get('p', 0.0)/float(len(diagnoses)-2)\n",
    "        diagnoses[MACRO]['r']=diagnoses[MACRO].get('r', 0.0)/float(len(diagnoses)-2)\n",
    "    else:\n",
    "        diagnoses[MACRO]['f']='n/a'\n",
    "        diagnoses[MACRO]['p']='n/a'\n",
    "        diagnoses[MACRO]['r']='n/a'\n",
    "    return diagnoses, predicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_fscores(new, overall):\n",
    "    MICRO = 'micro'\n",
    "    MACRO = 'macro'\n",
    "    \n",
    "    for k in new:\n",
    "        if k not in overall:\n",
    "            overall[k] = {}\n",
    "        overall[k]['tp'] = overall[k].get('tp', 0) + new[k].get('tp', 0)\n",
    "        overall[k]['fp'] = overall[k].get('fp', 0) + new[k].get('fp', 0)\n",
    "        overall[k]['fn'] = overall[k].get('fn', 0) + new[k].get('fn', 0)\n",
    "        overall[MICRO]['tp'] = overall[MICRO].get('tp', 0) + new[k].get('tp', 0)\n",
    "        overall[MICRO]['fp'] = overall[MICRO].get('fp', 0) + new[k].get('fp', 0)\n",
    "        overall[MICRO]['fn'] = overall[MICRO].get('fn', 0) + new[k].get('fn', 0)\n",
    "        \n",
    "    overall[MACRO] = {}\n",
    "    for d in overall:        \n",
    "        if d is MACRO:\n",
    "            continue\n",
    "        try:\n",
    "            overall[d]['p']=overall[d].get('tp', 0)/(overall[d].get('tp', 0)+overall[d].get('fp', 0))            \n",
    "        except:            \n",
    "            overall[d]['p']=0.0\n",
    "        if d is not MICRO:\n",
    "            overall[MACRO]['p']=overall[MACRO].get('p', 0.0)+overall[d]['p']                \n",
    "            \n",
    "        try:\n",
    "            overall[d]['r']=overall[d].get('tp', 0)/(overall[d].get('tp', 0)+overall[d].get('fn', 0))            \n",
    "        except:\n",
    "            overall[d]['r']=0.0\n",
    "        if d is not MICRO:\n",
    "            overall[MACRO]['r']=overall[MACRO].get('r', 0.0)+overall[d]['r']\n",
    "        \n",
    "        try:\n",
    "            overall[d]['f']=2/(1/overall[d]['p']+1/overall[d]['r'])            \n",
    "        except:\n",
    "            overall[d]['f']=0.0\n",
    "        if d is not MICRO:\n",
    "                overall[MACRO]['f']=overall[MACRO].get('f', 0.0)+overall[d]['f']\n",
    "\n",
    "    \n",
    "    if len(overall)-2 > 0:\n",
    "        overall[MACRO]['f']=overall[MACRO].get('f', 0.0)/float(len(overall)-2)\n",
    "        overall[MACRO]['p']=overall[MACRO].get('p', 0.0)/float(len(overall)-2)\n",
    "        overall[MACRO]['r']=overall[MACRO].get('r', 0.0)/float(len(overall)-2)\n",
    "    else:\n",
    "        overall[MACRO]['f']='n/a'\n",
    "        overall[MACRO]['p']='n/a'\n",
    "        overall[MACRO]['r']='n/a'\n",
    "    return overall\n",
    "\n",
    "def train(model, iterator, optimizer, criterion, model_type):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    #epoch_acc = 0\n",
    "    epoch_fscore = 0\n",
    "    \n",
    "    model.train()\n",
    "    fscores = {}    \n",
    "    for batch in iterator:        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        if model_type == 0:            \n",
    "            predictions = model(batch.all_text)\n",
    "        else:\n",
    "            predictions = model(batch.bh_text, batch.ep_text)\n",
    "            \n",
    "        labels = torch.cat((batch.major_depressive.unsqueeze(1), batch.schizophrenia.unsqueeze(1),\n",
    "                            batch.biploar.unsqueeze(1), batch.minor_depressive.unsqueeze(1), \n",
    "                            batch.dementia.unsqueeze(1)), 1)\n",
    "\n",
    "        loss = criterion(predictions, labels)\n",
    "        \n",
    "        #acc = binary_accuracy(predictions, labels)\n",
    "        fscore, _ = f_measure(predictions, labels)            \n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        #epoch_fscore += fscores['micro'][\"f\"]\n",
    "        fscores = update_fscores(fscore, fscores)\n",
    "        \n",
    "    return epoch_loss / len(iterator), fscores['micro'][\"f\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion, model_type):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    #epoch_acc = 0\n",
    "    epoch_fscore = 0\n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "            if model_type == 0:\n",
    "                predictions = model(batch.all_text)\n",
    "            else:\n",
    "                predictions = model(batch.bh_text, batch.ep_text)\n",
    "        \n",
    "            labels = torch.cat((batch.major_depressive.unsqueeze(1), batch.schizophrenia.unsqueeze(1),\n",
    "                            batch.biploar.unsqueeze(1), batch.minor_depressive.unsqueeze(1), \n",
    "                            batch.dementia.unsqueeze(1)), 1)\n",
    "        \n",
    "            loss = criterion(predictions, labels)\n",
    "            \n",
    "            fscores, _ = f_measure(predictions, labels)            \n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_fscore += fscores['micro'][\"f\"]\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_fscore / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(epoches, model, train_iterator, optimizer, criterion, model_type, model_name, \n",
    "                valid_iterator = None, interval = 50, early_stop = False, period = 20, gap = 0.005, threshold = 0.5):\n",
    "    best_valid_loss = float('inf')\n",
    "    best_valid_fscore = 0\n",
    "    train_losses = []\n",
    "    valid_losses = []\n",
    "    train_accs = []\n",
    "    valid_accs = []\n",
    "    observed_time = 0\n",
    "    for epoch in range(epoches):\n",
    "        start_time = time.time()\n",
    "        \n",
    "        train_loss, train_acc = train(model, train_iterator, optimizer, criterion, model_type)\n",
    "        train_losses.append(train_loss)\n",
    "        train_accs.append(train_acc)\n",
    "        \n",
    "        end_time = time.time()\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "        \n",
    "        if valid_iterator:\n",
    "            valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, model_type)\n",
    "            valid_losses.append(valid_loss)\n",
    "            valid_accs.append(valid_acc)\n",
    "        else:\n",
    "            valid_loss = 0 \n",
    "        \n",
    "        if (epoch + 1) % interval == 0:\n",
    "            print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "            print(f'\\tTrain Loss: {train_loss:.3f} | Train micro-F-score: {train_acc*100:.2f}%')\n",
    "            if valid_iterator:\n",
    "                #print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')\n",
    "                print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. micro-F-score: {valid_acc*100:.2f}%')\n",
    "        elif epoch == epoches - 1:\n",
    "            print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "            print(f'\\tTrain Loss: {train_loss:.3f} | Train micro-F-score: {train_acc*100:.2f}%')\n",
    "            if valid_iterator:\n",
    "                #print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')\n",
    "                print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. micro-F-score: {valid_acc*100:.2f}%')\n",
    "        if valid_loss < best_valid_loss:\n",
    "            best_valid_loss = valid_loss\n",
    "            torch.save(model.state_dict(), os.path.join(DATA_FOLDER, model_name + '_loss.pt'))\n",
    "        if early_stop and best_valid_fscore > threshold and best_valid_fscore - valid_acc > gap:\n",
    "            observed_time += 1\n",
    "            print(f'\\rBest validation F-measure: {best_valid_fscore:.3f}/Current F-measure: {valid_acc:.3f} [Times: {observed_time}/{period}]')  \n",
    "            if observed_time >= period:\n",
    "                print(f'Early stop at epoch {epoch+1:02}.')\n",
    "                break                        \n",
    "        if valid_acc > best_valid_fscore:\n",
    "            best_valid_fscore = valid_acc\n",
    "            torch.save(model.state_dict(), os.path.join(DATA_FOLDER, model_name + '_fscore.pt'))\n",
    "            observed_time = 0        \n",
    "    return train_losses, valid_losses, train_accs, valid_accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analysis_plotter(fig, ax, train, valid, title, param_dict1, param_dict2):\n",
    "    out = ax.plot(train, **param_dict1)\n",
    "    out = ax.plot(valid, **param_dict2)\n",
    "    ax.title.set_text(title)\n",
    "    ax.legend()\n",
    "    pv = float('inf')\n",
    "    x = []\n",
    "    y = []\n",
    "    for k, v in enumerate(valid):\n",
    "        if v > pv:\n",
    "            x.append(k)\n",
    "            y.append(v)\n",
    "        pv = v\n",
    "    scatter = ax.scatter(x, y)\n",
    "    labels = []\n",
    "    for x, y in zip(x,y):\n",
    "        labels.append(f'{x}: {y}')\n",
    "    tooltip = mpld3.plugins.PointLabelTooltip(scatter, labels=labels)\n",
    "    mpld3.plugins.connect(fig, tooltip)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, iterator, criterion, model_type, model_name = None):    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    if model_name:\n",
    "        model.load_state_dict(torch.load(os.path.join(DATA_FOLDER, model_name + '.pt')))\n",
    "\n",
    "    model.eval()\n",
    "    \n",
    "    diagnoses = {}\n",
    "    predicts = []\n",
    "    MICRO = 'micro'\n",
    "    MACRO = 'macro'\n",
    "    diagnoses[MICRO] = {}\n",
    "    with torch.no_grad():                    \n",
    "        for batch in iterator:\n",
    "            if model_type == 0:\n",
    "                predictions = model(batch.all_text)\n",
    "            else:\n",
    "                predictions = model(batch.bh_text, batch.ep_text)\n",
    "            rounded_preds = torch.round(torch.sigmoid(predictions))\n",
    "            predicts.extend(rounded_preds.data.tolist())\n",
    "            labels = torch.cat((batch.major_depressive.unsqueeze(1), batch.schizophrenia.unsqueeze(1),\n",
    "                            batch.biploar.unsqueeze(1), batch.minor_depressive.unsqueeze(1), \n",
    "                            batch.dementia.unsqueeze(1)), 1)\n",
    "            \n",
    "            for index, value in enumerate(rounded_preds):\n",
    "                for did, dvalue in enumerate(rounded_preds[index]):\n",
    "                    v = dvalue.item()                    \n",
    "                    if v == 1:\n",
    "                        if dvalue == labels[index, did]:\n",
    "                            if did not in diagnoses:\n",
    "                                diagnoses[did] = {}                                \n",
    "                            diagnoses[did]['tp'] = diagnoses[did].get('tp', 0) + 1\n",
    "                            diagnoses[MICRO]['tp'] = diagnoses[MICRO].get('tp', 0) + 1 \n",
    "                        else:\n",
    "                            if did not in diagnoses:\n",
    "                                diagnoses[did] = {}\n",
    "                            diagnoses[did]['fp'] = diagnoses[did].get('fp', 0) + 1\n",
    "                            diagnoses[MICRO]['fp'] = diagnoses[MICRO].get('fp', 0) + 1\n",
    "                    elif v == 0:\n",
    "                        if 1 == labels[index, did].item():\n",
    "                            if did not in diagnoses:\n",
    "                                diagnoses[did] = {}\n",
    "                            diagnoses[did]['fn'] = diagnoses[did].get('fn', 0) + 1\n",
    "                            diagnoses[MICRO]['fn'] = diagnoses[MICRO].get('fn', 0) + 1\n",
    "                        else:\n",
    "                            if did not in diagnoses:\n",
    "                                diagnoses[did] = {}\n",
    "                            diagnoses[did]['tn'] = diagnoses[did].get('tn', 0) + 1\n",
    "                            diagnoses[MICRO]['tn'] = diagnoses[MICRO].get('tn', 0) + 1\n",
    "    diagnoses[MACRO] = {}\n",
    "    for d in diagnoses:        \n",
    "        if d is MACRO:\n",
    "            continue\n",
    "        try:\n",
    "            diagnoses[d]['p']=diagnoses[d].get('tp', 0)/(diagnoses[d].get('tp', 0)+diagnoses[d].get('fp', 0))\n",
    "            if d is not MICRO:\n",
    "                diagnoses[MACRO]['p']=diagnoses[MACRO].get('p', 0.0)+diagnoses[d]['p']                \n",
    "        except:            \n",
    "            diagnoses[d]['p']=0.0\n",
    "            \n",
    "        try:\n",
    "            diagnoses[d]['r']=diagnoses[d].get('tp', 0)/(diagnoses[d].get('tp', 0)+diagnoses[d].get('fn', 0))\n",
    "            if d is not MICRO:\n",
    "                diagnoses[MACRO]['r']=diagnoses[MACRO].get('r', 0.0)+diagnoses[d]['r']\n",
    "        except:\n",
    "            diagnoses[d]['r']=0.0\n",
    "        \n",
    "        try:\n",
    "            diagnoses[d]['f']=2/(1/diagnoses[d]['p']+1/diagnoses[d]['r'])\n",
    "            if d is not MICRO:\n",
    "                diagnoses[MACRO]['f']=diagnoses[MACRO].get('f', 0.0)+diagnoses[d]['f']\n",
    "        except:\n",
    "            diagnoses[d]['f']=0.0\n",
    "    diagnoses[MACRO]['f']=diagnoses[MACRO].get('f', 0.0)/float(len(diagnoses)-2)\n",
    "    diagnoses[MACRO]['p']=diagnoses[MACRO].get('p', 0.0)/float(len(diagnoses)-2)\n",
    "    diagnoses[MACRO]['r']=diagnoses[MACRO].get('r', 0.0)/float(len(diagnoses)-2)\n",
    "    return diagnoses, predicts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names =['ID', 'BH Text', 'EP Text', 'Major Depressive', 'Schizophrenia', 'Biploar', 'Minor Depressive', 'Dementia']\n",
    "ntuhdataset = pd.read_csv('../Datasets/NTUH/train_preprocessing.txt', sep ='\\t', names = names)\n",
    "ntuhdataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(ntuhdataset.groupby('Major Depressive').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Schizophrenia').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Biploar').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Minor Depressive').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Dementia').count().iloc[:,:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bh_text = ntuhdataset['BH Text'].apply(lambda x: len(str(x).split()))\n",
    "plt.figure(figsize=(5,5))\n",
    "avg_bh_text = mean(bh_text)\n",
    "plt.title(f'BH Text Length Distribution: Average legnth: {avg_bh_text}')\n",
    "plt.hist(bh_text, bins = 50)\n",
    "plt.show()\n",
    "\n",
    "ep_text = ntuhdataset['EP Text'].apply(lambda x: len(str(x).split()))\n",
    "avg_ep_text = mean(ep_text)\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.title(f'EP Text Length Distribution: Average length: {avg_ep_text}')\n",
    "plt.hist(ep_text, bins = 50)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ntuhdataset = pd.read_csv('../Datasets/NTUH/test_preprocessing.txt', sep ='\\t', names = names)\n",
    "ntuhdataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(ntuhdataset.groupby('Major Depressive').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Schizophrenia').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Biploar').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Minor Depressive').count().iloc[:,:3])\n",
    "display(ntuhdataset.groupby('Dementia').count().iloc[:,:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bh_text = ntuhdataset['BH Text'].apply(lambda x: len(str(x).split()))\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.title('BH Text Length Distribution')\n",
    "plt.hist(bh_text, bins = 50)\n",
    "plt.show()\n",
    "\n",
    "ep_text = ntuhdataset['EP Text'].apply(lambda x: len(str(x).split()))\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.title('EP Text Length Distribution')\n",
    "plt.hist(ep_text, bins = 50)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NTUHDataset(data.Dataset):\n",
    "    #urls = ['Datasets\\\\NTUH\\\\corpus.txt']\n",
    "    name = 'ntuh'\n",
    "    dirname = 'ntuh'\n",
    "    diagnosis_types = ['major_depressive', 'schizophrenia', 'biploar', 'minor_depressive', 'dementia']\n",
    "    \n",
    "    @staticmethod\n",
    "    def sort_key(ex):\n",
    "        return len(ex.all_text) # TODO add ep_text?\n",
    "\n",
    "    def __init__(self, path, bh_text_field, ep_text_field, all_text_field,\n",
    "                 major_label_field, sch_label_field, bipolar_label_field, minor_label_field, dementia_label_field,\n",
    "                 **kwargs):\n",
    "        fields = [('patient_id', None), \n",
    "                  ('bh_text', bh_text_field),\n",
    "                  ('ep_text', ep_text_field),\n",
    "                  ('all_text', all_text_field),\n",
    "                  ('major_depressive', major_label_field),\n",
    "                  ('schizophrenia', sch_label_field),\n",
    "                  ('biploar', bipolar_label_field),\n",
    "                  ('minor_depressive', minor_label_field),\n",
    "                  ('dementia', dementia_label_field)]\n",
    "        examples = []\n",
    "        \n",
    "        for fname in glob.iglob(path + '.txt'):\n",
    "            with io.open(fname, 'r', encoding=\"utf-8\") as f:\n",
    "                for line in f:\n",
    "                    pid, bh_text, ep_text, major_d, sc, bp, minor_d, de = line.strip().split('\\t')\n",
    "                    all_text = \"%s <sep> %s\" % (bh_text, ep_text)\n",
    "                    examples.append(data.Example.fromlist([pid, bh_text, ep_text, all_text, major_d, sc, bp, minor_d, de], \n",
    "                                                          fields))\n",
    "        super(NTUHDataset, self).__init__(examples, fields, **kwargs)\n",
    "\n",
    "    @classmethod\n",
    "    def splits(cls, \n",
    "               bh_text_field, ep_text_field, all_text_field,\n",
    "               major_label_field, sch_label_field, bipolar_label_field, minor_label_field, dementia_label_field,\n",
    "               root='..\\\\Datasets\\\\NTUH',\n",
    "               #train='train', test='test', **kwargs):\n",
    "               train='train_preprocessing', test='test_preprocessing', **kwargs):\n",
    "        return super(NTUHDataset, cls).splits(\n",
    "            path = root, root=root, \n",
    "            bh_text_field = bh_text_field, ep_text_field = ep_text_field, all_text_field = all_text_field, \n",
    "            major_label_field = major_label_field, sch_label_field = sch_label_field, \n",
    "            bipolar_label_field = bipolar_label_field, minor_label_field = minor_label_field, \n",
    "            dementia_label_field = dementia_label_field,\n",
    "            train=train, validation=None, test=test, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = ntuhdataset['BH Text'][0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize = str.split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_cut(sentence):\n",
    "    #sentence = sentence.replace('<unk>', '？')\n",
    "    sentences = re.split(r'\\s*<sep>(?:\\s*<sep>)*\\s*', sentence)\n",
    "    filtered_sentence = list(filter(lambda sent: '<unk>' not in sent, sentences))\n",
    "    sents = [tokenize(sent) for sent in filtered_sentence]\n",
    "    tokens = []\n",
    "    sents = [allsents.split() for allsents in \n",
    "             [' [SEP] '.join(sent) for sent in [[' '.join(token) for token in sents]]]]    \n",
    "    tokens.extend(sents[0])\n",
    "    tokens = tokens[:MAX_SEQUENCE]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenize_and_cut(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N-gram features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_ngrams(token_list, n = 2):\n",
    "    ngram_list = token_list.copy()\n",
    "    for i in range(2, n+1):\n",
    "        ngrams = set(zip(*[token_list[j:] for j in range(i)]))\n",
    "        for ngram in ngrams:\n",
    "            ngram_list.append(' '.join(ngram))\n",
    "    return ngram_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_ngrams(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BH_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True, lower = True,\n",
    "                     preprocessing = generate_ngrams)\n",
    "EP_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True, lower = True,\n",
    "                     preprocessing = generate_ngrams)\n",
    "ALL_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True, lower = True,\n",
    "                     preprocessing = generate_ngrams)\n",
    "\n",
    "MAJ_LABEL = data.LabelField(dtype = torch.float)\n",
    "SCH_LABEL = data.LabelField(dtype = torch.float)\n",
    "BIP_LABEL = data.LabelField(dtype = torch.float)\n",
    "MIN_LABEL = data.LabelField(dtype = torch.float)\n",
    "DEM_LABEL = data.LabelField(dtype = torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_train_data, test_data = NTUHDataset.splits(BH_TEXT, EP_TEXT, ALL_TEXT, \n",
    "                                           MAJ_LABEL, SCH_LABEL, BIP_LABEL, MIN_LABEL, DEM_LABEL)\n",
    "train_data, valid_data = full_train_data.split(random_state = random.seed(SEED), split_ratio = TRAIN_RATIO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dic = {'ID': []}\n",
    "for example in train_data.examples:\n",
    "    v = data_dic.get('ID')\n",
    "    v.append(len(data_dic['ID']))\n",
    "    data_dic['ID']=v\n",
    "    for k in example.__dict__.keys():\n",
    "        if k in NTUHDataset.diagnosis_types:\n",
    "            v = data_dic.get(k, [])\n",
    "            v.append(example.__dict__[k])\n",
    "            data_dic[k]=v            \n",
    "            \n",
    "trainpd = pd.DataFrame(data_dic)\n",
    "#display(trainpd)\n",
    "ratios = trainpd.groupby('major_depressive').count().iloc[:,:3]['ID']\n",
    "print(f'Training set: Major Depressive ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = trainpd.groupby('schizophrenia').count().iloc[:,:3]['ID']\n",
    "print(f'Training set: Schizophrenia ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = trainpd.groupby('biploar').count().iloc[:,:3]['ID']\n",
    "print(f'Training set: Biploar ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = trainpd.groupby('minor_depressive').count().iloc[:,:3]['ID']\n",
    "print(f'Training set: Minor Depressive ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = trainpd.groupby('dementia').count().iloc[:,:3]['ID']\n",
    "print(f'Training set: Dementia ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "\n",
    "data_dic = {'ID': []}\n",
    "for example in valid_data.examples:\n",
    "    v = data_dic.get('ID')\n",
    "    v.append(len(data_dic['ID']))\n",
    "    data_dic['ID']=v\n",
    "    for k in example.__dict__.keys():\n",
    "        if k in NTUHDataset.diagnosis_types:\n",
    "            v = data_dic.get(k, [])\n",
    "            v.append(example.__dict__[k])\n",
    "            data_dic[k]=v        \n",
    "\n",
    "testpd = pd.DataFrame.from_dict(data_dic)\n",
    "#display(testpd)\n",
    "ratios = testpd.groupby('major_depressive').count().iloc[:,:3]['ID']\n",
    "print(f'\\nTest set: Major Depressive ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = testpd.groupby('schizophrenia').count().iloc[:,:3]['ID']\n",
    "print(f'Test set: Schizophrenia ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = testpd.groupby('biploar').count().iloc[:,:3]['ID']\n",
    "print(f'Test set: Biploar ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = testpd.groupby('minor_depressive').count().iloc[:,:3]['ID']\n",
    "print(f'Test set: Minor Depressive ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')\n",
    "ratios = testpd.groupby('dementia').count().iloc[:,:3]['ID']\n",
    "print(f'Test set: Dementia ({ratios[0]}/{ratios[1]}): {ratios[0]/ratios[1]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Number of training examples: {len(train_data)}\")\n",
    "print(f\"Number of validation examples: {len(valid_data)}\")\n",
    "print(f\"Number of testing examples: {len(test_data)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(vars(train_data.examples[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BH_TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
    "EP_TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
    "ALL_TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
    "\n",
    "MAJ_LABEL.build_vocab(train_data)\n",
    "SCH_LABEL.build_vocab(train_data)\n",
    "BIP_LABEL.build_vocab(train_data)\n",
    "MIN_LABEL.build_vocab(train_data)\n",
    "DEM_LABEL.build_vocab(train_data)\n",
    "\n",
    "print(BH_TEXT.vocab.stoi)\n",
    "print(EP_TEXT.vocab.stoi)\n",
    "print(ALL_TEXT.vocab.stoi)\n",
    "print(MAJ_LABEL.vocab.stoi)\n",
    "print(SCH_LABEL.vocab.stoi)\n",
    "print(BIP_LABEL.vocab.stoi)\n",
    "print(MIN_LABEL.vocab.stoi)\n",
    "print(DEM_LABEL.vocab.stoi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, shuffle = False,\n",
    "    device = device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(valid_iterator))\n",
    "print(vars(batch))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember to recreate the iterators or we will miss the first batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastTextBaseline(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, output_dim, pad_idx):        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)\n",
    "        \n",
    "        self.dropout = nn.Dropout(DROPOUT)\n",
    "        \n",
    "        self.fc = nn.Linear(embedding_dim, output_dim)\n",
    "        \n",
    "    def forward(self, text):\n",
    "        \n",
    "        embedded = self.embedding(text)\n",
    "                \n",
    "        pooled = F.avg_pool2d(embedded, (embedded.shape[1], 1)).squeeze(1) \n",
    "        \n",
    "        return self.fc(self.dropout(pooled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastText(nn.Module):\n",
    "    def __init__(self, vocab_size1, vocab_size2, embedding_dim, output_dim, pad_idx):        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding1 = nn.Embedding(vocab_size1, embedding_dim, padding_idx=pad_idx)\n",
    "        self.embedding2 = nn.Embedding(vocab_size2, embedding_dim, padding_idx=pad_idx)\n",
    "        \n",
    "        self.fc = nn.Linear(embedding_dim*2, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(DROPOUT)\n",
    "    def forward(self, bh_text, ep_text):\n",
    "        \n",
    "        embedded1 = self.embedding1(bh_text)\n",
    "        embedded2 = self.embedding2(ep_text)\n",
    "                \n",
    "        pooled1 = F.avg_pool2d(embedded1, (embedded1.shape[1], 1)).squeeze(1) \n",
    "        pooled2 = F.avg_pool2d(embedded2, (embedded2.shape[1], 1)).squeeze(1) \n",
    "        \n",
    "        return self.fc(self.dropout(torch.cat((pooled1, pooled2), 1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize Our Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = len(ALL_TEXT.vocab)\n",
    "UNK_IDX = ALL_TEXT.vocab.stoi[ALL_TEXT.unk_token]\n",
    "PAD_IDX = ALL_TEXT.vocab.stoi[ALL_TEXT.pad_token]\n",
    "SEP_IDX = ALL_TEXT.vocab.stoi['[sep]']\n",
    "\n",
    "print(\"Input dimension: %s\\nUnknown word index: %s\\nPadding index: %s\\nSeperator index: %s\" % \n",
    "      (INPUT_DIM, UNK_IDX, PAD_IDX, SEP_IDX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = FastTextBaseline(INPUT_DIM, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'The model has {count_parameters(model):,} trainable parameters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.embedding.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.embedding.weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())\n",
    "criterion = nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_losses, valid_losses, train_accs, valid_accs = \\\n",
    "    train_epoch(N_EPOCHS, model, train_iterator, optimizer, criterion, 0, 'baseline1', valid_iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Model Type 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "INPUT_DIM1 = len(BH_TEXT.vocab)\n",
    "INPUT_DIM2 = len(EP_TEXT.vocab)\n",
    "\n",
    "UNK_IDX = ALL_TEXT.vocab.stoi[ALL_TEXT.unk_token]\n",
    "PAD_IDX = ALL_TEXT.vocab.stoi[ALL_TEXT.pad_token]\n",
    "\n",
    "print(\"Input dimension: (%s/%s)\\nUnknown word index: %s\\nPadding index: %s\" % (INPUT_DIM1, INPUT_DIM2, UNK_IDX, PAD_IDX))\n",
    "\n",
    "model2 = FastText(INPUT_DIM1, INPUT_DIM2, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "print(model2)\n",
    "\n",
    "model2.embedding1.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding1.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding1.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model2.parameters())\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "model2 = model2.to(device)\n",
    "criterion = criterion.to(device)\n",
    "\n",
    "train_losses2, valid_losses2, train_accs2, valid_accs2 = \\\n",
    "    train_epoch(N_EPOCHS, model2, train_iterator, optimizer, criterion, 1, 'baseline2', valid_iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, train_losses, valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, train_accs, valid_accs, 'Training/Validation Micro-F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, train_losses2, valid_losses2, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, train_accs2, valid_accs2, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model2, test_iterator, criterion, 1)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0, 'baseline1_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model2, test_iterator, criterion, 1, 'baseline2_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deal with Imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "model = FastTextBaseline(INPUT_DIM, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)\n",
    "criterion = criterion.to(device)\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "train_losses, valid_losses, train_accs, valid_accs = \\\n",
    "    train_epoch(N_EPOCHS, model, train_iterator, optimizer, criterion, 0, 'ft_1_rand', valid_iterator, early_stop=True,\n",
    "               period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, train_losses, valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, train_accs, valid_accs, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0, 'ft_1_rand_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply Early Stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "model = FastTextBaseline(INPUT_DIM, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "train_losses, valid_losses, train_accs, valid_accs = \\\n",
    "    train_epoch(N_EPOCHS, model, train_iterator, optimizer, criterion, 0, \n",
    "                'imb1_ea', valid_iterator, early_stop=True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, train_losses, valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, train_accs, valid_accs, 'Training/Validation Accuracy', {'label': 'Training Accuracy'}, {'label': 'Validation Accuracy'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model, test_iterator, criterion, 0, 'imb1_ea_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Type 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "model2 = FastText(INPUT_DIM1, INPUT_DIM2, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "model2.embedding1.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding1.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding1.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2.embedding2.weight.data[SEP_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model2 = model2.to(device)\n",
    "\n",
    "optimizer = optim.Adam(model2.parameters())\n",
    "criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "criterion = criterion.to(device)\n",
    "\n",
    "train_losses2, valid_losses2, train_accs2, valid_accs2 = \\\n",
    "    train_epoch(N_EPOCHS, model2, train_iterator, optimizer, criterion, 1, 'ft_2_rand', valid_iterator, early_stop = True\n",
    "               , period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, train_losses2, valid_losses2, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, train_accs2, valid_accs2, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model2, test_iterator, criterion, 1)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(model2, test_iterator, criterion, 1, 'ft_2_rand_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = Vectors(name='word2vec_skipgram_model.bin', cache=DATA_FOLDER)\n",
    "WV_EMBEDDING_DIM = vectors.vectors.shape[1]\n",
    "print(vectors.vectors.shape)\n",
    "print(vectors.vectors)\n",
    "print(vectors.itos[0])\n",
    "print(vectors.vectors[vectors.stoi[',']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "WV_ALL_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "WV_BH_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "WV_EP_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "WV_MAJ_LABEL = data.LabelField(dtype = torch.float)\n",
    "WV_SCH_LABEL = data.LabelField(dtype = torch.float)\n",
    "WV_BIP_LABEL = data.LabelField(dtype = torch.float)\n",
    "WV_MIN_LABEL = data.LabelField(dtype = torch.float)\n",
    "WV_DEM_LABEL = data.LabelField(dtype = torch.float)\n",
    "\n",
    "full_wv_train_data, wv_test_data = NTUHDataset.splits(WV_BH_TEXT, WV_EP_TEXT, WV_ALL_TEXT, \n",
    "                                           WV_MAJ_LABEL, WV_SCH_LABEL, WV_BIP_LABEL, \n",
    "                                                            WV_MIN_LABEL, WV_DEM_LABEL)\n",
    "\n",
    "wv_train_data, wv_valid_data = full_wv_train_data.split(random_state = random.seed(SEED), \n",
    "                                                                split_ratio = TRAIN_RATIO)\n",
    "\n",
    "WV_ALL_TEXT.build_vocab(wv_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = vectors, unk_init = torch.Tensor.normal_)\n",
    "\n",
    "WV_BH_TEXT.build_vocab(wv_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE), \n",
    "                 vectors = vectors, unk_init = torch.Tensor.normal_)\n",
    "\n",
    "WV_EP_TEXT.build_vocab(wv_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE), \n",
    "                 vectors = vectors, unk_init = torch.Tensor.normal_)\n",
    "\n",
    "WV_MAJ_LABEL.build_vocab(wv_train_data)\n",
    "WV_SCH_LABEL.build_vocab(wv_train_data)\n",
    "WV_BIP_LABEL.build_vocab(wv_train_data)\n",
    "WV_MIN_LABEL.build_vocab(wv_train_data)\n",
    "WV_DEM_LABEL.build_vocab(wv_train_data)\n",
    "\n",
    "WV_ALL_INPUT_DIM = len(WV_ALL_TEXT.vocab)\n",
    "WV_ALL_UNK_IDX = WV_ALL_TEXT.vocab.stoi[WV_ALL_TEXT.unk_token]\n",
    "WV_ALL_PAD_IDX = WV_ALL_TEXT.vocab.stoi[WV_ALL_TEXT.pad_token]\n",
    "WV_SEP_IDX_ALL = WV_ALL_TEXT.vocab.stoi['[sep]']\n",
    "WV_SEP_IDX_BH = WV_BH_TEXT.vocab.stoi['[sep]']\n",
    "WV_SEP_IDX_EP = WV_EP_TEXT.vocab.stoi['[sep]']\n",
    "\n",
    "print(\"All Text\\nInput dimension: %s\\nUnknown word index: %s\\nPadding index: %s\\nSeperator index: %s/%s/%s\" % \n",
    "                (WV_ALL_INPUT_DIM, WV_ALL_UNK_IDX, WV_ALL_PAD_IDX, WV_SEP_IDX_ALL, WV_SEP_IDX_BH, WV_SEP_IDX_EP))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "wv_train_iterator, wv_valid_iterator, wv_test_iterator = data.BucketIterator.splits(\n",
    "    (wv_train_data, wv_valid_data, wv_test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "wv_model = FastTextBaseline(WV_ALL_INPUT_DIM, WV_EMBEDDING_DIM, OUTPUT_DIM, WV_ALL_PAD_IDX)\n",
    "print(wv_model.embedding.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(wv_model.embedding.weight[WV_ALL_TEXT.vocab.stoi[',']][:10])\n",
    "print(vectors.vectors[vectors.stoi[',']][:10])\n",
    "for s in WV_ALL_TEXT.vocab.stoi:    \n",
    "    if s in vectors.stoi:\n",
    "        with torch.no_grad():\n",
    "            wv_model.embedding.weight[WV_ALL_TEXT.vocab.stoi[s]].copy_(vectors.vectors[vectors.stoi[s]])#.clone()\n",
    "print(wv_model.embedding.weight[WV_ALL_TEXT.vocab.stoi[',']][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_model.embedding.weight.data[WV_ALL_UNK_IDX] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model.embedding.weight.data[WV_SEP_IDX_ALL] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model.embedding.weight.data[WV_ALL_PAD_IDX] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "print(wv_model.embedding.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_optimizer = optim.Adam([param for param in wv_model.parameters() if param.requires_grad == True])\n",
    "wv_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "wv_model = wv_model.to(device)\n",
    "wv_criterion = wv_criterion.to(device)\n",
    "\n",
    "wv_train_losses, wv_valid_losses, wv_train_accs, wv_valid_accs = \\\n",
    "     train_epoch(N_EPOCHS, wv_model, wv_train_iterator, wv_optimizer, wv_criterion, 0, \n",
    "                 'wv1', wv_valid_iterator, early_stop=True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, wv_train_losses, wv_valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, wv_train_accs, wv_valid_accs, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(wv_model, wv_test_iterator, criterion, 0)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "\n",
    "print('#'*40)\n",
    "test_f_scores, predicts = test(wv_model, wv_test_iterator, criterion, 0, 'wv1_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Type 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WV_INPUT_DIM1 = len(WV_BH_TEXT.vocab)\n",
    "WV_INPUT_DIM2 = len(WV_EP_TEXT.vocab)\n",
    "WV_PAD_IDX_BH = WV_BH_TEXT.vocab.stoi[WV_BH_TEXT.pad_token]\n",
    "WV_UNK_IDX_BH = WV_BH_TEXT.vocab.stoi[WV_BH_TEXT.unk_token]\n",
    "WV_PAD_IDX_EP = WV_EP_TEXT.vocab.stoi[WV_EP_TEXT.pad_token]\n",
    "WV_UNK_IDX_EP = WV_EP_TEXT.vocab.stoi[WV_EP_TEXT.unk_token]\n",
    "WV_SEP_IDX_BH = WV_BH_TEXT.vocab.stoi['[sep]']\n",
    "WV_SEP_IDX_EP = WV_EP_TEXT.vocab.stoi['[sep]']\n",
    "\n",
    "print(\"Input dimension: %s/%s\\nUnknown word index: %s/%s\\nPadding index: %s/%s\\nSeperator index: %s/%s\" % \n",
    "                (WV_INPUT_DIM1, WV_INPUT_DIM2, WV_UNK_IDX_BH, WV_UNK_IDX_EP, WV_PAD_IDX_BH, \n",
    "                 WV_PAD_IDX_EP, WV_SEP_IDX_BH, WV_SEP_IDX_EP))\n",
    "assert WV_PAD_IDX_EP == WV_PAD_IDX_BH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "wv_model2 = FastText(WV_INPUT_DIM1, WV_INPUT_DIM2, WV_EMBEDDING_DIM, OUTPUT_DIM, WV_PAD_IDX_EP)\n",
    "\n",
    "for s in WV_BH_TEXT.vocab.stoi:    \n",
    "    if s in vectors.stoi:\n",
    "        with torch.no_grad():\n",
    "            wv_model2.embedding1.weight[WV_BH_TEXT.vocab.stoi[s]].copy_(vectors.vectors[vectors.stoi[s]])\n",
    "\n",
    "for s in WV_EP_TEXT.vocab.stoi:    \n",
    "    if s in vectors.stoi:\n",
    "        with torch.no_grad():\n",
    "            wv_model2.embedding2.weight[WV_EP_TEXT.vocab.stoi[s]].copy_(vectors.vectors[vectors.stoi[s]])\n",
    "\n",
    "wv_model2.embedding1.weight.data[WV_UNK_IDX_BH] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model2.embedding1.weight.data[WV_SEP_IDX_BH] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model2.embedding1.weight.data[WV_PAD_IDX_BH] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model2.embedding2.weight.data[WV_UNK_IDX_EP] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model2.embedding2.weight.data[WV_SEP_IDX_EP] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "wv_model2.embedding2.weight.data[WV_PAD_IDX_EP] = torch.zeros(WV_EMBEDDING_DIM)\n",
    "\n",
    "wv_optimizer = optim.Adam([param for param in wv_model2.parameters() if param.requires_grad == True])\n",
    "wv_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "wv_model2 = wv_model2.to(device)\n",
    "wv_criterion = wv_criterion.to(device)\n",
    "\n",
    "wv_train_losses2, wv_valid_losses2, wv_train_accs2, wv_valid_accs2 = \\\n",
    "     train_epoch(N_EPOCHS, wv_model2, wv_train_iterator, wv_optimizer, wv_criterion, 1, \n",
    "                 'wv2', wv_valid_iterator, early_stop = True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, wv_train_losses2, wv_valid_losses2, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, wv_train_accs2, wv_valid_accs2, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(wv_model2, wv_test_iterator, wv_criterion, 1)\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "\n",
    "print('#'*40)\n",
    "test_f_scores, predicts = test(wv_model2, test_iterator, criterion, 1, 'wv2_fscore')\n",
    "\n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GLOVE_BH_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "GLOVE_EP_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "GLOVE_ALL_TEXT = data.Field(tokenize = tokenize_and_cut, batch_first = True,\n",
    "                     preprocessing = generate_ngrams, lower = True)\n",
    "GLOVE_MAJ_LABEL = data.LabelField(dtype = torch.float)\n",
    "GLOVE_SCH_LABEL = data.LabelField(dtype = torch.float)\n",
    "GLOVE_BIP_LABEL = data.LabelField(dtype = torch.float)\n",
    "GLOVE_MIN_LABEL = data.LabelField(dtype = torch.float)\n",
    "GLOVE_DEM_LABEL = data.LabelField(dtype = torch.float)\n",
    "\n",
    "full_glove_train_data, glove_test_data = NTUHDataset.splits(GLOVE_BH_TEXT, GLOVE_EP_TEXT, GLOVE_ALL_TEXT, \n",
    "                                           GLOVE_MAJ_LABEL, GLOVE_SCH_LABEL, GLOVE_BIP_LABEL, \n",
    "                                                            GLOVE_MIN_LABEL, GLOVE_DEM_LABEL)\n",
    "\n",
    "glove_train_data, glove_valid_data = full_glove_train_data.split(random_state = random.seed(SEED), \n",
    "                                                                 split_ratio = TRAIN_RATIO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we used Glove and gaussian distribution to initialize our word vectors and unknow words, respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GLOVE_ALL_TEXT.build_vocab(glove_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = \"glove.6B.300d\", \n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "GLOVE_BH_TEXT.build_vocab(glove_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = \"glove.6B.300d\", \n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "GLOVE_EP_TEXT.build_vocab(glove_train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = \"glove.6B.300d\", \n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "GLOVE_MAJ_LABEL.build_vocab(glove_train_data)\n",
    "GLOVE_SCH_LABEL.build_vocab(glove_train_data)\n",
    "GLOVE_BIP_LABEL.build_vocab(glove_train_data)\n",
    "GLOVE_MIN_LABEL.build_vocab(glove_train_data)\n",
    "GLOVE_DEM_LABEL.build_vocab(glove_train_data)\n",
    "\n",
    "GLOVE_ALL_TEXT.vocab.vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "N_EPOCHS = 500\n",
    "GLOVE_INPUT_DIM = len(GLOVE_ALL_TEXT.vocab)\n",
    "GLOVE_PAD_IDX = GLOVE_ALL_TEXT.vocab.stoi[GLOVE_ALL_TEXT.pad_token]\n",
    "GLOVE_UNK_IDX = GLOVE_ALL_TEXT.vocab.stoi[GLOVE_ALL_TEXT.unk_token]\n",
    "GLOVE_SEP_IDX = GLOVE_ALL_TEXT.vocab.stoi['[sep]']\n",
    "GLOVE_EMBEDDING_DIM = GLOVE_ALL_TEXT.vocab.vectors.shape[1]\n",
    "\n",
    "print(\"Input dimension: %s\\nUnknown word index: %s\\nPadding index: %s\\nSeparator index: %s\" % \n",
    "      (GLOVE_INPUT_DIM, GLOVE_UNK_IDX, GLOVE_PAD_IDX, GLOVE_SEP_IDX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_train_iterator, glove_valid_iterator, glove_test_iterator = data.BucketIterator.splits(\n",
    "    (glove_train_data, glove_valid_data, glove_test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "glove_model = FastTextBaseline(GLOVE_INPUT_DIM, GLOVE_EMBEDDING_DIM, OUTPUT_DIM, GLOVE_PAD_IDX)\n",
    "print(glove_model.embedding.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_model.embedding.weight.data.copy_(GLOVE_ALL_TEXT.vocab.vectors)\n",
    "print(glove_model.embedding.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_model.embedding.weight.data[GLOVE_UNK_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model.embedding.weight.data[GLOVE_PAD_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model.embedding.weight.data[GLOVE_SEP_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "print(glove_model.embedding.weight)\n",
    "glove_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_optimizer = optim.Adam([param for param in glove_model.parameters() if param.requires_grad == True])\n",
    "glove_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "glove_model = glove_model.to(device)\n",
    "glove_criterion = glove_criterion.to(device)\n",
    "\n",
    "glove_train_losses, glove_valid_losses, glove_train_accs, glove_valid_accs = \\\n",
    "     train_epoch(N_EPOCHS, glove_model, glove_train_iterator, glove_optimizer, glove_criterion, 0, \n",
    "                 'glove1', glove_valid_iterator, early_stop = True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, glove_train_losses, glove_valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, glove_train_accs, glove_valid_accs, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(glove_model, glove_test_iterator, glove_criterion, 0)\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          \n",
    "              \n",
    "print('#'*40)              \n",
    "test_f_scores, predicts = test(glove_model, glove_test_iterator, glove_criterion, 0, 'glove1_fscore')\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Type 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "glove_train_iterator, glove_valid_iterator, glove_test_iterator = data.BucketIterator.splits(\n",
    "    (glove_train_data, glove_valid_data, glove_test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)\n",
    "\n",
    "GLOVE_INPUT_DIM1 = len(GLOVE_BH_TEXT.vocab)\n",
    "GLOVE_INPUT_DIM2 = len(GLOVE_EP_TEXT.vocab)\n",
    "\n",
    "glove_model2 = FastText(GLOVE_INPUT_DIM1, GLOVE_INPUT_DIM2, GLOVE_EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)\n",
    "glove_model2.embedding1.weight.data.copy_(GLOVE_BH_TEXT.vocab.vectors)\n",
    "glove_model2.embedding1.weight.data[GLOVE_UNK_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model2.embedding1.weight.data[GLOVE_PAD_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model2.embedding1.weight.data[GLOVE_SEP_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "\n",
    "glove_model2.embedding2.weight.data.copy_(GLOVE_EP_TEXT.vocab.vectors)\n",
    "glove_model2.embedding2.weight.data[GLOVE_UNK_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model2.embedding2.weight.data[GLOVE_PAD_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "glove_model2.embedding2.weight.data[GLOVE_SEP_IDX] = torch.zeros(GLOVE_EMBEDDING_DIM)\n",
    "\n",
    "glove_model2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_optimizer = optim.Adam([param for param in glove_model2.parameters() if param.requires_grad == True])\n",
    "glove_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "glove_mode2 = glove_model2.to(device)\n",
    "glove_criterion = glove_criterion.to(device)\n",
    "\n",
    "glove_train_losses2, glove_valid_losses2, glove_train_accs2, glove_valid_accs2 = \\\n",
    "     train_epoch(N_EPOCHS, glove_model2, glove_train_iterator, glove_optimizer, glove_criterion, 1, \n",
    "                 'glove2', glove_valid_iterator, early_stop = True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, glove_train_losses2, glove_valid_losses2, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, glove_train_accs2, glove_valid_accs2, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(glove_model2, glove_test_iterator, glove_criterion, 1)\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          \n",
    "              \n",
    "print('#'*40)              \n",
    "test_f_scores, predicts = test(glove_model2, glove_test_iterator, glove_criterion, 1, 'glove2_fscore')\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "N_EPOCHS = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_MODEL = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL, do_lower_case = True)\n",
    "bert = BertModel.from_pretrained(BERT_MODEL, output_hidden_states = True)\n",
    "\n",
    "BERT_EOS_TOKEN = tokenizer.sep_token\n",
    "BERT_PAD_TOKEN = tokenizer.pad_token\n",
    "BERT_UNK_TOKEN = tokenizer.unk_token\n",
    "\n",
    "BERT_EOS_IDX = tokenizer.convert_tokens_to_ids(BERT_EOS_TOKEN)\n",
    "BERT_PAD_IDX = tokenizer.convert_tokens_to_ids(BERT_PAD_TOKEN)\n",
    "BERT_UNK_IDX = tokenizer.convert_tokens_to_ids(BERT_UNK_TOKEN)\n",
    "\n",
    "print(f'{BERT_EOS_TOKEN}:{BERT_EOS_IDX}, {BERT_PAD_TOKEN}:{BERT_PAD_IDX}, {BERT_UNK_TOKEN}:{BERT_UNK_IDX}') \n",
    "\n",
    "BERT_MAX_SEQUENCE = tokenizer.max_model_input_sizes[BERT_MODEL]\n",
    "print(BERT_MAX_SEQUENCE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_tokenize_and_cut(sentence):\n",
    "    sentences = re.split(r'\\s*<sep>(?:\\s*<sep>)*\\s*', sentence)\n",
    "    filtered_sentence = list(filter(lambda sent: '<unk>' not in sent, sentences))\n",
    "    sents = [tokenizer.tokenize(sent[:BERT_MAX_SEQUENCE-2]) for sent in filtered_sentence]\n",
    "    tokens = []\n",
    "    sents = [allsents.split() for allsents in \n",
    "             [' [SEP] '.join(sent) for sent in [[' '.join(token) for token in sents]]]]\n",
    "    tokens.extend(sents[0])\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_convert_tokens_to_ids(sents_tokens):\n",
    "    sents_tokens = \" \".join(sents_tokens)\n",
    "    sents_tokens = re.split(r'(?i)\\s*\\[sep\\](?:\\s*\\[sep\\])*\\s*', sents_tokens)\n",
    "    sents_tokens = list(filter(lambda x: len(x) > 2, [('[CLS] '+sent+' [SEP]').split() for sent in sents_tokens]))\n",
    "    sents = [tokenizer.convert_tokens_to_ids(tokens) for tokens in sents_tokens]\n",
    "    tokens = []\n",
    "    for sent in sents:\n",
    "        tokens.extend(sent[:MAX_SEQUENCE-1-len(tokens)])\n",
    "    tokens.append(BERT_EOS_IDX)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_ALL_TEXT = data.Field(batch_first = True,\n",
    "                  use_vocab = False,\n",
    "                  tokenize = bert_tokenize_and_cut,\n",
    "                  preprocessing = my_convert_tokens_to_ids,\n",
    "                  pad_token = BERT_PAD_IDX,\n",
    "                  unk_token = BERT_UNK_IDX, lower = True)\n",
    "\n",
    "BERT_BH_TEXT = data.Field(batch_first = True,\n",
    "                  use_vocab = False,\n",
    "                  tokenize = bert_tokenize_and_cut,\n",
    "                  preprocessing = my_convert_tokens_to_ids,\n",
    "                  pad_token = BERT_PAD_IDX,\n",
    "                  unk_token = BERT_UNK_IDX, lower = True)\n",
    "\n",
    "BERT_EP_TEXT = data.Field(batch_first = True,\n",
    "                  use_vocab = False,\n",
    "                  tokenize = bert_tokenize_and_cut,\n",
    "                  preprocessing = my_convert_tokens_to_ids,\n",
    "                  pad_token = BERT_PAD_IDX,\n",
    "                  unk_token = BERT_UNK_IDX, lower = True)\n",
    "\n",
    "BERT_MAJ_LABEL = data.LabelField(dtype = torch.float)\n",
    "BERT_SCH_LABEL = data.LabelField(dtype = torch.float)\n",
    "BERT_BIP_LABEL = data.LabelField(dtype = torch.float)\n",
    "BERT_MIN_LABEL = data.LabelField(dtype = torch.float)\n",
    "BERT_DEM_LABEL = data.LabelField(dtype = torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_bert_train_data, bert_test_data = NTUHDataset.splits(BERT_BH_TEXT, BERT_EP_TEXT, BERT_ALL_TEXT, \n",
    "                                           BERT_MAJ_LABEL, BERT_SCH_LABEL, BERT_BIP_LABEL, \n",
    "                                                            BERT_MIN_LABEL, BERT_DEM_LABEL)\n",
    "\n",
    "bert_train_data, bert_valid_data = full_bert_train_data.split(random_state = random.seed(SEED), \n",
    "                                                                 split_ratio = TRAIN_RATIO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(vars(bert_train_data.examples[0])['all_text'])\n",
    "tokens = tokenizer.convert_ids_to_tokens(vars(bert_train_data.examples[0])['all_text'])\n",
    "print(tokens)\n",
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Cache for BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_cache={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_attention_masks(ids):\n",
    "        attention_masks = []\n",
    "        for id in ids:\n",
    "            id_mask = [float(i>0) for i in id]            \n",
    "            attention_masks.append(id_mask)\n",
    "        return torch.tensor(attention_masks)\n",
    "    \n",
    "def generate_bert_embedding(sents, sep_token, pad_idx, bert, embedding_dim):\n",
    "    # ID:102 is used to separate sentences\n",
    "    # [batch size, sent len]    \n",
    "    bert = bert.cpu()\n",
    "    bert.eval()\n",
    "    with torch.no_grad():\n",
    "        sep_idxes = (sents == sep_token).nonzero().squeeze(1).data.tolist()\n",
    "        seq_lengths = []\n",
    "        sents_ids = []\n",
    "        pv = -1\n",
    "        for k, v in enumerate(sep_idxes):                \n",
    "            sent_embedding = [pad_idx]*BERT_MAX_SEQUENCE\n",
    "            if k == 0:\n",
    "                seq_lengths.append(v+1)\n",
    "                sent_embedding[:v+1] = sents[:v+1].data.tolist()\n",
    "            else:\n",
    "                seq_lengths.append(v-pv)\n",
    "                sent_embedding[:v-pv] = sents[pv+1:v+1].data.tolist()\n",
    "            sents_ids.append(sent_embedding)\n",
    "            pv = v\n",
    "        attention_masks = create_attention_masks(sents_ids)#.to(device)\n",
    "        sents_ids = torch.tensor(sents_ids)#.to(device)\n",
    "        sent_embeddings = []\n",
    "    \n",
    "        _, _, hidden_states = bert(sents_ids, attention_masks)\n",
    "        #del attention_masks, sents_ids, bert\n",
    "        #torch.cuda.empty_cache()\n",
    "                \n",
    "        token_embeddings = torch.stack(hidden_states[:-1], dim=0)#.cpu()\n",
    "        token_embeddings = token_embeddings.permute(1, 2, 0, 3)\n",
    "        for id, tks in enumerate(token_embeddings):\n",
    "            token_vecs = []\n",
    "            for i in range(seq_lengths[id]):\n",
    "                #cat_vec = torch.cat((tks[i][-1], tks[i][-2], tks[i][-3], tks[i][-4]), dim =0)\n",
    "                sum_all_vec = torch.sum(tks[i][:], dim =0)\n",
    "                token_vecs.append(sum_all_vec)\n",
    "            token_vecs=torch.stack(token_vecs, 0)\n",
    "            sent_embeddings.append(token_vecs)\n",
    "        sent_embeddings = torch.cat(sent_embeddings, 0)                \n",
    "        if sent_embeddings.shape[0] != MAX_SEQUENCE:\n",
    "            sent_embeddings = torch.cat((sent_embeddings, \\\n",
    "                    torch.zeros(MAX_SEQUENCE - sent_embeddings.shape[0], embedding_dim)), 0)\n",
    "        # # sentences, # words, # layers, # features\n",
    "    return sent_embeddings\n",
    "\n",
    "i = 0\n",
    "for data in tqdm(full_bert_train_data):    \n",
    "    t = torch.tensor(data.all_text)\n",
    "    key = ' '.join(str(x) for x in t.data.tolist())\n",
    "    if key not in bert_cache:\n",
    "        sent_embedding = generate_bert_embedding(t, BERT_EOS_IDX, BERT_PAD_IDX, bert,\n",
    "                                                bert.config.to_dict()['hidden_size'])        \n",
    "        bert_cache[key] = sent_embedding\n",
    "    if i % 10 == 0:\n",
    "        torch.save(bert_cache, 'full_text_all_cache.pt')\n",
    "    i+=1\n",
    "torch.save(bert_cache, 'full_text_all_cache.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for data in tqdm(bert_test_data):    \n",
    "    t = torch.tensor(data.all_text)\n",
    "    key = ' '.join(str(x) for x in t.data.tolist())\n",
    "    if key not in bert_cache:\n",
    "        sent_embedding = generate_bert_embedding(t, BERT_EOS_IDX, BERT_PAD_IDX, bert,\n",
    "                                                bert.config.to_dict()['hidden_size'])        \n",
    "        bert_cache[key] = sent_embedding\n",
    "    if i % 10 == 0:\n",
    "        torch.save(bert_cache, 'test_cache.pt')\n",
    "    i+=1\n",
    "torch.save(bert_cache, 'test_cache.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load BERT Cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_cache=torch.load('full_text_all_cache.pt')\n",
    "len(bert_cache)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_MAJ_LABEL.build_vocab(bert_train_data)\n",
    "BERT_SCH_LABEL.build_vocab(bert_train_data)\n",
    "BERT_BIP_LABEL.build_vocab(bert_train_data)\n",
    "BERT_MIN_LABEL.build_vocab(bert_train_data)\n",
    "BERT_DEM_LABEL.build_vocab(bert_train_data)\n",
    "\n",
    "bert_train_iterator, bert_valid_iterator, bert_test_iterator = data.BucketIterator.splits(\n",
    "    (bert_train_data, bert_valid_data, bert_test_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    device = device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokens = tokenizer.convert_ids_to_tokens(next(iter(bert_train_iterator)).all_text[0].data.tolist())\n",
    "print(tokens)\n",
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastTextBERT(nn.Module):\n",
    "    def __init__(self, bert, output_dim, pad_idx, sep_token):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.pad_idx = pad_idx\n",
    "        self.bert = bert\n",
    "        self.sep_token = sep_token\n",
    "        self.bert.eval()\n",
    "        self.embedding_dim = bert.config.to_dict()['hidden_size'] #* 4 # here we concatenate the last four layers\n",
    "        self.fc = nn.Linear(self.embedding_dim, output_dim)\n",
    "    def create_attention_masks(self, ids):\n",
    "        attention_masks = []\n",
    "        for id in ids:\n",
    "            id_mask = [float(i>0) for i in id]            \n",
    "            attention_masks.append(id_mask)\n",
    "        return torch.tensor(attention_masks).to(device)\n",
    "    \n",
    "    def embedding(self, batch):\n",
    "        # ID:102 is used to separate sentences\n",
    "        # [batch size, sent len]\n",
    "        batch_embeddings = []\n",
    "        for sents in batch:      \n",
    "            key = ' '.join(str(x) for x in sents.data.tolist())\n",
    "            key = re.sub(r'(\\s+0)+\\s*', '', key)\n",
    "            if key in bert_cache:\n",
    "                sent_embeddings = bert_cache[key]\n",
    "            else:\n",
    "                sep_idxes = (sents == self.sep_token).nonzero().squeeze(1).data.tolist()\n",
    "                seq_lengths = []\n",
    "                sents_ids = []\n",
    "                pv = -1\n",
    "                for k, v in enumerate(sep_idxes):                \n",
    "                    sent_embedding = [self.pad_idx]*BERT_MAX_SEQUENCE\n",
    "                    if k == 0:\n",
    "                        seq_lengths.append(v+1)\n",
    "                        sent_embedding[:v+1] = sents[:v+1].data.tolist()\n",
    "                    else:\n",
    "                        seq_lengths.append(v-pv)\n",
    "                        sent_embedding[:v-pv] = sents[pv+1:v+1].data.tolist()\n",
    "                    sents_ids.append(sent_embedding)\n",
    "                    pv = v\n",
    "                attention_masks = self.create_attention_masks(sents_ids)\n",
    "                sents_ids = torch.tensor(sents_ids).to(device)\n",
    "                sent_embeddings = []\n",
    "                with torch.no_grad():\n",
    "                    last_hidden_state, _, hidden_states = self.bert(sents_ids, attention_masks)\n",
    "                    token_embeddings = torch.stack(hidden_states[:-1], dim=0)\n",
    "                    token_embeddings = token_embeddings.permute(1, 2, 0, 3)\n",
    "                    for id, tks in enumerate(token_embeddings):\n",
    "                        token_vecs = []\n",
    "                        for i in range(seq_lengths[id]):\n",
    "                            #cat_vec = torch.cat((tks[i][-1], tks[i][-2], tks[i][-3], tks[i][-4]), dim =0)\n",
    "                            sum_all_vec = torch.sum(tks[i][:], dim =0)\n",
    "                            token_vecs.append(sum_all_vec)\n",
    "                            #token_vecs.append(cat_vec)\n",
    "                        token_vecs=torch.stack(token_vecs, 0)\n",
    "                        sent_embeddings.append(token_vecs)\n",
    "                    sent_embeddings = torch.cat(sent_embeddings, 0)                \n",
    "                    if sent_embeddings.shape[0] != MAX_SEQUENCE:\n",
    "                        sent_embeddings = torch.cat((sent_embeddings, \\\n",
    "                                torch.zeros(MAX_SEQUENCE - sent_embeddings.shape[0], self.embedding_dim).to(device)), 0)\n",
    "                    # # sentences, # words, # layers, # features\n",
    "                bert_cache[key] = sent_embeddings\n",
    "            batch_embeddings.append(sent_embeddings.to(device))\n",
    "        batch_embeddings = torch.stack(batch_embeddings, 0)\n",
    "        return batch_embeddings        \n",
    "        \n",
    "    def forward(self, text):        \n",
    "        embedded = self.embedding(text)\n",
    "                \n",
    "        pooled = F.avg_pool2d(embedded, (embedded.shape[1], 1)).squeeze(1) \n",
    "        \n",
    "        return self.fc(pooled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_model = FastTextBERT(bert, OUTPUT_DIM, BERT_PAD_IDX, BERT_EOS_IDX)\n",
    "bert_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_optimizer = optim.Adam([param for param in bert_model.parameters() if param.requires_grad == True])\n",
    "bert_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "bert_model = bert_model.to(device)\n",
    "bert_criterion = bert_criterion.to(device)\n",
    "\n",
    "bert_train_losses, bert_valid_losses, bert_train_accs, bert_valid_accs = \\\n",
    "     train_epoch(N_EPOCHS, bert_model, bert_train_iterator, bert_optimizer, bert_criterion, 0, 'bert1', \n",
    "                 bert_valid_iterator, early_stop= True, period = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, bert_train_losses, bert_valid_losses, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, bert_train_accs, bert_valid_accs, 'Training/Validation F-Measure', {'label': 'Training F-Measure'}, {'label': 'Validation F-Measure'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(bert_model, bert_test_iterator, bert_criterion, 0)\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(bert_model, bert_test_iterator, bert_criterion, 0, 'bert1_fscore')\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_bh_cache=torch.load('bh_text_cache_all.pt')\n",
    "print(len(bert_bh_cache))\n",
    "bert_ep_cache=torch.load('ep_text_cache_all.pt')\n",
    "print(len(bert_ep_cache))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastText2BERT(nn.Module):\n",
    "    def __init__(self, bert, output_dim, pad_idx, sep_token):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.pad_idx = pad_idx\n",
    "        self.bert = bert\n",
    "        self.sep_token = sep_token\n",
    "        self.bert.eval()\n",
    "        self.embedding_dim = bert.config.to_dict()['hidden_size'] #* 4 # here we concatenate the last four layers\n",
    "        self.fc = nn.Linear(self.embedding_dim*2, output_dim)\n",
    "        #self.dropout = nn.Dropout(DROPOUT)\n",
    "        \n",
    "    def create_attention_masks(self, ids):\n",
    "        attention_masks = []\n",
    "        for id in ids:\n",
    "            id_mask = [float(i>0) for i in id]            \n",
    "            attention_masks.append(id_mask)\n",
    "        return torch.tensor(attention_masks).to(device)\n",
    "    \n",
    "    def embedding(self, batch, cache):\n",
    "        # ID:102 is used to separate sentences\n",
    "        # [batch size, sent len]\n",
    "        batch_embeddings = []\n",
    "        for sents in batch:      \n",
    "            key = ' '.join(str(x) for x in sents.data.tolist())\n",
    "            key = re.sub(r'(\\s+0)+\\s*', '', key)\n",
    "            if key in cache:\n",
    "                sent_embeddings = cache[key]\n",
    "            else:\n",
    "                print('Not_found')\n",
    "                return\n",
    "                sep_idxes = (sents == self.sep_token).nonzero().squeeze(1).data.tolist()\n",
    "                seq_lengths = []\n",
    "                sents_ids = []\n",
    "                pv = -1\n",
    "                for k, v in enumerate(sep_idxes):                \n",
    "                    sent_embedding = [self.pad_idx]*BERT_MAX_SEQUENCE\n",
    "                    if k == 0:\n",
    "                        seq_lengths.append(v+1)\n",
    "                        sent_embedding[:v+1] = sents[:v+1].data.tolist()\n",
    "                    else:\n",
    "                        seq_lengths.append(v-pv)\n",
    "                        sent_embedding[:v-pv] = sents[pv+1:v+1].data.tolist()\n",
    "                    sents_ids.append(sent_embedding)\n",
    "                    pv = v\n",
    "                attention_masks = self.create_attention_masks(sents_ids)\n",
    "                sents_ids = torch.tensor(sents_ids).to(device)\n",
    "                sent_embeddings = []\n",
    "                with torch.no_grad():\n",
    "                    last_hidden_state, _, hidden_states = self.bert(sents_ids, attention_masks)\n",
    "                    token_embeddings = torch.stack(hidden_states[:-1], dim=0)\n",
    "                    token_embeddings = token_embeddings.permute(1, 2, 0, 3)\n",
    "                    for id, tks in enumerate(token_embeddings):\n",
    "                        token_vecs = []\n",
    "                        for i in range(seq_lengths[id]):\n",
    "                            #cat_vec = torch.cat((tks[i][-1], tks[i][-2], tks[i][-3], tks[i][-4]), dim =0)\n",
    "                            sum_all_vec = torch.sum(tks[i][:], dim =0)\n",
    "                            token_vecs.append(sum_all_vec)\n",
    "                            #token_vecs.append(cat_vec)\n",
    "                        token_vecs=torch.stack(token_vecs, 0)\n",
    "                        sent_embeddings.append(token_vecs)\n",
    "                    sent_embeddings = torch.cat(sent_embeddings, 0)                \n",
    "                    if sent_embeddings.shape[0] != MAX_SEQUENCE:\n",
    "                        sent_embeddings = torch.cat((sent_embeddings, \\\n",
    "                                torch.zeros(MAX_SEQUENCE - sent_embeddings.shape[0], self.embedding_dim).to(device)), 0)\n",
    "                    # # sentences, # words, # layers, # features\n",
    "                bert_cache[key] = sent_embeddings\n",
    "            batch_embeddings.append(sent_embeddings.to(device))\n",
    "        batch_embeddings = torch.stack(batch_embeddings, 0)\n",
    "        return batch_embeddings        \n",
    "        \n",
    "    def forward(self, bh_text, ep_text):        \n",
    "        embedded1 = self.embedding(bh_text, bert_bh_cache)\n",
    "        embedded2 = self.embedding(ep_text, bert_ep_cache)\n",
    "                \n",
    "        pooled1 = F.avg_pool2d(embedded1, (embedded1.shape[1], 1)).squeeze(1) \n",
    "        pooled2 = F.avg_pool2d(embedded2, (embedded2.shape[1], 1)).squeeze(1) \n",
    "        \n",
    "        return self.fc(torch.cat((pooled1, pooled2), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "bert_model2 = FastText2BERT(bert, OUTPUT_DIM, BERT_PAD_IDX, BERT_EOS_IDX)\n",
    "\n",
    "bert_optimizer = optim.Adam([param for param in bert_model2.parameters() if param.requires_grad == True])\n",
    "bert_criterion = nn.BCEWithLogitsLoss(pos_weight = POS_WEIGHT)\n",
    "bert_model2 = bert_model2.to(device)\n",
    "bert_criterion = bert_criterion.to(device)\n",
    "\n",
    "bert_train_losses2, bert_valid_losses2, bert_train_accs2, bert_valid_accs2 = \\\n",
    "     train_epoch(N_EPOCHS, bert_model2, bert_train_iterator, bert_optimizer, bert_criterion, 1, 'bert2', \n",
    "                 bert_valid_iterator, early_stop = True, period = 30)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(15,10))\n",
    "analysis_plotter(fig, ax1, bert_train_losses2, bert_valid_losses2, 'Training/Validation Loss', {'label': 'Training Loss'}, {'label': 'Validation Loss'})\n",
    "analysis_plotter(fig, ax2, bert_train_accs2, bert_valid_accs2, 'Training/Validation F-score', {'label': 'Training F-score'}, {'label': 'Validation Accuracy'})\n",
    "\n",
    "test_f_scores, predicts = test(bert_model2, bert_test_iterator, bert_criterion, 1)\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_f_scores, predicts = test(bert_model2, bert_test_iterator, bert_criterion, 1, 'bert2_fscore')\n",
    "          \n",
    "for f in test_f_scores:\n",
    "    if f is MICRO or f is MACRO:\n",
    "        print(f'{f}-average:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')\n",
    "    else:\n",
    "        print(f'{NTUHDataset.diagnosis_types[f]}:\\n\\tprecision: {test_f_scores[f][\"p\"]:0.3f}\\n\\trecall: {test_f_scores[f][\"r\"]:0.3f}\\n\\tf-score: {test_f_scores[f][\"f\"]:0.3f}\\n')          "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}