full_model_mimic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dd162671",
   "metadata": {},
   "source": [
    "### MIMIC CXR\n",
    "\n",
    "* [Dataset](https://physionet.org/content/mimic-cxr/2.0.0/)\n",
    "\n",
    "* [Original Paper](https://www.nature.com/articles/s41597-019-0322-0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2a2e91a",
   "metadata": {},
   "source": [
    "### Setup Environment:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "775f4170",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "from src.classifiers import process_labels, split_data\n",
    "from src.classifiers_base import preprocess_df\n",
    "\n",
    "from transformers import BertTokenizer\n",
    "\n",
    "from src.multimodal_data_loader import VQADataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from src.classifiers_base import train_early_fusion, train_late_fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "435e5833",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'datasets/mimic/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "da5099aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_path = os.path.join(PATH, 'labels.csv')\n",
    "images_path = os.path.join(PATH, 'images')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aed0343",
   "metadata": {},
   "source": [
    "## Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d5950ed9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>race_label</th>\n",
       "      <th>sex_label</th>\n",
       "      <th>disease_label</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>study_id</th>\n",
       "      <th>split</th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>image_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p19/p19702416/s51321189/d85c9f15-f0f84927-761f...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>19702416</td>\n",
       "      <td>51321189</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p19/p19702416/s51321189.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s51321189_d85c9f15-f0f84927-761f30e0-51c2d319-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>p13/p13339704/s51292704/0024603b-12db30e2-ab32...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>13339704</td>\n",
       "      <td>51292704</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p13/p13339704/s51292704.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s51292704_0024603b-12db30e2-ab32c9cb-dae5a3fc-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>p13/p13339704/s51292704/7953848d-2411f0df-859f...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>13339704</td>\n",
       "      <td>51292704</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p13/p13339704/s51292704.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s51292704_7953848d-2411f0df-859f5cea-38c618e0-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>p12/p12668169/s54048859/8a4aaaee-55fcf98f-a036...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12668169</td>\n",
       "      <td>54048859</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p12/p12668169/s54048859.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s54048859_8a4aaaee-55fcf98f-a036a8e7-da71eed1-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>p10/p10309415/s58144222/9886b0fe-9121c65e-c8d7...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10309415</td>\n",
       "      <td>58144222</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p10/p10309415/s58144222.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s58144222_9886b0fe-9121c65e-c8d74649-4b88c530-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153123</th>\n",
       "      <td>p14/p14476373/s53343726/f231fe18-30e5023f-617d...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14476373</td>\n",
       "      <td>53343726</td>\n",
       "      <td>val</td>\n",
       "      <td>datasets/mimic/files/p14/p14476373/s53343726.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s53343726_f231fe18-30e5023f-617d5710-b7343694-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153124</th>\n",
       "      <td>p12/p12491157/s54173393/6aa095e2-8ec1eeae-432f...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12491157</td>\n",
       "      <td>54173393</td>\n",
       "      <td>val</td>\n",
       "      <td>datasets/mimic/files/p12/p12491157/s54173393.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s54173393_6aa095e2-8ec1eeae-432fbe0a-951014ba-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153125</th>\n",
       "      <td>p14/p14036332/s52691805/f52e19e0-9569d75a-7c2e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14036332</td>\n",
       "      <td>52691805</td>\n",
       "      <td>val</td>\n",
       "      <td>datasets/mimic/files/p14/p14036332/s52691805.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s52691805_f52e19e0-9569d75a-7c2e1cca-588fe579-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153126</th>\n",
       "      <td>p10/p10972527/s53691151/f4f75648-baff1e55-0086...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10972527</td>\n",
       "      <td>53691151</td>\n",
       "      <td>val</td>\n",
       "      <td>datasets/mimic/files/p10/p10972527/s53691151.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s53691151_f4f75648-baff1e55-0086d06c-cf27d72e-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153127</th>\n",
       "      <td>p17/p17125250/s53917291/eb3d4f22-ba546ba4-c1c0...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17125250</td>\n",
       "      <td>53917291</td>\n",
       "      <td>val</td>\n",
       "      <td>datasets/mimic/files/p17/p17125250/s53917291.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>s53917291_eb3d4f22-ba546ba4-c1c06deb-40ee5d13-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>153128 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     path  race_label  \\\n",
       "0       p19/p19702416/s51321189/d85c9f15-f0f84927-761f...           0   \n",
       "1       p13/p13339704/s51292704/0024603b-12db30e2-ab32...           2   \n",
       "2       p13/p13339704/s51292704/7953848d-2411f0df-859f...           2   \n",
       "3       p12/p12668169/s54048859/8a4aaaee-55fcf98f-a036...           0   \n",
       "4       p10/p10309415/s58144222/9886b0fe-9121c65e-c8d7...           0   \n",
       "...                                                   ...         ...   \n",
       "153123  p14/p14476373/s53343726/f231fe18-30e5023f-617d...           0   \n",
       "153124  p12/p12491157/s54173393/6aa095e2-8ec1eeae-432f...           0   \n",
       "153125  p14/p14036332/s52691805/f52e19e0-9569d75a-7c2e...           0   \n",
       "153126  p10/p10972527/s53691151/f4f75648-baff1e55-0086...           0   \n",
       "153127  p17/p17125250/s53917291/eb3d4f22-ba546ba4-c1c0...           0   \n",
       "\n",
       "        sex_label disease_label  subject_id  study_id  split  \\\n",
       "0               0             3    19702416  51321189  train   \n",
       "1               0             3    13339704  51292704  train   \n",
       "2               0             3    13339704  51292704  train   \n",
       "3               0             3    12668169  54048859  train   \n",
       "4               0             3    10309415  58144222  train   \n",
       "...           ...           ...         ...       ...    ...   \n",
       "153123          0             0    14476373  53343726    val   \n",
       "153124          0             3    12491157  54173393    val   \n",
       "153125          0             2    14036332  52691805    val   \n",
       "153126          0             3    10972527  53691151    val   \n",
       "153127          1             0    17125250  53917291    val   \n",
       "\n",
       "                                               file_path  \\\n",
       "0       datasets/mimic/files/p19/p19702416/s51321189.txt   \n",
       "1       datasets/mimic/files/p13/p13339704/s51292704.txt   \n",
       "2       datasets/mimic/files/p13/p13339704/s51292704.txt   \n",
       "3       datasets/mimic/files/p12/p12668169/s54048859.txt   \n",
       "4       datasets/mimic/files/p10/p10309415/s58144222.txt   \n",
       "...                                                  ...   \n",
       "153123  datasets/mimic/files/p14/p14476373/s53343726.txt   \n",
       "153124  datasets/mimic/files/p12/p12491157/s54173393.txt   \n",
       "153125  datasets/mimic/files/p14/p14036332/s52691805.txt   \n",
       "153126  datasets/mimic/files/p10/p10972527/s53691151.txt   \n",
       "153127  datasets/mimic/files/p17/p17125250/s53917291.txt   \n",
       "\n",
       "                                                     text  \\\n",
       "0                                        FINAL REPORT\\...   \n",
       "1                                        FINAL REPORT\\...   \n",
       "2                                        FINAL REPORT\\...   \n",
       "3                                        FINAL REPORT\\...   \n",
       "4                                        FINAL REPORT\\...   \n",
       "...                                                   ...   \n",
       "153123                                   FINAL REPORT\\...   \n",
       "153124                                   FINAL REPORT\\...   \n",
       "153125                                   FINAL REPORT\\...   \n",
       "153126                                   FINAL REPORT\\...   \n",
       "153127                                   FINAL REPORT\\...   \n",
       "\n",
       "                                                 image_id  \n",
       "0       s51321189_d85c9f15-f0f84927-761f30e0-51c2d319-...  \n",
       "1       s51292704_0024603b-12db30e2-ab32c9cb-dae5a3fc-...  \n",
       "2       s51292704_7953848d-2411f0df-859f5cea-38c618e0-...  \n",
       "3       s54048859_8a4aaaee-55fcf98f-a036a8e7-da71eed1-...  \n",
       "4       s58144222_9886b0fe-9121c65e-c8d74649-4b88c530-...  \n",
       "...                                                   ...  \n",
       "153123  s53343726_f231fe18-30e5023f-617d5710-b7343694-...  \n",
       "153124  s54173393_6aa095e2-8ec1eeae-432fbe0a-951014ba-...  \n",
       "153125  s52691805_f52e19e0-9569d75a-7c2e1cca-588fe579-...  \n",
       "153126  s53691151_f4f75648-baff1e55-0086d06c-cf27d72e-...  \n",
       "153127  s53917291_eb3d4f22-ba546ba4-c1c06deb-40ee5d13-...  \n",
       "\n",
       "[153128 rows x 10 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(text_path)\n",
    "df.disease_label = df.disease_label.astype(str)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4163974d",
   "metadata": {},
   "source": [
    "## Data Perparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ae0fc946",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 153128/153128 [00:02<00:00, 68288.38it/s] \n",
      "100%|██████████| 153128/153128 [00:07<00:00, 20009.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Shape: (107695, 10)\n",
      "Test Shape: (30360, 10)\n"
     ]
    }
   ],
   "source": [
    "# Select features and labels vectors\n",
    "text_columns = 'text'\n",
    "image_columns = 'image_id'\n",
    "label_columns = 'disease_label'\n",
    "\n",
    "df = preprocess_df(df, image_columns, images_path)\n",
    "\n",
    "# Split the data\n",
    "train_df, test_df = split_data(df)\n",
    "\n",
    "# Process and one-hot encode labels for training set\n",
    "train_labels, mlb, train_columns = process_labels(train_df, col=label_columns)\n",
    "test_labels = process_labels(test_df, col=label_columns, train_columns=train_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5f69e888",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>race_label</th>\n",
       "      <th>sex_label</th>\n",
       "      <th>disease_label</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>study_id</th>\n",
       "      <th>split</th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>image_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p19/p19702416/s51321189/d85c9f15-f0f84927-761f...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>19702416</td>\n",
       "      <td>51321189</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p19/p19702416/s51321189.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s51321189_d85c9f15-f0f84...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>p13/p13339704/s51292704/0024603b-12db30e2-ab32...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>13339704</td>\n",
       "      <td>51292704</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p13/p13339704/s51292704.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s51292704_0024603b-12db3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>p13/p13339704/s51292704/7953848d-2411f0df-859f...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>13339704</td>\n",
       "      <td>51292704</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p13/p13339704/s51292704.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s51292704_7953848d-2411f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>p12/p12668169/s54048859/8a4aaaee-55fcf98f-a036...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12668169</td>\n",
       "      <td>54048859</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p12/p12668169/s54048859.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s54048859_8a4aaaee-55fcf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>p10/p10309415/s58144222/9886b0fe-9121c65e-c8d7...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10309415</td>\n",
       "      <td>58144222</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p10/p10309415/s58144222.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s58144222_9886b0fe-9121c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107690</th>\n",
       "      <td>p17/p17675016/s51152932/3c4ef22b-6452df42-fe88...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>17675016</td>\n",
       "      <td>51152932</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p17/p17675016/s51152932.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s51152932_3c4ef22b-6452d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107691</th>\n",
       "      <td>p11/p11932181/s53058995/a6bfecfe-281e20c1-3d9a...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>11932181</td>\n",
       "      <td>53058995</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p11/p11932181/s53058995.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s53058995_a6bfecfe-281e2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107692</th>\n",
       "      <td>p14/p14987339/s56635090/8ff6567d-509914b7-cc32...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14987339</td>\n",
       "      <td>56635090</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p14/p14987339/s56635090.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s56635090_8ff6567d-50991...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107693</th>\n",
       "      <td>p19/p19647041/s54660985/915784ab-a79bebe9-9f18...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>19647041</td>\n",
       "      <td>54660985</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p19/p19647041/s54660985.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s54660985_915784ab-a79be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107694</th>\n",
       "      <td>p10/p10197137/s50647174/2ff05948-7e4e7c44-c2c3...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10197137</td>\n",
       "      <td>50647174</td>\n",
       "      <td>train</td>\n",
       "      <td>datasets/mimic/files/p10/p10197137/s50647174.txt</td>\n",
       "      <td>FINAL REPORT\\...</td>\n",
       "      <td>datasets/mimic/images/s50647174_2ff05948-7e4e7...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>107695 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     path  race_label  \\\n",
       "0       p19/p19702416/s51321189/d85c9f15-f0f84927-761f...           0   \n",
       "1       p13/p13339704/s51292704/0024603b-12db30e2-ab32...           2   \n",
       "2       p13/p13339704/s51292704/7953848d-2411f0df-859f...           2   \n",
       "3       p12/p12668169/s54048859/8a4aaaee-55fcf98f-a036...           0   \n",
       "4       p10/p10309415/s58144222/9886b0fe-9121c65e-c8d7...           0   \n",
       "...                                                   ...         ...   \n",
       "107690  p17/p17675016/s51152932/3c4ef22b-6452df42-fe88...           0   \n",
       "107691  p11/p11932181/s53058995/a6bfecfe-281e20c1-3d9a...           2   \n",
       "107692  p14/p14987339/s56635090/8ff6567d-509914b7-cc32...           2   \n",
       "107693  p19/p19647041/s54660985/915784ab-a79bebe9-9f18...           1   \n",
       "107694  p10/p10197137/s50647174/2ff05948-7e4e7c44-c2c3...           0   \n",
       "\n",
       "        sex_label disease_label  subject_id  study_id  split  \\\n",
       "0               0             3    19702416  51321189  train   \n",
       "1               0             3    13339704  51292704  train   \n",
       "2               0             3    13339704  51292704  train   \n",
       "3               0             3    12668169  54048859  train   \n",
       "4               0             3    10309415  58144222  train   \n",
       "...           ...           ...         ...       ...    ...   \n",
       "107690          0             3    17675016  51152932  train   \n",
       "107691          1             0    11932181  53058995  train   \n",
       "107692          0             0    14987339  56635090  train   \n",
       "107693          0             3    19647041  54660985  train   \n",
       "107694          1             0    10197137  50647174  train   \n",
       "\n",
       "                                               file_path  \\\n",
       "0       datasets/mimic/files/p19/p19702416/s51321189.txt   \n",
       "1       datasets/mimic/files/p13/p13339704/s51292704.txt   \n",
       "2       datasets/mimic/files/p13/p13339704/s51292704.txt   \n",
       "3       datasets/mimic/files/p12/p12668169/s54048859.txt   \n",
       "4       datasets/mimic/files/p10/p10309415/s58144222.txt   \n",
       "...                                                  ...   \n",
       "107690  datasets/mimic/files/p17/p17675016/s51152932.txt   \n",
       "107691  datasets/mimic/files/p11/p11932181/s53058995.txt   \n",
       "107692  datasets/mimic/files/p14/p14987339/s56635090.txt   \n",
       "107693  datasets/mimic/files/p19/p19647041/s54660985.txt   \n",
       "107694  datasets/mimic/files/p10/p10197137/s50647174.txt   \n",
       "\n",
       "                                                     text  \\\n",
       "0                                        FINAL REPORT\\...   \n",
       "1                                        FINAL REPORT\\...   \n",
       "2                                        FINAL REPORT\\...   \n",
       "3                                        FINAL REPORT\\...   \n",
       "4                                        FINAL REPORT\\...   \n",
       "...                                                   ...   \n",
       "107690                                   FINAL REPORT\\...   \n",
       "107691                                   FINAL REPORT\\...   \n",
       "107692                                   FINAL REPORT\\...   \n",
       "107693                                   FINAL REPORT\\...   \n",
       "107694                                   FINAL REPORT\\...   \n",
       "\n",
       "                                                 image_id  \n",
       "0       datasets/mimic/images/s51321189_d85c9f15-f0f84...  \n",
       "1       datasets/mimic/images/s51292704_0024603b-12db3...  \n",
       "2       datasets/mimic/images/s51292704_7953848d-2411f...  \n",
       "3       datasets/mimic/images/s54048859_8a4aaaee-55fcf...  \n",
       "4       datasets/mimic/images/s58144222_9886b0fe-9121c...  \n",
       "...                                                   ...  \n",
       "107690  datasets/mimic/images/s51152932_3c4ef22b-6452d...  \n",
       "107691  datasets/mimic/images/s53058995_a6bfecfe-281e2...  \n",
       "107692  datasets/mimic/images/s56635090_8ff6567d-50991...  \n",
       "107693  datasets/mimic/images/s54660985_915784ab-a79be...  \n",
       "107694  datasets/mimic/images/s50647174_2ff05948-7e4e7...  \n",
       "\n",
       "[107695 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e8c83780",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b30de2fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = VQADataset(train_df, text_columns, image_columns, label_columns, mlb, train_columns, tokenizer)\n",
    "test_dataset = VQADataset(test_df, text_columns, image_columns, label_columns, mlb, train_columns, tokenizer)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=2)\n",
    "test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32022d26",
   "metadata": {},
   "source": [
    "### Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4bcb2314",
   "metadata": {},
   "outputs": [],
   "source": [
    "if label_columns == 'DR_2':\n",
    "    output_size = 1\n",
    "else:\n",
    "    output_size = len(pd.unique(train_df[label_columns]))\n",
    "multilabel = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2e95aaeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Early Fusion Model:\n",
      "The number of parameters of the model are: 197508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Train early fusion model\n",
    "print(\"Training Early Fusion Model:\")\n",
    "train_early_fusion(train_loader, test_loader, output_size, num_epochs=30, multilabel=multilabel, report=True, lr=0.001, freeze_backbone=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1737581c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train late fusion model\n",
    "print(\"Training Late Fusion Model:\")\n",
    "train_late_fusion(train_loader, test_loader, output_size, num_epochs=30, multilabel=multilabel, report=True, freeze_backbone=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4449f88a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:retina_embeddings_v0_0_1]",
   "language": "python",
   "name": "conda-env-retina_embeddings_v0_0_1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}