Data Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import nibabel as nib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gather_view(root, view_type, grade):\n",
    "    scans = []\n",
    "    count = 0\n",
    "    for GG in os.listdir(root):\n",
    "        if GG == grade:\n",
    "            GG_dir = os.path.join(root, GG)\n",
    "            if os.path.isdir(GG_dir):\n",
    "                for patient in os.listdir(GG_dir):\n",
    "                    patient_dir = os.path.join(GG_dir, patient)\n",
    "                    if os.path.isdir(patient_dir):\n",
    "                        for patient_files in os.listdir(patient_dir):\n",
    "                            if view_type in patient_files:\n",
    "                                scan_root = os.path.join(patient_dir, patient_files)\n",
    "                                img = nib.load(scan_root)\n",
    "                                scans.append(img.get_fdata())\n",
    "                                count += 1\n",
    "                                if count%10 == 0:\n",
    "                                    print(count)\n",
    "                \n",
    "    return np.asarray(scans).astype(np.uint8)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RUN ONE AT A TIME OR ELSE MEMORY ERROR AND YOURE DOOMED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/MICCAI_BraTS_2018_Data_Training'\n",
    "seg_scans_hgg = gather_view(data_path, 'seg.nii.gz', 'HGG')\n",
    "seg_scans_lgg = gather_view(data_path, 'seg.nii.gz', 'LGG')\n",
    "seg_scans = np.concatenate((seg_scans_hgg, seg_scans_lgg), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/MICCAI_BraTS_2018_Data_Validation'\n",
    "flair_scans_hgg = gather_view(data_path, 'flair.nii.gz', 'HGG')\n",
    "flair_scans_lgg = gather_view(data_path, 'flair.nii.gz', 'LGG')\n",
    "flair_scans = np.concatenate((flair_scans_hgg, flair_scans_lgg), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/MICCAI_BraTS_2018_Data_Validation'\n",
    "t1_scans_hgg = gather_view(data_path, 't1.nii.gz', 'HGG')\n",
    "t1_scans_lgg = gather_view(data_path, 't1.nii.gz', 'LGG')\n",
    "t1_scans = np.concatenate((t1_scans_hgg, t1_scans_lgg), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/MICCAI_BraTS_2018_Data_Validation'\n",
    "t1ce_scans_hgg = gather_view(data_path, 't1ce.nii.gz', 'HGG')\n",
    "t1ce_scans_lgg = gather_view(data_path, 't1ce.nii.gz', 'LGG')\n",
    "t1ce_scans = np.concatenate((t1ce_scans_hgg, t1ce_scans_lgg), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/MICCAI_BraTS_2018_Data_Training'\n",
    "t2_scans_hgg = gather_view(data_path, 't2.nii.gz', 'HGG')\n",
    "t2_scans_lgg = gather_view(data_path, 't2.nii.gz', 'LGG')\n",
    "t2_scans = np.concatenate((t2_scans_hgg, t2_scans_lgg), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"data/pkl_files/flair_scans.pkl\", \"rb\") as fp:\n",
    "    flair_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/t1_scans.pkl\", \"rb\") as fp:\n",
    "    t1_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/t2_scans.pkl\", \"rb\") as fp:\n",
    "    t2_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/t1ce_scans.pkl\", \"rb\") as fp:\n",
    "    t1ce_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/seg_scans.pkl\", \"rb\") as fp:\n",
    "    seg_scans = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_scans = flair_scans[:,:,:,30:120:1]\n",
    "t1_scans = t1_scans[:,:,:,30:120:1]\n",
    "t2_scans = t2_scans[:,:,:,30:120:1]\n",
    "t1ce_scans = t1ce_scans[:,:,:,30:120:1]\n",
    "seg_scans = seg_scans[:,:,:,30:120:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_scans = np.transpose(flair_scans, (0,3,1,2))\n",
    "t1_scans = np.transpose(t1_scans, (0,3,1,2))\n",
    "t2_scans = np.transpose(t2_scans, (0,3,1,2))\n",
    "t1ce_scans = np.transpose(t1ce_scans, (0,3,1,2))\n",
    "seg_scans = np.transpose(seg_scans, (0,3,1,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_scans = np.reshape(flair_scans, (25650, 1, 240, 240))\n",
    "t1_scans = np.reshape(t1_scans, (25650, 1, 240, 240))\n",
    "t2_scans = np.reshape(t2_scans, (25650, 1, 240, 240))\n",
    "t1ce_scans = np.reshape(t1ce_scans, (25650, 1, 240, 240))\n",
    "seg_scans = np.reshape(seg_scans, (25650, 1, 240, 240))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stacked_inputs = np.concatenate((flair_scans, t1_scans, t1ce_scans, t2_scans), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DO NOT RUN THE OPEN PICKLE FILES AGAIN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/prelim_inputs_0.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(stacked_inputs[0:12825:1,:,:,:], fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/prelim_inputs_1.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(stacked_inputs[12825:,:,:,:], fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/prelim_outputs.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(seg_scans, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plotting to see the views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# idx = np.random.randint(0,25650)\n",
    "idx = 13756\n",
    "seg_img = seg_scans[idx,0,:,:]\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.title('Ground Truth Segmented Tumor')\n",
    "plt.imshow(seg_img)\n",
    "plt.show()\n",
    "# fig.savefig('example_segmented_ground_truth.png')\n",
    "\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.title('FLAIR View')\n",
    "flair_img = flair_scans[idx,0,:,:]\n",
    "plt.imshow(flair_img)\n",
    "plt.show()\n",
    "# fig.savefig('example_flair_view.png')\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.title('T1 View')\n",
    "t1_img = t1_scans[idx,0,:,:]\n",
    "plt.imshow(t1_img)\n",
    "plt.show()\n",
    "# fig.savefig('example_t1_view.png')\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.title('T2 View')\n",
    "t2_img = t2_scans[idx,0,:,:]\n",
    "plt.imshow(t2_img)\n",
    "plt.show()\n",
    "# fig.savefig('example_t2_view.png')\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.title('T1ce view')\n",
    "t1ce_img = t1ce_scans[idx,0,:,:]\n",
    "plt.imshow(t1ce_img)\n",
    "plt.show()\n",
    "# fig.savefig('example_t1ce_view.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Truncating Images to 30x30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"data/pkl_files/prelim_inputs_0.pkl\", \"rb\") as fp:\n",
    "    prelim_inputs_0 = pickle.load(fp)\n",
    "with open(\"data/pkl_files/prelim_inputs_1.pkl\", \"rb\") as fp:\n",
    "    prelim_inputs_1 = pickle.load(fp)\n",
    "with open(\"data/pkl_files/prelim_outputs.pkl\", \"rb\") as fp:\n",
    "    outputs = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = np.concatenate((prelim_inputs_0, prelim_inputs_1), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_inputs = np.zeros((inputs.shape[0], inputs.shape[1], 32, 32))\n",
    "sampled_outputs = np.zeros((outputs.shape[0], outputs.shape[1], 32, 32))\n",
    "# numpy.random.randint chooses a value from 0 to high_value, non_inclusive.\n",
    "rand_x0_cutoffs = np.random.randint(low=50, high=170, size=inputs.shape[0])\n",
    "rand_y0_cutoffs = np.random.randint(low=50, high=170, size=inputs.shape[0])\n",
    "for idx in range(inputs.shape[0]):\n",
    "    x0 = rand_x0_cutoffs[idx]\n",
    "    y0 = rand_y0_cutoffs[idx]\n",
    "    sampled_inputs[idx] = inputs[idx, : , x0:x0+32, y0:y0+32]\n",
    "    sampled_outputs[idx] = outputs[idx, :, x0:x0+32, y0:y0+32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 4, 240, 240)\n",
      "(25650, 1, 240, 240)\n",
      "(25650, 4, 32, 32)\n",
      "(25650, 1, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "# Sanity check\n",
    "print(inputs.shape)\n",
    "print(outputs.shape)\n",
    "print(sampled_inputs.shape)\n",
    "print(sampled_outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/trimmed_inputs.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(sampled_inputs, fp)\n",
    "with open(\"data/pkl_files/trimmed_outputs.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(sampled_outputs, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 4, 32, 32)\n",
      "(25650, 1, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/pkl_files/trimmed_inputs.pkl\", \"rb\") as fp:\n",
    "    test_inputs = pickle.load(fp)\n",
    "with open(\"data/pkl_files/trimmed_outputs.pkl\", \"rb\") as fp:\n",
    "    test_outputs = pickle.load(fp)\n",
    "print(test_inputs.shape)\n",
    "print(test_outputs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only using 3 views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"data/pkl_files/flair_scans.pkl\", \"rb\") as fp:\n",
    "    flair_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/t1_scans.pkl\", \"rb\") as fp:\n",
    "    t1_scans = pickle.load(fp)\n",
    "with open(\"data/pkl_files/t2_scans.pkl\", \"rb\") as fp:\n",
    "    t2_scans = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_scans = flair_scans[:,:,:,30:120:1]\n",
    "t1_scans = t1_scans[:,:,:,30:120:1]\n",
    "t2_scans = t2_scans[:,:,:,30:120:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_scans = np.reshape(flair_scans, (25650, 1, 240, 240))\n",
    "t1_scans = np.reshape(t1_scans, (25650, 1, 240, 240))\n",
    "t2_scans = np.reshape(t2_scans, (25650, 1, 240, 240))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stacked_inputs = np.concatenate((flair_scans, t1_scans, t2_scans), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/prelim_outputs.pkl\", \"rb\") as fp:\n",
    "    outputs = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_inputs = np.zeros((stacked_inputs.shape[0], stacked_inputs.shape[1], 32, 32))\n",
    "sampled_outputs = np.zeros((outputs.shape[0], outputs.shape[1], 32, 32))\n",
    "# numpy.random.randint chooses a value from 0 to high_value, non_inclusive.\n",
    "rand_x0_cutoffs = np.random.randint(low=50, high=170, size=stacked_inputs.shape[0])\n",
    "rand_y0_cutoffs = np.random.randint(low=50, high=170, size=stacked_inputs.shape[0])\n",
    "for idx in range(stacked_inputs.shape[0]):\n",
    "    x0 = rand_x0_cutoffs[idx]\n",
    "    y0 = rand_y0_cutoffs[idx]\n",
    "    sampled_inputs[idx] = stacked_inputs[idx, : , x0:x0+32, y0:y0+32]\n",
    "    sampled_outputs[idx] = outputs[idx, :, x0:x0+32, y0:y0+32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 1, 240, 240)\n"
     ]
    }
   ],
   "source": [
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 3, 240, 240)\n"
     ]
    }
   ],
   "source": [
    "print(stacked_inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 3, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "print(sampled_inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 1, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "print(sampled_outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/pkl_files/3_chan_trimmed_inputs.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(sampled_inputs, fp)\n",
    "with open(\"data/pkl_files/3_chan_trimmed_outputs.pkl\", \"wb\") as fp:\n",
    "    pickle.dump(sampled_outputs, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25650, 3, 32, 32)\n",
      "(25650, 1, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/pkl_files/3_chan_trimmed_inputs.pkl\", \"rb\") as fp:\n",
    "    test_inputs = pickle.load(fp)\n",
    "with open(\"data/pkl_files/3_chan_trimmed_outputs.pkl\", \"rb\") as fp:\n",
    "    test_outputs = pickle.load(fp)\n",
    "print(test_inputs.shape)\n",
    "print(test_outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}