notebooks/Chainer_MultiGPU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "MULTI_GPU = True  # TOGGLE THIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%bash\n",
    "#wget -N https://ikpublictutorial.blob.core.windows.net/deeplearningframeworks/DenseNet_121.caffemodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/cupy/core/fusion.py:659: FutureWarning: cupy.core.fusion is experimental. The interface can change in the future.\n",
      "  util.experimental('cupy.core.fusion')\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import chainer\n",
    "import chainer.functions as F\n",
    "import chainer.links as L\n",
    "import collections\n",
    "from chainer import optimizers, cuda, dataset, training\n",
    "from chainer.training import extensions, updaters, StandardUpdater\n",
    "from chainer.dataset import concat_examples\n",
    "from chainer.links.caffe import CaffeFunction\n",
    "from sklearn.metrics.ranking import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from PIL import Image\n",
    "from chainercv import transforms\n",
    "import random\n",
    "from common.utils import download_data_chextxray, get_imgloc_labels, get_train_valid_test_split\n",
    "from common.utils import compute_roc_auc, get_cuda_version, get_cudnn_version, get_gpu_name\n",
    "from common.params_dense import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCHSIZE = 56"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance Improvement\n",
    "# 1. Auto-tune\n",
    "# This adds very little now .. not sure if True by default?\n",
    "chainer.cuda.set_max_workspace_size(512 * 1024 * 1024)\n",
    "chainer.global_config.autotune = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Chainer:  4.0.0b3\n",
      "CuPy:  4.0.0b3\n",
      "Numpy:  1.14.2\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 9.0.176\n",
      "CuDNN Version  7.0.5\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Chainer: \", chainer.__version__)\n",
    "print(\"CuPy: \", chainer.cuda.cupy.__version__)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPUs:  24\n",
      "GPUs:  4\n"
     ]
    }
   ],
   "source": [
    "CPU_COUNT = multiprocessing.cpu_count()\n",
    "GPU_COUNT = len(get_gpu_name())\n",
    "print(\"CPUs: \", CPU_COUNT)\n",
    "print(\"GPUs: \", GPU_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 1, 2, 3)\n"
     ]
    }
   ],
   "source": [
    "DEVICES=tuple(list(range(GPU_COUNT)))\n",
    "print(DEVICES)\n",
    "if MULTI_GPU:\n",
    "    from cupy.cuda import nccl  # Test that nccl works for multi-gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chestxray/images chestxray/Data_Entry_2017.csv\n"
     ]
    }
   ],
   "source": [
    "# Model-params\n",
    "IMAGENET_RGB_MEAN_CAFFE = np.array([123.68, 116.78, 103.94], dtype=np.float32)\n",
    "IMAGENET_SCALE_FACTOR_CAFFE = 0.017\n",
    "# Paths\n",
    "CSV_DEST = \"chestxray\"\n",
    "IMAGE_FOLDER = os.path.join(CSV_DEST, \"images\")\n",
    "LABEL_FILE = os.path.join(CSV_DEST, \"Data_Entry_2017.csv\")\n",
    "print(IMAGE_FOLDER, LABEL_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0004\n"
     ]
    }
   ],
   "source": [
    "# Manually scale to multi-gpu\n",
    "if MULTI_GPU:\n",
    "    # BATCH is auto-scaled\n",
    "    LR *= GPU_COUNT\n",
    "print(LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please make sure to download\n",
      "https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-linux#download-and-install-azcopy\n",
      "Data already exists\n",
      "CPU times: user 696 ms, sys: 215 ms, total: 911 ms\n",
      "Wall time: 908 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Download data\n",
    "print(\"Please make sure to download\")\n",
    "print(\"https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-linux#download-and-install-azcopy\")\n",
    "download_data_chextxray(CSV_DEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class XrayData(dataset.DatasetMixin):\n",
    "    def __init__(self, patient_ids, height=HEIGHT, width=WIDTH,\n",
    "                 imagenet_mean=IMAGENET_RGB_MEAN_CAFFE, imagenet_scaling = IMAGENET_SCALE_FACTOR_CAFFE,\n",
    "                 img_dir=IMAGE_FOLDER, lbl_file=LABEL_FILE, augmentation=None):\n",
    "          \n",
    "        self.img_locs, self.labels = get_imgloc_labels(img_dir, lbl_file, patient_ids)\n",
    "        self.augmentation = augmentation\n",
    "        self.imagenet_mean = imagenet_mean\n",
    "        self.imagenet_scaling = imagenet_scaling\n",
    "        self.h = height\n",
    "        self.w = width\n",
    "        print(\"Loaded {} labels and {} images\".format(len(self.labels), len(self.img_locs)))\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.img_locs)   \n",
    "    \n",
    "    def get_example(self, idx):\n",
    "        im_file = self.img_locs[idx]\n",
    "        # RGB Image\n",
    "        im_rgb = Image.open(im_file)\n",
    "        im_rgb = self._apply_data_preprocessing(im_rgb)\n",
    "        label = self.labels[idx]\n",
    "        if self.augmentation is not None:\n",
    "            # Random crop to 224, random flip\n",
    "            im_rgb = self._apply_data_augmentation(im_rgb)\n",
    "        else:\n",
    "            # Train/Val resize from 264 to 224\n",
    "            im_rgb = transforms.resize(im_rgb, size=(self.h, self.w))\n",
    "        return np.array(im_rgb, dtype=np.float32), np.array(label, dtype=np.int32)\n",
    "    \n",
    "    def _apply_data_preprocessing(self, rgb_im):\n",
    "        # Array\n",
    "        im = np.asarray(rgb_im, dtype=np.float32)\n",
    "        # (w, h, c) to (c, h, w)\n",
    "        im = im.transpose(2, 0, 1)\n",
    "        # Caffe normalisation\n",
    "        im -= self.imagenet_mean[:, None, None]\n",
    "        im *= self.imagenet_scaling\n",
    "        return im\n",
    "\n",
    "    def _apply_data_augmentation(self, im):\n",
    "        im = transforms.random_crop(im, size=(self.h,self.w))\n",
    "        im = transforms.random_flip(im)\n",
    "        return im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train:21563 valid:3080 test:6162\n"
     ]
    }
   ],
   "source": [
    "train_set, valid_set, test_set = get_train_valid_test_split(TOT_PATIENT_NUMBER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 87306 labels and 87306 images\n",
      "Loaded 7616 labels and 7616 images\n",
      "Loaded 17198 labels and 17198 images\n"
     ]
    }
   ],
   "source": [
    "train_dataset = XrayData(img_dir=IMAGE_FOLDER, lbl_file=LABEL_FILE, patient_ids=train_set, augmentation=True)\n",
    "valid_dataset = XrayData(img_dir=IMAGE_FOLDER, lbl_file=LABEL_FILE, patient_ids=valid_set, augmentation=False)\n",
    "test_dataset  = XrayData(img_dir=IMAGE_FOLDER, lbl_file=LABEL_FILE, patient_ids=test_set, augmentation=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def truncate_bn(sym):\n",
    "    # Need to truncate batchnorm - eps\n",
    "    for layer in list(sym._children):\n",
    "        if \"bn\" in layer:\n",
    "            if sym.__dict__[layer].eps < 1e-5:\n",
    "                sym.__dict__[layer].eps = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CaffeFunctionDenseNet121(CaffeFunction):\n",
    "        \n",
    "    # Standard function saves all variables so cannot use big batch\n",
    "    # This lets me run BATCH of 56 over 32 - still can't get to 64\n",
    "    # https://github.com/chainer/chainer/blob/master/chainer/links/caffe/caffe_function.py#L176\n",
    "    def __call__(self, inputs, **kwargs):\n",
    "        variables = dict(inputs)\n",
    "        # Pools not to save\n",
    "        # These layers are not concatenated\n",
    "        _NOSAVE = set(['pool5', 'concat_5_16', 'concat_4_24', 'concat_3_12', 'concat_2_6'])\n",
    "        # Forward through all layers\n",
    "        for func_name, bottom, top in self.layers:\n",
    "\n",
    "            func = self.forwards[func_name]\n",
    "            # Concat ops require some previous layers that are saved\n",
    "            if \"concat\" in func_name:\n",
    "                input_vars = tuple([variables[bottom[0]], variables['data']])\n",
    "            else:\n",
    "                input_vars = tuple([variables['data']])\n",
    "            output_vars = func(*input_vars)\n",
    "            # Delete layers for concat once used\n",
    "            if \"concat\" in func_name:\n",
    "                del variables[bottom[0]]\n",
    "            if not isinstance(output_vars, collections.Iterable):\n",
    "                output_vars = output_vars,\n",
    "            # Save to dict\n",
    "            variables['data'] = output_vars[0]\n",
    "            top = top[0]\n",
    "            # Save for concat\n",
    "            if (\"pool\" in top) and (top not in _NOSAVE):\n",
    "                variables[top] = output_vars[0]\n",
    "            elif (\"concat\" in top) and (top not in _NOSAVE):\n",
    "                variables[top] = output_vars[0]\n",
    "                \n",
    "        return tuple([variables['data']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DenseNet121(chainer.Chain):\n",
    "    # Class to wrap base (up to pool5 output)\n",
    "    def __init__(self, base_symbol, n_classes=14):\n",
    "        super(DenseNet121, self).__init__()\n",
    "        with self.init_scope():\n",
    "            self.base_symbol = base_symbol\n",
    "            self.fc = L.Linear(1024, n_classes)\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        h = self.base_symbol(inputs={'data':x}, \n",
    "                             outputs=['pool5'])[0]\n",
    "        return self.fc(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_symbol(model_name='densenet121', multi_gpu=MULTI_GPU, classes=CLASSES):\n",
    "    if model_name == 'densenet121':\n",
    "        # Load base\n",
    "        #base_symbol = CaffeFunction(\"DenseNet_121.caffemodel\")\n",
    "        base_symbol = CaffeFunctionDenseNet121(\"DenseNet_121.caffemodel\")\n",
    "        # Fix batch-norm\n",
    "        truncate_bn(base_symbol)\n",
    "        # Remove unused \n",
    "        base_symbol.__delattr__('fc6')\n",
    "        del base_symbol.forwards['fc6']\n",
    "        del base_symbol.layers[-1]\n",
    "        m = DenseNet121(base_symbol, classes)\n",
    "    else:\n",
    "        raise ValueError(\"Unknown model-name\")\n",
    "    # CUDA\n",
    "    if not multi_gpu:\n",
    "        print(\"One GPU\")\n",
    "        chainer.cuda.get_device(0).use()  # Make a specified GPU current\n",
    "        m.to_gpu()  \n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_symbol(sym, lr=LR):\n",
    "    opt = optimizers.Adam(alpha=lr, beta1=0.9, beta2=0.999)\n",
    "    opt.setup(sym)\n",
    "    return opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_roc_auc(data_gt, data_pd, full=True, classes=CLASSES):\n",
    "    roc_auc = []\n",
    "    for i in range(classes):\n",
    "        roc_auc.append(roc_auc_score(data_gt[:, i], data_pd[:, i]))\n",
    "    print(\"Full AUC\", roc_auc)\n",
    "    roc_auc = np.mean(roc_auc)\n",
    "    return roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lossfun(x, t):\n",
    "    return F.sigmoid_cross_entropy(x, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 737 ms, sys: 107 ms, total: 844 ms\n",
      "Wall time: 858 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load symbol\n",
    "predictor = get_symbol()\n",
    "chexnet_sym = L.Classifier(predictor, lossfun=lossfun)\n",
    "# Won't work for multi-class\n",
    "chexnet_sym.compute_accuracy = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.4 ms, sys: 0 ns, total: 1.4 ms\n",
      "Wall time: 1.41 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load optimiser\n",
    "optimizer = init_symbol(chexnet_sym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data-iterators\n",
    "# Open-CV runs on GPU by default!\n",
    "# Weird-bug to set shared_mem (set random value and adjust)\n",
    "if MULTI_GPU:\n",
    "    train_iters = [\n",
    "        chainer.iterators.MultiprocessIterator(\n",
    "            i, BATCHSIZE, n_prefetch=10, n_processes=int(CPU_COUNT/len(DEVICES))) \n",
    "        for i in chainer.datasets.split_dataset_n_random(train_dataset, len(DEVICES))]\n",
    "else:\n",
    "    train_iter = chainer.iterators.MultiprocessIterator(\n",
    "        train_dataset, BATCHSIZE, n_prefetch=10, n_processes=CPU_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These can have a higher batch-size than train since no grads stored\n",
    "valid_iter = chainer.iterators.MultiprocessIterator(\n",
    "    valid_dataset, BATCHSIZE, repeat=False, shuffle=False, n_prefetch=10, n_processes=CPU_COUNT)\n",
    "test_iter = chainer.iterators.MultiprocessIterator(\n",
    "    test_dataset, BATCHSIZE, repeat=False, shuffle=False, n_prefetch=10, n_processes=CPU_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/chainer/training/updaters/multiprocess_parallel_updater.py:136: UserWarning: optimizer.eps is changed to 4e-08 by MultiprocessParallelUpdater for new batch size.\n",
      "  format(optimizer.eps))\n"
     ]
    }
   ],
   "source": [
    "# MultiprocessParallelUpdater requires NCCL.\n",
    "# https://github.com/nvidia/nccl#build--run\n",
    "if MULTI_GPU:\n",
    "    updater = updaters.MultiprocessParallelUpdater(train_iters, optimizer, devices=DEVICES)\n",
    "else:\n",
    "    updater = StandardUpdater(train_iter, optimizer, device=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_interval = (1, 'epoch')\n",
    "trainer = training.Trainer(updater, stop_trigger=(EPOCHS, 'epoch'))\n",
    "trainer.extend(extensions.Evaluator(valid_iter, chexnet_sym, device=DEVICES[0]), trigger=val_interval)\n",
    "trainer.extend(extensions.LogReport(trigger=val_interval))\n",
    "trainer.extend(extensions.PrintReport(['epoch', 'iteration', 'main/loss', 'validation/main/loss']), \n",
    "               trigger=val_interval)\n",
    "trainer.extend(extensions.ProgressBar(update_interval=500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[J     total [###...............................................]  6.41%\n",
      "this epoch [################..................................] 32.07%\n",
      "       500 iter, 0 epoch / 5 epochs\n",
      "       inf iters/sec. Estimated time to finish: 0:00:00.\n",
      "\u001b[4A\u001b[J     total [######............................................] 12.83%\n",
      "this epoch [################################..................] 64.14%\n",
      "      1000 iter, 0 epoch / 5 epochs\n",
      "    1.6663 iters/sec. Estimated time to finish: 1:07:58.055573.\n",
      "\u001b[4A\u001b[J     total [#########.........................................] 19.24%\n",
      "this epoch [################################################..] 96.21%\n",
      "      1500 iter, 0 epoch / 5 epochs\n",
      "    1.6698 iters/sec. Estimated time to finish: 1:02:50.071869.\n",
      "\u001b[4Aepoch       iteration   main/loss   validation/main/loss\n",
      "\u001b[J1           1560        0.153617    0.541447              \n",
      "\u001b[J     total [############......................................] 25.66%\n",
      "this epoch [##############....................................] 28.28%\n",
      "      2000 iter, 1 epoch / 5 epochs\n",
      "    1.6217 iters/sec. Estimated time to finish: 0:59:33.487208.\n",
      "\u001b[4A\u001b[J     total [################..................................] 32.07%\n",
      "this epoch [##############################....................] 60.36%\n",
      "      2500 iter, 1 epoch / 5 epochs\n",
      "    1.6306 iters/sec. Estimated time to finish: 0:54:07.296699.\n",
      "\u001b[4A\u001b[J     total [###################...............................] 38.49%\n",
      "this epoch [##############################################....] 92.43%\n",
      "      3000 iter, 1 epoch / 5 epochs\n",
      "     1.635 iters/sec. Estimated time to finish: 0:48:52.790171.\n",
      "\u001b[4A\u001b[J2           3119        0.140675    0.213281              \n",
      "\u001b[J     total [######################............................] 44.90%\n",
      "this epoch [############......................................] 24.50%\n",
      "      3500 iter, 2 epoch / 5 epochs\n",
      "    1.6207 iters/sec. Estimated time to finish: 0:44:10.173502.\n",
      "\u001b[4A\u001b[J     total [#########################.........................] 51.31%\n",
      "this epoch [############################......................] 56.57%\n",
      "      4000 iter, 2 epoch / 5 epochs\n",
      "    1.6271 iters/sec. Estimated time to finish: 0:38:52.509182.\n",
      "\u001b[4A\u001b[J     total [############################......................] 57.73%\n",
      "this epoch [############################################......] 88.64%\n",
      "      4500 iter, 2 epoch / 5 epochs\n",
      "    1.6312 iters/sec. Estimated time to finish: 0:33:40.129112.\n",
      "\u001b[4A\u001b[J3           4678        0.13239     0.340662              \n",
      "\u001b[J     total [################################..................] 64.14%\n",
      "this epoch [##########........................................] 20.71%\n",
      "      5000 iter, 3 epoch / 5 epochs\n",
      "    1.6191 iters/sec. Estimated time to finish: 0:28:46.343201.\n",
      "\u001b[4A\u001b[J     total [###################################...............] 70.56%\n",
      "this epoch [##########################........................] 52.78%\n",
      "      5500 iter, 3 epoch / 5 epochs\n",
      "    1.6233 iters/sec. Estimated time to finish: 0:23:33.921361.\n",
      "\u001b[4A\u001b[J     total [######################################............] 76.97%\n",
      "this epoch [##########################################........] 84.85%\n",
      "      6000 iter, 3 epoch / 5 epochs\n",
      "    1.6263 iters/sec. Estimated time to finish: 0:18:23.815633.\n",
      "\u001b[4A\u001b[J4           6237        0.122391    0.254132              \n",
      "\u001b[J     total [#########################################.........] 83.38%\n",
      "this epoch [########..........................................] 16.92%\n",
      "      6500 iter, 4 epoch / 5 epochs\n",
      "     1.619 iters/sec. Estimated time to finish: 0:13:19.977504.\n",
      "\u001b[4A\u001b[J     total [############################################......] 89.80%\n",
      "this epoch [########################..........................] 49.00%\n",
      "      7000 iter, 4 epoch / 5 epochs\n",
      "    1.6223 iters/sec. Estimated time to finish: 0:08:10.161848.\n",
      "\u001b[4A\u001b[J     total [################################################..] 96.21%\n",
      "this epoch [########################################..........] 81.07%\n",
      "      7500 iter, 4 epoch / 5 epochs\n",
      "    1.6264 iters/sec. Estimated time to finish: 0:03:01.490248.\n",
      "\u001b[4A\u001b[J5           7796        0.109312    0.226068              \n",
      "\u001b[JCPU times: user 1h 13min 38s, sys: 7min 45s, total: 1h 21min 24s\n",
      "Wall time: 1h 20min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# GPU Memory: 15.8GB (batch=56)\n",
    "# 1 GPU = 1hr5min8s (2.0 iter/sec) \n",
    "# 4 GPU = 1hr20min (1.6 iter/sec)\n",
    "trainer.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Test CheXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 54.3 s, sys: 4.54 s, total: 58.9 s\n",
      "Wall time: 54.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Testing loop: 1min17s\n",
    "y_truth = test_dataset.labels\n",
    "y_guess = []\n",
    "test_iter.reset()\n",
    "with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):\n",
    "    for test_batch in test_iter:\n",
    "        # Data\n",
    "        x_test, y_test = concat_examples(test_batch, device=DEVICES[0])\n",
    "        # Prediction (need to apply sigmoid to turn into probability)\n",
    "        pred = cuda.to_cpu(F.sigmoid(predictor(x_test)).data)\n",
    "        # Collect results\n",
    "        y_guess.append(pred)           \n",
    "# Concatenate\n",
    "y_guess = np.concatenate(y_guess, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full AUC [0.7575668143415677, 0.805527432681448, 0.7038935124616585, 0.8479640325412677, 0.8280486727752898, 0.7817729157168827, 0.6171188093203123, 0.7205446254188392, 0.5601419432626714, 0.7327927688364194, 0.6854968637005807, 0.6944282195416897, 0.7055518030695335, 0.788693085623374]\n",
      "Test AUC: 0.7307\n"
     ]
    }
   ],
   "source": [
    "# Test AUC: 0.73\n",
    "print(\"Test AUC: {0:.4f}\".format(compute_roc_auc(y_truth, y_guess)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}