notebooks/PyTorch_MultiGPU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################################################################################\n",
    "# Summary\n",
    "# 1. PyTorch Multi-GPU example\n",
    "# 2. On-the-fly data-augmentation (random crop, random flip)\n",
    "#######################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MULTI_GPU = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torchvision.models as models\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torchvision.transforms as transforms\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from sklearn.metrics.ranking import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from PIL import Image\n",
    "from common.utils import download_data_chextxray, get_imgloc_labels, get_train_valid_test_split\n",
    "from common.utils import compute_roc_auc, get_cuda_version, get_cudnn_version, get_gpu_name\n",
    "from common.utils import yield_mb\n",
    "from common.params_dense import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "PyTorch:  0.3.1.post2\n",
      "Numpy:  1.14.2\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 9.0.176\n",
      "CuDNN Version  7.0.5\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"PyTorch: \", torch.__version__)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPUs:  24\n",
      "GPUs:  4\n"
     ]
    }
   ],
   "source": [
    "CPU_COUNT = multiprocessing.cpu_count()\n",
    "GPU_COUNT = len(get_gpu_name())\n",
    "print(\"CPUs: \", CPU_COUNT)\n",
    "print(\"GPUs: \", GPU_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chestxray/images chestxray/Data_Entry_2017.csv\n"
     ]
    }
   ],
   "source": [
    "# Model-params\n",
    "IMAGENET_RGB_MEAN_TORCH = [0.485, 0.456, 0.406]\n",
    "IMAGENET_RGB_SD_TORCH = [0.229, 0.224, 0.225]\n",
    "# Paths\n",
    "CSV_DEST = \"chestxray\"\n",
    "IMAGE_FOLDER = os.path.join(CSV_DEST, \"images\")\n",
    "LABEL_FILE = os.path.join(CSV_DEST, \"Data_Entry_2017.csv\")\n",
    "print(IMAGE_FOLDER, LABEL_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually scale to multi-gpu\n",
    "if MULTI_GPU:\n",
    "    assert torch.cuda.is_available()\n",
    "     # enables cudnn's auto-tuner\n",
    "    torch.backends.cudnn.benchmark=True\n",
    "    LR *= GPU_COUNT \n",
    "    BATCHSIZE *= GPU_COUNT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please make sure to download\n",
      "https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-linux#download-and-install-azcopy\n",
      "Data already exists\n",
      "CPU times: user 758 ms, sys: 288 ms, total: 1.05 s\n",
      "Wall time: 1.05 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Download data\n",
    "# Wall time: 17min 58s\n",
    "print(\"Please make sure to download\")\n",
    "print(\"https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-linux#download-and-install-azcopy\")\n",
    "download_data_chextxray(CSV_DEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalise by imagenet mean/sd\n",
    "normalize = transforms.Normalize(IMAGENET_RGB_MEAN_TORCH,\n",
    "                                 IMAGENET_RGB_SD_TORCH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class XrayData(Dataset):\n",
    "    def __init__(self, img_dir, lbl_file, patient_ids, transform=None):\n",
    "        \n",
    "        self.img_locs, self.labels = get_imgloc_labels(img_dir, lbl_file, patient_ids)\n",
    "        self.transform = transform\n",
    "        print(\"Loaded {} labels and {} images\".format(len(self.labels), len(self.img_locs)))\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        im_file = self.img_locs[idx]\n",
    "        im_rgb = Image.open(im_file)\n",
    "        label = self.labels[idx]\n",
    "        if self.transform is not None:\n",
    "            im_rgb = self.transform(im_rgb)\n",
    "        return im_rgb, torch.FloatTensor(label)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.img_locs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def no_augmentation_dataset(img_dir, lbl_file, patient_ids, normalize):\n",
    "    dataset = XrayData(img_dir, lbl_file, patient_ids,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.Resize(WIDTH),\n",
    "                           transforms.ToTensor(),  \n",
    "                           normalize]))\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train:21563 valid:3080 test:6162\n"
     ]
    }
   ],
   "source": [
    "train_set, valid_set, test_set = get_train_valid_test_split(TOT_PATIENT_NUMBER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 87306 labels and 87306 images\n"
     ]
    }
   ],
   "source": [
    "# Dataset for training\n",
    "train_dataset = XrayData(img_dir=IMAGE_FOLDER,\n",
    "                         lbl_file=LABEL_FILE,\n",
    "                         patient_ids=train_set,\n",
    "                         transform=transforms.Compose([\n",
    "                             transforms.RandomResizedCrop(size=WIDTH),\n",
    "                             transforms.RandomHorizontalFlip(),\n",
    "                             transforms.ToTensor(),  # need to convert image to tensor!\n",
    "                             normalize]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 7616 labels and 7616 images\n",
      "Loaded 17198 labels and 17198 images\n"
     ]
    }
   ],
   "source": [
    "valid_dataset = no_augmentation_dataset(IMAGE_FOLDER, LABEL_FILE, valid_set, normalize)\n",
    "test_dataset = no_augmentation_dataset(IMAGE_FOLDER, LABEL_FILE, test_set, normalize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_symbol(out_features=CLASSES, multi_gpu=MULTI_GPU):\n",
    "    model = models.densenet.densenet121(pretrained=True)\n",
    "    # Replace classifier (FC-1000) with (FC-14)\n",
    "    model.classifier = nn.Sequential(\n",
    "        nn.Linear(model.classifier.in_features, out_features), \n",
    "        nn.Sigmoid())\n",
    "    if multi_gpu:\n",
    "        model = nn.DataParallel(model)\n",
    "    # CUDA\n",
    "    model.cuda()  \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_symbol(sym, lr=LR):\n",
    "    opt = optim.Adam(sym.parameters(), lr=lr, betas=(0.9, 0.999))\n",
    "    # BCE Loss since classes not mutually exclusive + Sigmoid FC-layer\n",
    "    cri = nn.BCELoss()\n",
    "    sch = ReduceLROnPlateau(opt, factor=0.1, patience=5, mode='min')\n",
    "    return opt, cri, sch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(model, dataloader, optimizer, criterion, epoch, batch=BATCHSIZE):\n",
    "    model.train()\n",
    "    print(\"Training epoch {}\".format(epoch+1))\n",
    "    loss_val = 0\n",
    "    loss_cnt = 0\n",
    "    for data, target in dataloader:\n",
    "        # Get samples\n",
    "        data = Variable(torch.FloatTensor(data).cuda())\n",
    "        target = Variable(torch.FloatTensor(target).cuda())\n",
    "        # Init\n",
    "        optimizer.zero_grad()\n",
    "        # Forwards\n",
    "        output = model(data)\n",
    "        # Loss\n",
    "        loss = criterion(output, target)\n",
    "        # Back-prop\n",
    "        loss.backward()\n",
    "        optimizer.step()   \n",
    "         # Log the loss\n",
    "        loss_val += loss.data[0]\n",
    "        loss_cnt += 1\n",
    "    print(\"Training loss: {0:.4f}\".format(loss_val/loss_cnt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_epoch(model, dataloader, criterion, epoch, phase='valid', \n",
    "                batch=BATCHSIZE, cl=CLASSES):\n",
    "    model.eval()\n",
    "    if phase == 'testing':\n",
    "        print(\"Testing epoch {}\".format(epoch+1))\n",
    "    else:\n",
    "        print(\"Validating epoch {}\".format(epoch+1))\n",
    "    out_pred = torch.FloatTensor().cuda()\n",
    "    out_gt = torch.FloatTensor().cuda()\n",
    "    loss_val = 0\n",
    "    loss_cnt = 0\n",
    "    for data, target in dataloader:\n",
    "        # Get samples\n",
    "        data = Variable(torch.FloatTensor(data).cuda(), volatile=True)\n",
    "        target = Variable(torch.FloatTensor(target).cuda(), volatile=True)\n",
    "         # Forwards\n",
    "        output = model(data)\n",
    "        # Loss\n",
    "        loss = criterion(output, target)\n",
    "        # Log the loss\n",
    "        loss_val += loss.data[0]\n",
    "        loss_cnt += 1\n",
    "        # Log for AUC\n",
    "        out_pred = torch.cat((out_pred, output.data), 0)\n",
    "        out_gt = torch.cat((out_gt, target.data), 0)\n",
    "        \n",
    "    loss_mean = loss_val/loss_cnt\n",
    "    out_gt = out_gt.cpu().numpy()\n",
    "    out_pred = out_pred.cpu().numpy()    \n",
    "    \n",
    "    if phase == 'testing':\n",
    "        print(\"Test-Dataset loss: {0:.4f}\".format(loss_mean))\n",
    "        print(\"Test-Dataset AUC: {0:.4f}\".format(compute_roc_auc(out_gt, out_pred, cl)))\n",
    "    else:\n",
    "        print(\"Validation loss: {0:.4f}\".format(loss_mean))\n",
    "        print(\"Validation AUC: {0:.4f}\".format(compute_roc_auc(out_gt, out_pred, cl)))\n",
    "    return loss_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def print_learning_rate(opt):\n",
    "#    for param_group in opt.param_groups:\n",
    "#        print(\"Learning rate: \", param_group['lr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataLoaders\n",
    "# pin_memory brings errors on 4 GPUs\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCHSIZE,\n",
    "                          shuffle=True, num_workers=CPU_COUNT)\n",
    "valid_loader = DataLoader(dataset=valid_dataset, batch_size=16*BATCHSIZE,\n",
    "                          shuffle=False, num_workers=CPU_COUNT)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=16*BATCHSIZE,\n",
    "                         shuffle=False, num_workers=CPU_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Train CheXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.15 s, sys: 1.12 s, total: 3.27 s\n",
      "Wall time: 3.29 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load symbol\n",
    "chexnet_sym = get_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.26 ms, sys: 0 ns, total: 2.26 ms\n",
      "Wall time: 2.27 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load optimiser, loss\n",
    "optimizer, criterion, scheduler = init_symbol(chexnet_sym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch 1\n",
      "Training loss: 0.1726\n",
      "Validating epoch 1\n",
      "Validation loss: 0.1470\n",
      "Full AUC [0.7827262054948135, 0.8678967857711399, 0.7868547413012692, 0.8992170143507066, 0.8917194602381643, 0.8857012338275809, 0.7189545955150085, 0.7053378911385748, 0.6496618519658628, 0.7558621524717974, 0.6854691633675443, 0.7658281617973451, 0.7675622713999648, 0.8501016713428744]\n",
      "Validation AUC: 0.7866\n",
      "Epoch time: 245 seconds\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Training epoch 2\n",
      "Training loss: 0.1583\n",
      "Validating epoch 2\n",
      "Validation loss: 0.1436\n",
      "Full AUC [0.784890576815251, 0.882323912068192, 0.7823860155608633, 0.9101201447161874, 0.8948739945316595, 0.8831696527066704, 0.7705390269240082, 0.8127793846962925, 0.6550024281377378, 0.813639900352265, 0.723578406539455, 0.7864047339111591, 0.7823377484674239, 0.8725379537993049]\n",
      "Validation AUC: 0.8110\n",
      "Epoch time: 195 seconds\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Training epoch 3\n",
      "Training loss: 0.1557\n",
      "Validating epoch 3\n",
      "Validation loss: 0.1442\n",
      "Full AUC [0.7992570787776844, 0.8869006787363058, 0.7925096294312125, 0.9243751728256561, 0.8990431757148668, 0.9107737869624987, 0.767327204417818, 0.9278332895082829, 0.6499199798141702, 0.8313912788051241, 0.727661500647316, 0.779914513580028, 0.7866385378745325, 0.8593133447750763]\n",
      "Validation AUC: 0.8245\n",
      "Epoch time: 188 seconds\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Training epoch 4\n",
      "Training loss: 0.1541\n",
      "Validating epoch 4\n",
      "Validation loss: 0.1406\n",
      "Full AUC [0.801305149635548, 0.8878876684177773, 0.7937782589393115, 0.9257304413927829, 0.9041708951137644, 0.9082323058036851, 0.7668335585462442, 0.9400341835393111, 0.6586664725041044, 0.8340833821033208, 0.7419760962155469, 0.7872563364375289, 0.7967444639306844, 0.8823458982382648]\n",
      "Validation AUC: 0.8306\n",
      "Epoch time: 189 seconds\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Training epoch 5\n",
      "Training loss: 0.1523\n",
      "Validating epoch 5\n",
      "Validation loss: 0.1399\n",
      "Full AUC [0.8054814532520325, 0.8906402408139048, 0.802081395666074, 0.9195534027334579, 0.9057814485960654, 0.9094429390101094, 0.7670835058229905, 0.9407310018406521, 0.663313879482988, 0.8192214291960688, 0.7246183337398603, 0.7978692609201078, 0.8049379751557589, 0.882774864481159]\n",
      "Validation AUC: 0.8310\n",
      "Epoch time: 189 seconds\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "CPU times: user 14min 38s, sys: 3min 22s, total: 18min\n",
      "Wall time: 16min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 1 GPU - Main training loop: 41min 7s\n",
    "# 4 GPU - Main training loop: 16min 45s\n",
    "loss_min = float(\"inf\")    \n",
    "# Main train/val loop\n",
    "for j in range(EPOCHS):\n",
    "    stime = time.time()\n",
    "    train_epoch(chexnet_sym, train_loader, optimizer, criterion, j)\n",
    "    loss_val = valid_epoch(chexnet_sym, valid_loader, criterion, j)\n",
    "    # LR Schedule\n",
    "    scheduler.step(loss_val)\n",
    "    #print_learning_rate(optimizer)\n",
    "    \n",
    "    # I comment this out to create a fair test against Keras\n",
    "    # Keras cannot checkpoint multi-gpu models at the moment\n",
    "    \n",
    "    #if loss_val < loss_min:\n",
    "    #    print(\"Loss decreased. Saving ...\")\n",
    "    #    loss_min = loss_val\n",
    "    #    torch.save({'epoch': j + 1, \n",
    "    #                'state_dict': chexnet_sym.state_dict(), \n",
    "    #                'best_loss': loss_min, \n",
    "    #                'optimizer' : optimizer.state_dict()}, 'best_chexnet.pth.tar')\n",
    "    \n",
    "    etime = time.time()\n",
    "    print(\"Epoch time: {0:.0f} seconds\".format(etime-stime))\n",
    "    print(\"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Test CheXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 1e+03 ns, total: 5 µs\n",
      "Wall time: 10.5 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load model for testing\n",
    "# I comment this out to create a fair test against Keras\n",
    "#chexnet_sym_test = get_symbol()\n",
    "#chkpt = torch.load(\"best_chexnet.pth.tar\")\n",
    "#chexnet_sym_test.load_state_dict(chkpt['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing epoch 0\n",
      "Test-Dataset loss: 0.1511\n",
      "Full AUC [0.8151923157000877, 0.8759298579325923, 0.8046419265823063, 0.8941065070796208, 0.8885609803069663, 0.9194565806582362, 0.7133234576650442, 0.8601636832768909, 0.6237482379558829, 0.8441295028575039, 0.7526728274292551, 0.801820880646043, 0.7669334058813965, 0.8902362513366339]\n",
      "Test-Dataset AUC: 0.8179\n",
      "CPU times: user 7.13 s, sys: 5.93 s, total: 13.1 s\n",
      "Wall time: 37.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## Evaluate: 37s\n",
    "# AUC: 0.8179\n",
    "#test_loss = valid_epoch(chexnet_sym_test, test_loader, criterion, -1, 'testing')\n",
    "test_loss = valid_epoch(chexnet_sym, test_loader, criterion, -1, 'testing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################################################\n",
    "## Synthetic Data (Pure Training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87296\n"
     ]
    }
   ],
   "source": [
    "# Test on fake-data -> no IO lag\n",
    "batch_in_epoch = len(train_dataset.labels)//BATCHSIZE\n",
    "tot_num = batch_in_epoch * BATCHSIZE\n",
    "print(tot_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "fake_X = np.random.rand(tot_num, 3, 224, 224).astype(np.float32)\n",
    "fake_y = np.random.rand(tot_num, CLASSES).astype(np.float32) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch 1\n",
      "Training loss: 0.7126\n",
      "Training epoch 2\n",
      "Training loss: 0.6933\n",
      "Training epoch 3\n",
      "Training loss: 0.6933\n",
      "Training epoch 4\n",
      "Training loss: 0.6933\n",
      "Training epoch 5\n",
      "Training loss: 0.6933\n",
      "CPU times: user 13min 48s, sys: 1min 50s, total: 15min 38s\n",
      "Wall time: 11min 37s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 4 GPU - Synthetic data: 11min 37s\n",
    "for j in range(EPOCHS):\n",
    "    train_epoch(chexnet_sym, \n",
    "                yield_mb(fake_X, fake_y, BATCHSIZE, shuffle=False),\n",
    "                optimizer, \n",
    "                criterion,\n",
    "                j)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}