MIAS2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4114,
     "status": "ok",
     "timestamp": 1520772109661,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "gaku9wlCpasi",
    "outputId": "2d3d029d-349c-4d95-a21d-63cd3d091191"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tarfile\n",
    "import numpy as np\n",
    "import re\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import os\n",
    "import wget\n",
    "import tensorflow as tf\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "from mammo_utils import extract_tar, read_pgm, download_file\n",
    "\n",
    "\n",
    "# Batch generator\n",
    "def get_batches(X, y, batch_size, distort=True):\n",
    "    # Shuffle X,y\n",
    "    shuffled_idx = np.arange(len(y))\n",
    "    np.random.shuffle(shuffled_idx)\n",
    "    i, h, w, c = X.shape\n",
    "    \n",
    "    # Enumerate indexes by steps of batch_size\n",
    "    for i in range(0, len(y), batch_size):\n",
    "        batch_idx = shuffled_idx[i:i+batch_size]\n",
    "        X_return = X[batch_idx]\n",
    "        \n",
    "        # do random flipping of images\n",
    "        coin = np.random.binomial(1, 0.5, size=None)\n",
    "        if coin and distort:\n",
    "            X_return = X_return[...,::-1,:]\n",
    "        \n",
    "        yield X_return, y[batch_idx]\n",
    "        \n",
    "def read_queue(filename_queue, batch_size=32):\n",
    "    feature = {'image': tf.FixedLenFeature([], tf.string), \n",
    "               'label_class': tf.FixedLenFeature([], tf.int64),\n",
    "               'label_normal': tf.FixedLenFeature([], tf.int64),\n",
    "               'label_pathology': tf.FixedLenFeature([], tf.int64),\n",
    "               'label_type': tf.FixedLenFeature([], tf.int64),\n",
    "              }\n",
    "\n",
    "    reader = tf.TFRecordReader()\n",
    "    _, serialized_example = reader.read(filename_queue)\n",
    "\n",
    "    features = tf.parse_single_example(serialized_example, features=feature)\n",
    " \n",
    "    #Convert the image data from string back to the numbers\n",
    "    image = tf.decode_raw(features['image'], tf.uint8)\n",
    "\n",
    "    # scale the image\n",
    "    image = tf.to_float(image)\n",
    "    image = tf.divide(tf.subtract(image, 128), 255.0)\n",
    "\n",
    "    # Cast label data into int32\n",
    "    label = tf.cast(features['label_class'], tf.int64)\n",
    "\n",
    "    # Reshape image data into the original shape\n",
    "    image = tf.reshape(image, [299, 299, 1])\n",
    "\n",
    "    images, labels = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=30, num_threads=1, min_after_dequeue=10)\n",
    "\n",
    "    return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pmr-RPhPyocJ"
   },
   "source": [
    "## Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "FOesn2sKpl2E"
   },
   "outputs": [],
   "source": [
    "train_labels = np.load(os.path.join(\"data\", \"train_labels.npy\"))\n",
    "test_labels = np.load(os.path.join(\"data\", \"test_labels.npy\"))\n",
    "\n",
    "train_images = np.load(os.path.join(\"data\",\"train_images299.npy\"))\n",
    "test_images = np.load(os.path.join(\"data\",\"test_images299.npy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 217,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 561,
     "status": "error",
     "timestamp": 1520772152108,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "s32N9yLr3HcZ",
    "outputId": "c5ceb976-798e-4b15-876d-e8286a372ff7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr: (2285, 299, 299, 1)\n",
      "y_tr: (2285,)\n",
      "X_cv: (572, 299, 299, 1)\n",
      "y_cv: (572,)\n",
      "\n",
      "Validation distribution:\n",
      " 1    0.300699\n",
      "3    0.283217\n",
      "0    0.230769\n",
      "2    0.185315\n",
      "dtype: float64\n",
      "\n",
      "Training distribution:\n",
      " 1    0.307221\n",
      "3    0.259956\n",
      "0    0.256455\n",
      "2    0.176368\n",
      "dtype: float64\n",
      "\n",
      "Classes: ['BENIGN_calcification' 'BENIGN_mass' 'MALIGNANT_calcification'\n",
      " 'MALIGNANT_mass']\n",
      "Num Classes: 4\n"
     ]
    }
   ],
   "source": [
    "# encode the labels\n",
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(train_labels)\n",
    "y_tr = le.transform(train_labels)\n",
    "y_cv = le.transform(test_labels)\n",
    "\n",
    "y_all = np.concatenate([y_tr, y_cv], axis=0)\n",
    "\n",
    "# pick how to classify the data\n",
    "#y_tr = labels.ABNORMAL.values\n",
    "num_classes = len(np.unique(y_tr))\n",
    "\n",
    "# scale the data\n",
    "X_tr = (train_images - 128) / 255.0\n",
    "X_cv = (test_images - 128) / 255.0\n",
    "\n",
    "X_all = np.concatenate([X_tr, X_cv], axis=0)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_cv, y_tr, y_cv = train_test_split(X_all, y_all, test_size=0.2, random_state=3)\n",
    "\n",
    "print(\"X_tr:\", X_tr.shape)\n",
    "print(\"y_tr:\", y_tr.shape)\n",
    "print(\"X_cv:\", X_cv.shape)\n",
    "print(\"y_cv:\", y_cv.shape)\n",
    "\n",
    "print(\"\\nValidation distribution:\\n\",pd.value_counts(y_cv, normalize=True))\n",
    "print(\"\\nTraining distribution:\\n\",pd.value_counts(y_tr, normalize=True))\n",
    "\n",
    "print(\"\\nClasses:\", le.classes_)\n",
    "print(\"Num Classes:\", num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "A_p8z4_o12KQ"
   },
   "source": [
    "## Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_threads = 16\n",
    "min_after_dequeue = 10000\n",
    "capacity = min_after_dequeue + (num_threads + 1) * batch_size\n",
    "\n",
    "def create_batches(file_names, labels, batch_size=32):\n",
    "    # create tensors\n",
    "    labels = tf.convert_to_tensor(labels, dtype=tf.int32)\n",
    "    files = tf.convert_to_tensor(file_names, dtype=tf.string)\n",
    "    \n",
    "    # create the queue\n",
    "    file_names_q, label_q = tf.train.slice_input_producer([files, labels])\n",
    "    \n",
    "    # read the data\n",
    "    image_queue = tf.read_file(file_names_q)  # convert filenames to content\n",
    "    image_queue = tf.image.decode_jpeg(image_queue, channels=3)\n",
    "    image_queue.set_shape([512, 512, 3])\n",
    "    image_queue = tf.to_float(image_queue)  # convert uint8 to float32\n",
    "    \n",
    "    return tf.train.shuffle_batch([image_queue, label_q], batch_size, capacity, min_after_dequeue, num_threads=num_threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "t7KFYXTb2eSf"
   },
   "outputs": [],
   "source": [
    "# config\n",
    "epochs = 1                 \n",
    "batch_size = 32\n",
    "\n",
    "# learning rate decay variables\n",
    "steps_per_epoch = (1120 * 4) / batch_size\n",
    "\n",
    "## Hyperparameters\n",
    "# Small epsilon value for the BN transform\n",
    "epsilon = 1e-8\n",
    "\n",
    "# learning rate\n",
    "epochs_per_decay = 10\n",
    "starting_rate = 0.003\n",
    "decay_factor = 0.85\n",
    "staircase = True\n",
    "\n",
    "# lambdas\n",
    "lamC = 0.00005\n",
    "lamF = 0.00100\n",
    "\n",
    "# use dropout\n",
    "dropout = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "IuG290bY13qL"
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "# whether to retrain model from scratch or use saved model\n",
    "init = True\n",
    "model_name = \"model_0.0.20\"\n",
    "num_classes = 4\n",
    "\n",
    "with graph.as_default():\n",
    "    #data_set = tf.placeholder(dtype=tf.string)\n",
    "    data_path = os.path.join('data','train.tfrecords')\n",
    "    filename_queue = tf.train.string_input_producer([data_path], num_epochs=1)\n",
    "\n",
    "    # Placeholders\n",
    "    X, y = read_queue(filename_queue)\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 \n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,                  \n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(5, 5),          # Kernel size: 9x9\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        conv1 = tf.layers.batch_normalization(\n",
    "            conv1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn1'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(conv1, name='relu1')\n",
    "      \n",
    "        if dropout:\n",
    "            conv1_bn_relu = tf.layers.dropout(conv1_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    \n",
    "    with tf.name_scope('conv2') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv2 = tf.layers.conv2d(\n",
    "            conv1_bn_relu,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(5, 5),          # Kernel size: 9x9\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn2 = tf.layers.batch_normalization(\n",
    "            conv2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn2'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv2_bn_relu = tf.nn.relu(bn2, name='relu2')\n",
    "      \n",
    "        if dropout:\n",
    "            conv2_bn_relu = tf.layers.dropout(conv2_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        # Max pooling layer 1\n",
    "        pool1 = tf.layers.max_pooling2d(\n",
    "            conv2_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 3x3\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool1 = tf.layers.dropout(pool1, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv3') as scope:\n",
    "        # Convolutional layer 3\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool1,                       # Input data\n",
    "            filters=64,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn3 = tf.layers.batch_normalization(\n",
    "            conv3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn3'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv3_bn_relu = tf.nn.relu(bn3, name='relu3')\n",
    "      \n",
    "        if dropout:\n",
    "            conv3_bn_relu = tf.layers.dropout(conv3_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv4') as scope:\n",
    "        # Convolutional layer 4\n",
    "        conv4 = tf.layers.conv2d(\n",
    "            conv3_bn_relu,                       # Input data\n",
    "            filters=64,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn4 = tf.layers.batch_normalization(\n",
    "            conv4,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn4'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv4_bn_relu = tf.nn.relu(bn4, name='relu4')\n",
    "      \n",
    "        if dropout:\n",
    "            conv4_bn_relu = tf.layers.dropout(conv4_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        # Max pooling layer 2\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv4_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool2'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool2 = tf.layers.dropout(pool2, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        # Convolutional layer 5\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool2,                       # Input data\n",
    "            filters=96,                  # 64 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn5 = tf.layers.batch_normalization(\n",
    "            conv5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn5'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(bn5, name='relu5')\n",
    "      \n",
    "        if dropout:\n",
    "            conv5_bn_relu = tf.layers.dropout(conv5_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv6') as scope:\n",
    "        # Convolutional layer 3\n",
    "        conv6 = tf.layers.conv2d(\n",
    "            conv5_bn_relu,                       # Input data\n",
    "            filters=96,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv6'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn6 = tf.layers.batch_normalization(\n",
    "            conv6,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn6'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv6_bn_relu = tf.nn.relu(bn6, name='relu6')\n",
    "      \n",
    "        if dropout:\n",
    "            conv6_bn_relu = tf.layers.dropout(conv6_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        # Average pooling layer 3\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv6_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool3'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool3 = tf.layers.dropout(pool3, rate=0.25, seed=1, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv7') as scope:\n",
    "        # Convolutional layer 7\n",
    "        conv7 = tf.layers.conv2d(\n",
    "            pool3,                       # Input data\n",
    "            filters=128,                  # 64 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv7'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn7 = tf.layers.batch_normalization(\n",
    "            conv7,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn7'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv7_bn_relu = tf.nn.relu(bn7, name='relu7')\n",
    "      \n",
    "        if dropout:\n",
    "            conv7_bn_relu = tf.layers.dropout(conv7_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv8') as scope:\n",
    "        # Convolutional layer 8\n",
    "        conv8 = tf.layers.conv2d(\n",
    "            conv7_bn_relu,                       # Input data\n",
    "            filters=128,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv8'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn8 = tf.layers.batch_normalization(\n",
    "            conv8,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn8'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv8_bn_relu = tf.nn.relu(bn8, name='relu8')\n",
    "      \n",
    "        if dropout:\n",
    "            conv8_bn_relu = tf.layers.dropout(conv8_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool4') as scope:\n",
    "        # Average pooling layer 4\n",
    "        pool4 = tf.layers.max_pooling2d(\n",
    "            conv8_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool4'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool4 = tf.layers.dropout(pool4, rate=0.25, seed=1, training=training)\n",
    "    \n",
    "    # Flatten output\n",
    "    with tf.name_scope('flatten') as scope:\n",
    "        flat_output = tf.contrib.layers.flatten(pool4)\n",
    "\n",
    "        # dropout at 10%\n",
    "        flat_output = tf.layers.dropout(flat_output, rate=0.1, seed=5, training=training)\n",
    "   \n",
    "    # Fully connected layer 1\n",
    "    with tf.name_scope('fc1') as scope:\n",
    "        fc1 = tf.layers.dense(\n",
    "            flat_output,                 # input\n",
    "            1024,                        # 2048 hidden units\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=4),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc1\"\n",
    "        )\n",
    "        \n",
    "        bn_fc1 = tf.layers.batch_normalization(\n",
    "            fc1,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc1'\n",
    "        )\n",
    "        \n",
    "        fc1_relu = tf.nn.relu(bn_fc1, name='fc1_relu')\n",
    "      \n",
    "        # dropout at 25%\n",
    "        fc1_relu = tf.layers.dropout(fc1_relu, rate=0.5, seed=10, training=training)\n",
    "\n",
    "    # Fully connected layer 2\n",
    "    with tf.name_scope('fc2') as scope:\n",
    "        fc2 = tf.layers.dense(\n",
    "            fc1_relu,                     # input\n",
    "            1024,                       # 1024 hidden units\n",
    "            activation=None,            # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=5),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc2\"\n",
    "        )\n",
    "        \n",
    "        bn_fc2 = tf.layers.batch_normalization(\n",
    "            fc2,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc2'\n",
    "        )\n",
    "        \n",
    "        fc2_relu = tf.nn.relu(bn_fc2, name='fc2_relu')\n",
    "        \n",
    "        # dropout at 10%\n",
    "        fc2_relu = tf.layers.dropout(fc2_relu, rate=0.5, seed=11, training=training)\n",
    "\n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        fc2_relu,                      # input\n",
    "        num_classes,                 # One output unit per category\n",
    "        activation=None,             # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=6),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        name=\"logits\"\n",
    "    )\n",
    "    \n",
    "    # Mean cross-entropy\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    \n",
    "    # Minimize cross-entropy\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int64)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))\n",
    "    \n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    \n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    tf.summary.scalar('cross_entropy', mean_ce)\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    \n",
    "    # Merge all the summaries and write them out to /tmp/mnist_logs (by default)\n",
    "    merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4VeKl2i85tdR"
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mrBLt89h5uNu"
   },
   "outputs": [],
   "source": [
    "## CONFIGURE OPTIONS\n",
    "init = True                   # whether to initialize the model or use a saved version\n",
    "crop = False                  # do random cropping of images?\n",
    "\n",
    "meta_data_every = 5\n",
    "log_to_tensorboard = True\n",
    "print_every = 3                # how often to print metrics\n",
    "checkpoint_every = 1           # how often to save model in epochs\n",
    "use_gpu = False                 # whether or not to use the GPU\n",
    "print_metrics = False          # whether to print or plot metrics, if False a plot will be created and updated every epoch\n",
    "\n",
    "# Placeholders for metrics\n",
    "if init:\n",
    "    valid_acc_values = []\n",
    "    valid_cost_values = []\n",
    "    train_acc_values = []\n",
    "    train_cost_values = []\n",
    "    train_lr_values = []\n",
    "    train_loss_values = []\n",
    "    \n",
    "\n",
    "if use_gpu:\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allocator_type = 'BFC'\n",
    "    config.gpu_options.per_process_gpu_memory_fraction = 0.7\n",
    "else:\n",
    "    config = tf.ConfigProto(device_count = {'GPU': 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "tD012JPf5zJv"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model_0.0.20 ...\n"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    if log_to_tensorboard:\n",
    "        train_writer = tf.summary.FileWriter('./logs/tr_' + model_name, sess.graph)\n",
    "        test_writer = tf.summary.FileWriter('./logs/te_' + model_name)\n",
    "    \n",
    "    if not print_metrics:\n",
    "        # create a plot to be updated as model is trained\n",
    "        f, ax = plt.subplots(1,3,figsize=(20,5))\n",
    "    \n",
    "    # create the saver\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # If the model is new initialize variables, else restore the session\n",
    "    if init:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "    else:\n",
    "        saver.restore(sess, './model/'+model_name+'.ckpt')\n",
    "\n",
    "    # Set seed\n",
    "    np.random.seed(0)\n",
    "    \n",
    "    print(\"Training\", model_name, \"...\")\n",
    "    \n",
    "    # Train several epochs\n",
    "    for epoch in range(epochs):\n",
    "        # Accuracy values (train) after each batch\n",
    "        batch_acc = []\n",
    "        batch_cost = []\n",
    "        batch_loss = []\n",
    "        batch_lr = []\n",
    "        \n",
    "        # only log run metadata once per epoch\n",
    "        write_meta_data = False\n",
    "        run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "        run_metadata = tf.RunMetadata()\n",
    "        _, _, summary, acc_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, mean_ce, loss, global_step, learning_rate],\n",
    "                feed_dict = {\n",
    "                  training: True,\n",
    "                  #data_set: 'train.tfrecords',\n",
    "                },\n",
    "                options=run_options,\n",
    "                run_metadata=run_metadata)\n",
    "        \n",
    "        # Save accuracy (current batch)\n",
    "        batch_acc.append(acc_value)\n",
    "        batch_cost.append(cost_value)\n",
    "        batch_lr.append(lr)\n",
    "        batch_loss.append(loss_value)\n",
    "\n",
    "        # write the summary\n",
    "        train_writer.add_run_metadata(run_metadata, 'step %d' % step)\n",
    "        train_writer.add_summary(summary, step)\n",
    "        write_meta_data = False\n",
    "                \n",
    "        continue\n",
    "        \n",
    "        for X_batch, y_batch in get_batches(X_tr, y_tr, batch_size, distort=True):\n",
    "            if write_meta_data and log_to_tensboard:\n",
    "                # create the metadata\n",
    "                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "                run_metadata = tf.RunMetadata()\n",
    "            \n",
    "                # Run training and evaluate accuracy\n",
    "                _, _, summary, acc_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, mean_ce, loss, global_step, learning_rate]\n",
    "                  #, feed_dict={\n",
    "                  #  X: X_batch,\n",
    "                  #  y: y_batch,\n",
    "                  #  training: True\n",
    "                #}\n",
    "                ,\n",
    "                options=run_options,\n",
    "                run_metadata=run_metadata)\n",
    "\n",
    "                # Save accuracy (current batch)\n",
    "                batch_acc.append(acc_value)\n",
    "                batch_cost.append(cost_value)\n",
    "                batch_lr.append(lr)\n",
    "                batch_loss.append(loss_value)\n",
    "  \n",
    "                # write the summary\n",
    "                train_writer.add_run_metadata(run_metadata, 'step %d' % step)\n",
    "                train_writer.add_summary(summary, step)\n",
    "                write_meta_data = False\n",
    "                \n",
    "            else:\n",
    "                # Run training without meta data\n",
    "                _, _, summary, acc_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, mean_ce, loss, global_step, learning_rate], feed_dict={\n",
    "                    X: X_batch,\n",
    "                    y: y_batch,\n",
    "                    training: True\n",
    "                })\n",
    "\n",
    "                # Save accuracy (current batch)\n",
    "                batch_acc.append(acc_value)\n",
    "                batch_cost.append(cost_value)\n",
    "                batch_lr.append(lr)\n",
    "                batch_loss.append(loss_value)\n",
    "                \n",
    "                # write the summary\n",
    "                if log_to_tensorboard:\n",
    "                    train_writer.add_summary(summary, step)\n",
    "\n",
    "        # save checkpoint every nth epoch\n",
    "        if(epoch % checkpoint_every == 0):\n",
    "            print(\"Saving checkpoint\")\n",
    "            # save the model\n",
    "            save_path = saver.save(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "            # Now that model is saved set init to false so we reload it\n",
    "            init = False\n",
    "        \n",
    "        # init batch arrays\n",
    "        batch_cv_acc = []\n",
    "        batch_cv_cost = []\n",
    "        batch_cv_loss = []\n",
    "        \n",
    "        # Evaluate validation accuracy with batches so as to not crash the GPU\n",
    "        for X_batch, y_batch in get_batches(X_cv, y_cv, batch_size, distort=False):\n",
    "            summary, valid_acc, valid_cost, valid_loss = sess.run([merged, accuracy, mean_ce, loss], feed_dict={\n",
    "                X: X_batch,\n",
    "                y: y_batch,\n",
    "                training: False\n",
    "            })\n",
    "\n",
    "            batch_cv_acc.append(valid_acc)\n",
    "            batch_cv_cost.append(valid_cost)\n",
    "            batch_cv_loss.append(valid_loss)\n",
    "\n",
    "        # Write average of validation data to summary logs\n",
    "        if log_to_tensorboard:\n",
    "            summary = tf.Summary(value=[tf.Summary.Value(tag=\"accuracy\", simple_value=np.mean(batch_cv_acc)),tf.Summary.Value(tag=\"cross_entropy\", simple_value=np.mean(batch_cv_cost)),])\n",
    "            test_writer.add_summary(summary, step)\n",
    "            step += 1\n",
    "            \n",
    "        # take the mean of the values to add to the metrics\n",
    "        valid_acc_values.append(np.mean(batch_cv_acc))\n",
    "        valid_cost_values.append(np.mean(batch_cv_cost))\n",
    "        train_acc_values.append(np.mean(batch_acc))\n",
    "        train_cost_values.append(np.mean(batch_cost))\n",
    "        train_lr_values.append(np.mean(batch_lr))\n",
    "        train_loss_values.append(np.mean(batch_loss))\n",
    "        \n",
    "        if print_metrics:\n",
    "            # Print progress every nth epoch to keep output to reasonable amount\n",
    "            if(epoch % print_every == 0):\n",
    "                print('Epoch {:02d} - step {} - cv acc: {:.3f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "                ))\n",
    "        else:\n",
    "            # update the plot\n",
    "            ax[0].cla()\n",
    "            ax[0].plot(valid_acc_values, color=\"red\", label=\"Validation\")\n",
    "            ax[0].plot(train_acc_values, color=\"blue\", label=\"Training\")\n",
    "            ax[0].set_title('Validation accuracy: {:.4f} (mean last 3)'.format(np.mean(valid_acc_values[-4:])))\n",
    "            \n",
    "            # since we can't zoom in on plots like in tensorboard, scale y axis to give a decent amount of detail\n",
    "            if np.mean(valid_acc_values[-3:]) > 0.90:\n",
    "                ax[0].set_ylim([0.80,1.0])\n",
    "            elif np.mean(valid_acc_values[-3:]) > 0.85:\n",
    "                ax[0].set_ylim([0.75,1.0])\n",
    "            elif np.mean(valid_acc_values[-3:]) > 0.75:\n",
    "                ax[0].set_ylim([0.65,1.0])\n",
    "            elif np.mean(valid_acc_values[-3:]) > 0.65:\n",
    "                ax[0].set_ylim([0.55,1.0])\n",
    "            elif np.mean(valid_acc_values[-3:]) > 0.55:\n",
    "                ax[0].set_ylim([0.45,1.0])           \n",
    "            \n",
    "            ax[0].set_xlabel('Epoch')\n",
    "            ax[0].set_ylabel('Accuracy')\n",
    "            ax[0].legend()\n",
    "            \n",
    "            ax[1].cla()\n",
    "            ax[1].plot(valid_cost_values, color=\"red\", label=\"Validation\")\n",
    "            ax[1].plot(train_cost_values, color=\"blue\", label=\"Training\")\n",
    "            ax[1].set_title('Validation xentropy: {:.3f} (mean last 3)'.format(np.mean(valid_cost_values[-4:])))\n",
    "            ax[1].set_xlabel('Epoch')\n",
    "            ax[1].set_ylabel('Cross Entropy')\n",
    "            ax[1].set_ylim([0,2.0])\n",
    "            ax[1].legend()\n",
    "            \n",
    "            ax[2].cla()\n",
    "            ax[2].plot(train_lr_values)\n",
    "            ax[2].set_title(\"Learning rate: {:.6f}\".format(np.mean(train_lr_values[-1:])))\n",
    "            ax[2].set_xlabel(\"Epoch\")\n",
    "            ax[2].set_ylabel(\"Learning Rate\")\n",
    "            \n",
    "            display.display(plt.gcf())\n",
    "            display.clear_output(wait=True)\n",
    "            \n",
    "        # Print data every 50th epoch so I can write it down to compare models\n",
    "        if (not print_metrics) and (epoch % 50 == 0) and (epoch > 1):\n",
    "            if(epoch % print_every == 0):\n",
    "                print('Epoch {:02d} - step {} - cv acc: {:.4f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "                ))  \n",
    "            \n",
    "    # print results of last epoch\n",
    "    print('Epoch {} - cv acc: {:.4f} - train acc: {:.4f} (mean) - cv cost: {:.3f}'.format(\n",
    "                epochs, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost)\n",
    "            ))\n",
    "    \n",
    "    # save the session\n",
    "    save_path = saver.save(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "    # init the test data array\n",
    "    test_acc_values = []\n",
    "    \n",
    "    # Check on the test data\n",
    "    #for X_batch, y_batch in get_batches(X_te, y_te, batch_size, distort=False):\n",
    "    #    test_accuracy = sess.run(accuracy, feed_dict={\n",
    "    #        X: X_batch,\n",
    "    #        y: y_batch,\n",
    "    #        training: False\n",
    "    #    })\n",
    "    #    test_acc_values.append(test_accuracy)\n",
    "    \n",
    "    # average test accuracy across batches\n",
    "    #test_acc = np.mean(test_acc_values)\n",
    "    \n",
    "# show the plot\n",
    "plt.show()\n",
    "\n",
    "# print results of last epoch\n",
    "print('Epoch {} - cv acc: {:.4f} - train acc: {:.4f} (mean) - cv cost: {:.3f}'.format(\n",
    "      epochs, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost)\n",
    "    ))\n",
    "    \n",
    "# print test accuracy\n",
    "#print(\"Convolutional network accuracy (test set):\",test_acc, \" Validation set:\", valid_acc_values[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    write_meta_data = False\n",
    "    run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "    run_metadata = tf.RunMetadata()\n",
    "    \n",
    "    _, _, summary, acc_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, mean_ce, loss, global_step, learning_rate],\n",
    "                feed_dict = {\n",
    "                  training: True,  \n",
    "                },\n",
    "                options=run_options,\n",
    "                run_metadata=run_metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Old Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "# whether to retrain model from scratch or use saved model\n",
    "init = True\n",
    "model_name = \"model_0.0.9\"\n",
    "\n",
    "with graph.as_default():\n",
    "    # Placeholders\n",
    "    X = tf.placeholder(dtype=tf.float32, shape=[None, 512, 512, 1])\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    training = tf.placeholder(dtype=tf.bool)\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 \n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,                  \n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(7, 7),          # Kernel size: 9x9\n",
    "            strides=(2, 2),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn1 = tf.layers.batch_normalization(\n",
    "            conv1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn1'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(bn1, name='relu1')\n",
    "      \n",
    "        if dropout:\n",
    "            conv1_bn_relu = tf.layers.dropout(conv1_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('pool0') as scope:\n",
    "        # Max pooling layer 0\n",
    "        pool0 = tf.layers.average_pooling2d(\n",
    "            conv1_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 3x3\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool0'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool0 = tf.layers.dropout(pool0, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv2') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv2 = tf.layers.conv2d(\n",
    "            pool0,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 9x9\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn2 = tf.layers.batch_normalization(\n",
    "            conv2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn2'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv2_bn_relu = tf.nn.relu(bn2, name='relu2')\n",
    "      \n",
    "        if dropout:\n",
    "            conv2_bn_relu = tf.layers.dropout(conv2_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        # Max pooling layer 1\n",
    "        pool1 = tf.layers.average_pooling2d(\n",
    "            conv2_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 3x3\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool1 = tf.layers.dropout(pool1, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv3') as scope:\n",
    "        # Convolutional layer 3\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool1,                       # Input data\n",
    "            filters=48,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn3 = tf.layers.batch_normalization(\n",
    "            conv3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn3'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv3_bn_relu = tf.nn.relu(bn3, name='relu3')\n",
    "      \n",
    "        if dropout:\n",
    "            conv3_bn_relu = tf.layers.dropout(conv3_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv4') as scope:\n",
    "        # Convolutional layer 4\n",
    "        conv4 = tf.layers.conv2d(\n",
    "            conv3_bn_relu,                       # Input data\n",
    "            filters=48,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn4 = tf.layers.batch_normalization(\n",
    "            conv4,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn4'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv4_bn_relu = tf.nn.relu(bn4, name='relu4')\n",
    "      \n",
    "        if dropout:\n",
    "            conv4_bn_relu = tf.layers.dropout(conv4_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        # Max pooling layer 2\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv4_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool2'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool2 = tf.layers.dropout(pool2, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        # Convolutional layer 5\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool2,                       # Input data\n",
    "            filters=64,                  # 64 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn5 = tf.layers.batch_normalization(\n",
    "            conv5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn5'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(bn5, name='relu5')\n",
    "      \n",
    "        if dropout:\n",
    "            conv5_bn_relu = tf.layers.dropout(conv5_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        # Average pooling layer 3\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv5_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool3'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool3 = tf.layers.dropout(pool3, rate=0.1, seed=1, training=training)\n",
    "    \n",
    "    # Flatten output\n",
    "    with tf.name_scope('flatten') as scope:\n",
    "        flat_output = tf.contrib.layers.flatten(pool3)\n",
    "\n",
    "        # dropout at 10%\n",
    "        flat_output = tf.layers.dropout(flat_output, rate=0.25, seed=5, training=training)\n",
    "   \n",
    "    # Fully connected layer 1\n",
    "    with tf.name_scope('fc1') as scope:\n",
    "        fc1 = tf.layers.dense(\n",
    "            flat_output,                 # input\n",
    "            512,                        # 2048 hidden units\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=4),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc1\"\n",
    "        )\n",
    "        \n",
    "        bn7 = tf.layers.batch_normalization(\n",
    "            fc1,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn7'\n",
    "        )\n",
    "        \n",
    "        fc1_relu = tf.nn.relu(bn7, name='fc1_relu')\n",
    "      \n",
    "        # dropout at 25%\n",
    "        fc1_relu = tf.layers.dropout(fc1_relu, rate=0.5, seed=10, training=training)\n",
    "\n",
    "    # Fully connected layer 2\n",
    "    with tf.name_scope('fc2') as scope:\n",
    "        fc2 = tf.layers.dense(\n",
    "            fc1_relu,                     # input\n",
    "            256,                       # 1024 hidden units\n",
    "            activation=None,            # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=5),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc2\"\n",
    "        )\n",
    "        \n",
    "        bn8 = tf.layers.batch_normalization(\n",
    "            fc2,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn8'\n",
    "        )\n",
    "        \n",
    "        fc2_relu = tf.nn.relu(bn8, name='fc2_relu')\n",
    "        \n",
    "        # dropout at 10%\n",
    "        fc2_relu = tf.layers.dropout(fc2_relu, rate=0.5, seed=11, training=training)\n",
    "\n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        fc2_relu,                      # input\n",
    "        num_classes,                 # One output unit per category\n",
    "        activation=None,             # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=6),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        name=\"logits\"\n",
    "    )\n",
    "    \n",
    "    # Mean cross-entropy\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    \n",
    "    # Minimize cross-entropy\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))\n",
    "    \n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    \n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    tf.summary.scalar('cross_entropy', mean_ce)\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    \n",
    "    # Merge all the summaries and write them out to /tmp/mnist_logs (by default)\n",
    "    merged = tf.summary.merge_all()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "Pmr-RPhPyocJ"
   ],
   "default_view": {},
   "name": "MIAS.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}