DDSM_ROI_Slices_299x299_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 123147,
     "status": "ok",
     "timestamp": 1521809102411,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "gaku9wlCpasi",
    "outputId": "25b031ef-3e4f-4e36-a847-e3699dc4535f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tarfile\n",
    "import numpy as np\n",
    "import re\n",
    "import pickle\n",
    "import os\n",
    "import wget\n",
    "from IPython import display\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "## download a file to a location in the data folder\n",
    "def download_file(url, name):\n",
    "    print(\"Downloading \" + name + \"...\")\n",
    "\n",
    "    # check that the data directory exists\n",
    "    try:\n",
    "        os.stat(\"data\")\n",
    "    except:\n",
    "        os.mkdir(\"data\")\n",
    "    fname = wget.download(url, os.path.join('data', name))\n",
    "\n",
    "\n",
    "## Batch generator\n",
    "def get_batches(X, y, batch_size, distort=True):\n",
    "    # Shuffle X,y\n",
    "    shuffled_idx = np.arange(len(y))\n",
    "    np.random.shuffle(shuffled_idx)\n",
    "    i, h, w, c = X.shape\n",
    "\n",
    "    # Enumerate indexes by steps of batch_size\n",
    "    for i in range(0, len(y), batch_size):\n",
    "        batch_idx = shuffled_idx[i:i + batch_size]\n",
    "        X_return = X[batch_idx]\n",
    "\n",
    "        # do random flipping of images\n",
    "        coin = np.random.binomial(1, 0.5, size=None)\n",
    "        if coin and distort:\n",
    "            X_return = X_return[..., ::-1, :]\n",
    "\n",
    "        yield X_return, y[batch_idx]\n",
    "\n",
    "\n",
    "## read data from tfrecords file\n",
    "def read_and_decode_single_example(filenames, label_type='label_normal', normalize=False, num_epochs=None):\n",
    "    filename_queue = tf.train.string_input_producer(filenames, num_epochs=num_epochs)\n",
    "\n",
    "    reader = tf.TFRecordReader()\n",
    "\n",
    "    _, serialized_example = reader.read(filename_queue)\n",
    "    features = tf.parse_single_example(\n",
    "        serialized_example,\n",
    "        features={\n",
    "            'label': tf.FixedLenFeature([], tf.int64),\n",
    "            'label_normal': tf.FixedLenFeature([], tf.int64),\n",
    "            'label_mass': tf.FixedLenFeature([], tf.int64),\n",
    "            'label_benign': tf.FixedLenFeature([], tf.int64),\n",
    "            'image': tf.FixedLenFeature([], tf.string)\n",
    "        })\n",
    "\n",
    "    # extract the data\n",
    "    label = features[label_type]\n",
    "    image = tf.decode_raw(features['image'], tf.uint8)\n",
    "\n",
    "    # reshape and scale the image\n",
    "    image = tf.reshape(image, [299, 299, 1])\n",
    "\n",
    "    if normalize:\n",
    "        image = tf.image.per_image_standardization(image)\n",
    "\n",
    "    #tf.cast(image, dtype=tf.float32)\n",
    "\n",
    "    # return the image and the label\n",
    "    return image, label\n",
    "\n",
    "\n",
    "## load the test data from files\n",
    "def load_validation_data(how=\"class\", percentage=0.5):\n",
    "    X_cv = np.load(os.path.join(\"data\", \"test_data.npy\"))\n",
    "    labels = np.load(os.path.join(\"data\", \"test_labels.npy\"))\n",
    "\n",
    "    # encode the labels appropriately\n",
    "    if how == \"class\":\n",
    "        y_cv = labels\n",
    "    elif how == \"normal\":\n",
    "        y_cv = np.zeros(len(labels))\n",
    "        y_cv[labels != 0] = 1\n",
    "    elif how == \"mass\":\n",
    "        y_cv = np.zeros(len(labels))\n",
    "        y_cv[labels == 1] = 1\n",
    "        y_cv[labels == 3] = 1\n",
    "        y_cv[labels == 2] = 2\n",
    "        y_cv[labels == 4] = 4\n",
    "    elif how == \"benign\":\n",
    "        y_cv = np.zeros(len(labels))\n",
    "        y_cv[labels == 1] = 1\n",
    "        y_cv[labels == 2] = 1\n",
    "        y_cv[labels == 3] = 2\n",
    "        y_cv[labels == 4] = 2\n",
    "\n",
    "    # shuffle the data\n",
    "    X_cv, _, y_cv, _ = train_test_split(X_cv, y_cv, test_size=1 - percentage)\n",
    "\n",
    "    return X_cv, y_cv\n",
    "\n",
    "\n",
    "## evaluate the model to see the predictions\n",
    "def evaluate_model(graph, config):\n",
    "    X_cv, y_cv = load_validation_data(how=\"normal\")\n",
    "\n",
    "    with tf.Session(graph=graph, config=config) as sess:\n",
    "        # create the saver\n",
    "        saver = tf.train.Saver()\n",
    "        sess.run(tf.local_variables_initializer())\n",
    "        saver.restore(sess, './model/' + model_name + '.ckpt')\n",
    "\n",
    "        yhat, test_acc, test_recall = sess.run([predictions, accuracy, rec_op], feed_dict=\n",
    "        {\n",
    "            X: X_cv[0:128],\n",
    "            y: y_cv[0:128],\n",
    "            training: False\n",
    "        })\n",
    "\n",
    "    # initialize the counter\n",
    "    i = 0\n",
    "\n",
    "    # print the results\n",
    "    print(\"Recall:\", test_recall)\n",
    "    print(\"Accuracy:\", test_acc)\n",
    "\n",
    "    # print specifics\n",
    "    for item in yhat:\n",
    "        if item == y_cv[i]:\n",
    "            found = \"*\"\n",
    "        else:\n",
    "            found = \"\"\n",
    "\n",
    "        print(\"Prediction:\", item, \" Actual:\", y_cv[i], found)\n",
    "        i += 1\n",
    "\n",
    "\n",
    "## Download the data if it doesn't already exist\n",
    "def download_data():\n",
    "    if not os.path.exists(os.path.join(\"data\", \"training_0.tfrecords\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/training_0.tfrecords',\n",
    "                          'training_0.tfrecords')\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\", \"training_1.tfrecords\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/training_1.tfrecords',\n",
    "                          'training_1.tfrecords')\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\", \"training_2.tfrecords\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/training_2.tfrecords',\n",
    "                          'training_2.tfrecords')\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\", \"training_3.tfrecords\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/training_3.tfrecords',\n",
    "                          'training_3.tfrecords')\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\", \"test_labels.npy\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/test_labels.npy', 'test_labels.npy')\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\", \"test_data.npy\")):\n",
    "        _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/test_data.npy', 'test_data.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pmr-RPhPyocJ"
   },
   "source": [
    "## Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "uxOrlyz_3YtN"
   },
   "outputs": [],
   "source": [
    "download_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "A_p8z4_o12KQ"
   },
   "source": [
    "## Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1189,
     "status": "ok",
     "timestamp": 1521809105991,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "t7KFYXTb2eSf",
    "outputId": "c3f969c0-336e-4953-c1af-790c83be99ef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps per epoch: 418\n"
     ]
    }
   ],
   "source": [
    "# config\n",
    "epochs = 100                  \n",
    "batch_size = 64\n",
    "\n",
    "## Hyperparameters\n",
    "# Small epsilon value for the BN transform\n",
    "epsilon = 1e-8\n",
    "\n",
    "# learning rate\n",
    "epochs_per_decay = 25\n",
    "starting_rate = 0.002\n",
    "decay_factor = 0.85\n",
    "staircase = True\n",
    "\n",
    "# learning rate decay variables\n",
    "steps_per_epoch = int(26772 / batch_size)\n",
    "print(\"Steps per epoch:\", steps_per_epoch)\n",
    "\n",
    "# lambdas\n",
    "lamC = 0.00000\n",
    "lamF = 0.00100\n",
    "\n",
    "# use dropout\n",
    "dropout = True\n",
    "fcdropout_rate = 0.5\n",
    "convdropout_rate = 0.001\n",
    "pooldropout_rate = 0.1\n",
    "\n",
    "num_classes = 2\n",
    "\n",
    "train_path_0 = os.path.join(\"data\", \"training_0.tfrecords\")\n",
    "train_path_1 = os.path.join(\"data\", \"training_1.tfrecords\")\n",
    "train_path_2 = os.path.join(\"data\", \"training_2.tfrecords\")\n",
    "train_path_3 = os.path.join(\"data\", \"training_3.tfrecords\")\n",
    "test_path = os.path.join(\"data\", \"test.tfrecords\")\n",
    "train_files = [train_path_0, train_path_1, train_path_2, train_path_3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Rcbwk_Phh6LH"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "Graph created...\n"
     ]
    }
   ],
   "source": [
    "## Build the graph\n",
    "graph = tf.Graph()\n",
    "\n",
    "# whether to retrain model from scratch or use saved model\n",
    "init = True\n",
    "model_name = \"model_s0.0.0.12\"\n",
    "# 0.0.0.4 - increase pool3 to 3x3 with stride 3\n",
    "# 0.0.0.6 - reduce pool 3 stride back to 2\n",
    "# 0.0.0.7 - reduce lambda for l2 reg\n",
    "# 0.0.0.8 - increase conv1 to 7x7 stride 2\n",
    "# 0.0.0.9 - disable per image normalization\n",
    "# 0.0.0.10 - commented out batch norm in conv layers, added conv4 and changed stride of convs to 1, increased FC lambda\n",
    "# 0.0.0.11 - turn dropout for conv layers on\n",
    "# 0.0.0.12 - added batch norm after pooling layers, increase pool dropout, decrease conv dropout, added extra conv layer to reduce data dimensionality\n",
    "\n",
    "with graph.as_default():\n",
    "    training = tf.placeholder(dtype=tf.bool, name=\"is_training\")\n",
    "    is_testing = tf.placeholder(dtype=bool, shape=(), name=\"is_testing\")\n",
    "\n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,\n",
    "                                               global_step,\n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,\n",
    "                                               staircase=staircase)\n",
    "\n",
    "    with tf.name_scope('inputs') as scope:\n",
    "        image, label = read_and_decode_single_example(train_files, label_type=\"label_normal\", normalize=False)\n",
    "\n",
    "        X_def, y_def = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=2000,\n",
    "                                              min_after_dequeue=1000)\n",
    "\n",
    "        # Placeholders\n",
    "        X = tf.placeholder_with_default(X_def, shape=[None, 299, 299, 1])\n",
    "        y = tf.placeholder_with_default(y_def, shape=[None])\n",
    "\n",
    "        X = tf.cast(X, dtype=tf.float32)\n",
    "\n",
    "    # Convolutional layer 1\n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X,  # Input data\n",
    "            filters=32,  # 32 filters\n",
    "            kernel_size=(5, 5),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'\n",
    "        )\n",
    "\n",
    "        #conv1 = tf.layers.batch_normalization(\n",
    "        #    conv1,\n",
    "        #    axis=-1,\n",
    "        #    momentum=0.99,\n",
    "        #    epsilon=epsilon,\n",
    "        #    center=True,\n",
    "        #    scale=True,\n",
    "        #    beta_initializer=tf.zeros_initializer(),\n",
    "        #    gamma_initializer=tf.ones_initializer(),\n",
    "        #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "        #    moving_variance_initializer=tf.ones_initializer(),\n",
    "        #    training=training,\n",
    "        #    name='bn1'\n",
    "        #)\n",
    "\n",
    "        # apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(conv1, name='relu1')\n",
    "\n",
    "        if dropout:\n",
    "            conv1_bn_relu = tf.layers.dropout(conv1_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    with tf.name_scope('conv1.1') as scope:\n",
    "        conv11 = tf.layers.conv2d(\n",
    "            conv1_bn_relu,  # Input data\n",
    "            filters=32,  # 32 filters\n",
    "            kernel_size=(5, 5),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1.1'\n",
    "        )\n",
    "\n",
    "        #conv11 = tf.layers.batch_normalization(\n",
    "        #    conv11,\n",
    "        #    axis=-1,\n",
    "        #    momentum=0.99,\n",
    "        #    epsilon=epsilon,\n",
    "        #    center=True,\n",
    "        #    scale=True,\n",
    "        #    beta_initializer=tf.zeros_initializer(),\n",
    "        #    gamma_initializer=tf.ones_initializer(),\n",
    "        #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "        #    moving_variance_initializer=tf.ones_initializer(),\n",
    "        #    training=training,\n",
    "        #    name='bn1.1'\n",
    "        #)\n",
    "\n",
    "        # apply relu\n",
    "        conv11_bn_relu = tf.nn.relu(conv11, name='relu1.1')\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            conv11_bn_relu = tf.layers.dropout(conv11_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 1\n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        pool1 = tf.layers.max_pooling2d(\n",
    "            conv11_bn_relu,  # Input\n",
    "            pool_size=(3, 3),  # Pool size: 3x3\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "\n",
    "        pool1 = tf.layers.batch_normalization(\n",
    "            pool1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_p1'\n",
    "        )\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            pool1 = tf.layers.dropout(pool1, rate=pooldropout_rate, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 2\n",
    "    with tf.name_scope('conv2') as scope:\n",
    "        conv2 = tf.layers.conv2d(\n",
    "            pool1,  # Input data\n",
    "            filters=64,  # 32 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 9x9\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2'\n",
    "        )\n",
    "\n",
    "        #conv2 = tf.layers.batch_normalization(\n",
    "        #    conv2,\n",
    "        #    axis=-1,\n",
    "        #    momentum=0.99,\n",
    "        #    epsilon=epsilon,\n",
    "        #    center=True,\n",
    "        #    scale=True,\n",
    "        #    beta_initializer=tf.zeros_initializer(),\n",
    "        #    gamma_initializer=tf.ones_initializer(),\n",
    "        #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "        #    moving_variance_initializer=tf.ones_initializer(),\n",
    "        #    training=training,\n",
    "        #    name='bn2'\n",
    "        #)\n",
    "\n",
    "        # apply relu\n",
    "        conv2_bn_relu = tf.nn.relu(conv2, name='relu2')\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            conv2_bn_relu = tf.layers.dropout(conv2_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 2\n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv2_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 3x3\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "\n",
    "        pool2 = tf.layers.batch_normalization(\n",
    "            pool2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_p2'\n",
    "        )\n",
    "\n",
    "        # optional dropout\n",
    "        if dropout:\n",
    "            pool2 = tf.layers.dropout(pool2, rate=pooldropout_rate, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 3\n",
    "    with tf.name_scope('conv3') as scope:\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool2,  # Input data\n",
    "            filters=128,  # 48 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3'\n",
    "        )\n",
    "\n",
    "        #conv3 = tf.layers.batch_normalization(\n",
    "        #    conv3,\n",
    "        #    axis=-1,\n",
    "        #    momentum=0.99,\n",
    "        #    epsilon=epsilon,\n",
    "        #    center=True,\n",
    "        #    scale=True,\n",
    "        #    beta_initializer=tf.zeros_initializer(),\n",
    "        #    gamma_initializer=tf.ones_initializer(),\n",
    "        #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "        #    moving_variance_initializer=tf.ones_initializer(),\n",
    "        #    training=training,\n",
    "        #    name='bn3'\n",
    "        #)\n",
    "\n",
    "        # apply relu\n",
    "        conv3_bn_relu = tf.nn.relu(conv3, name='relu3')\n",
    "\n",
    "        if dropout:\n",
    "            conv3_bn_relu = tf.layers.dropout(conv3_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 3\n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv3_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool3'\n",
    "        )\n",
    "\n",
    "        pool3 = tf.layers.batch_normalization(\n",
    "            pool3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_p3'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            pool3 = tf.layers.dropout(pool3, rate=pooldropout_rate, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 4\n",
    "    with tf.name_scope('conv4') as scope:\n",
    "            conv4 = tf.layers.conv2d(\n",
    "                pool3,  # Input data\n",
    "                filters=256,  # 48 filters\n",
    "                kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "                strides=(1, 1),  # Stride: 1\n",
    "                padding='SAME',  # \"same\" padding\n",
    "                activation=None,  # None\n",
    "                kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "                kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "                name='conv4'\n",
    "            )\n",
    "\n",
    "            # conv4 = tf.layers.batch_normalization(\n",
    "            #    conv4,\n",
    "            #    axis=-1,\n",
    "            #    momentum=0.99,\n",
    "            #    epsilon=epsilon,\n",
    "            #    center=True,\n",
    "            #    scale=True,\n",
    "            #    beta_initializer=tf.zeros_initializer(),\n",
    "            #    gamma_initializer=tf.ones_initializer(),\n",
    "            #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "            #    moving_variance_initializer=tf.ones_initializer(),\n",
    "            #    training=training,\n",
    "            #    name='bn4'\n",
    "            # )\n",
    "\n",
    "            # apply relu\n",
    "            conv4_bn_relu = tf.nn.relu(conv4, name='relu4')\n",
    "\n",
    "            if dropout:\n",
    "                conv4_bn_relu = tf.layers.dropout(conv4_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 4\n",
    "    with tf.name_scope('pool4') as scope:\n",
    "            pool4 = tf.layers.max_pooling2d(\n",
    "                conv4_bn_relu,  # Input\n",
    "                pool_size=(2, 2),  # Pool size: 2x2\n",
    "                strides=(2, 2),  # Stride: 2\n",
    "                padding='SAME',  # \"same\" padding\n",
    "                name='pool4'\n",
    "            )\n",
    "\n",
    "            pool4 = tf.layers.batch_normalization(\n",
    "                pool4,\n",
    "                axis=-1,\n",
    "                momentum=0.99,\n",
    "                epsilon=epsilon,\n",
    "                center=True,\n",
    "                scale=True,\n",
    "                beta_initializer=tf.zeros_initializer(),\n",
    "                gamma_initializer=tf.ones_initializer(),\n",
    "                moving_mean_initializer=tf.zeros_initializer(),\n",
    "                moving_variance_initializer=tf.ones_initializer(),\n",
    "                training=training,\n",
    "                name='bn_p4'\n",
    "            )\n",
    "\n",
    "            if dropout:\n",
    "                pool4 = tf.layers.dropout(pool4, rate=pooldropout_rate, seed=1, training=training)\n",
    "\n",
    "            # Convolutional layer 4\n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool4,  # Input data\n",
    "            filters=512,  # 48 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'\n",
    "        )\n",
    "\n",
    "        # conv4 = tf.layers.batch_normalization(\n",
    "        #    conv4,\n",
    "        #    axis=-1,\n",
    "        #    momentum=0.99,\n",
    "        #    epsilon=epsilon,\n",
    "        #    center=True,\n",
    "        #    scale=True,\n",
    "        #    beta_initializer=tf.zeros_initializer(),\n",
    "        #    gamma_initializer=tf.ones_initializer(),\n",
    "        #    moving_mean_initializer=tf.zeros_initializer(),\n",
    "        #    moving_variance_initializer=tf.ones_initializer(),\n",
    "        #    training=training,\n",
    "        #    name='bn4'\n",
    "        # )\n",
    "\n",
    "        # apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(conv5, name='relu5')\n",
    "\n",
    "        if dropout:\n",
    "            conv5_bn_relu = tf.layers.dropout(conv5_bn_relu, rate=convdropout_rate, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 4\n",
    "    with tf.name_scope('pool5') as scope:\n",
    "        pool5 = tf.layers.max_pooling2d(\n",
    "            conv5_bn_relu,\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',\n",
    "            name='pool5'\n",
    "        )\n",
    "\n",
    "        pool5 = tf.layers.batch_normalization(\n",
    "            pool5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_p5'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            pool5 = tf.layers.dropout(pool5, rate=pooldropout_rate, seed=1, training=training)\n",
    "\n",
    "    # Flatten output\n",
    "    with tf.name_scope('flatten') as scope:\n",
    "        flat_output = tf.contrib.layers.flatten(pool5)\n",
    "\n",
    "        # dropout at fc rate\n",
    "        flat_output = tf.layers.dropout(flat_output, rate=fcdropout_rate, seed=5, training=training)\n",
    "\n",
    "    # Fully connected layer 1\n",
    "    with tf.name_scope('fc1') as scope:\n",
    "        fc1 = tf.layers.dense(\n",
    "            flat_output,\n",
    "            1024,\n",
    "            activation=None,\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=4),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc1\"\n",
    "        )\n",
    "\n",
    "        bn_fc1 = tf.layers.batch_normalization(\n",
    "            fc1,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc1'\n",
    "        )\n",
    "\n",
    "        fc1_relu = tf.nn.relu(bn_fc1, name='fc1_relu')\n",
    "\n",
    "        # dropout\n",
    "        fc1_relu = tf.layers.dropout(fc1_relu, rate=fcdropout_rate, seed=10, training=training)\n",
    "\n",
    "    # Fully connected layer 2\n",
    "    with tf.name_scope('fc2') as scope:\n",
    "        fc2 = tf.layers.dense(\n",
    "            fc1_relu,  # input\n",
    "            1024,  # 1024 hidden units\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=5),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc2\"\n",
    "        )\n",
    "\n",
    "        bn_fc2 = tf.layers.batch_normalization(\n",
    "            fc2,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc2'\n",
    "        )\n",
    "\n",
    "        fc2_relu = tf.nn.relu(bn_fc2, name='fc2_relu')\n",
    "\n",
    "        # dropout\n",
    "        fc2_relu = tf.layers.dropout(fc2_relu, rate=fcdropout_rate, seed=11, training=training)\n",
    "\n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        fc2_relu,\n",
    "        num_classes,      # One output unit per category\n",
    "        activation=None,  # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=6),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        name=\"logits\"\n",
    "    )\n",
    "\n",
    "    with tf.variable_scope('conv1', reuse=True):\n",
    "        conv_kernels1 = tf.get_variable('kernel')\n",
    "        kernel_transposed = tf.transpose(conv_kernels1, [3, 0, 1, 2])\n",
    "\n",
    "    with tf.variable_scope('visualization'):\n",
    "        tf.summary.image('conv1/filters', kernel_transposed, max_outputs=32)\n",
    "\n",
    "    ## Loss function options\n",
    "    # Regular mean cross entropy\n",
    "    # mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "\n",
    "    # weighted mean cross entropy\n",
    "    # onehot_labels = tf.one_hot(y, depth=num_classes)\n",
    "    # mean_ce = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(targets=tf.one_hot(y, depth=num_classes), logits=logits, pos_weight=classes_weights))\n",
    "\n",
    "    # Different weighting method\n",
    "    # This will weight the positive examples higher so as to improve recall\n",
    "    weights = tf.multiply(3, tf.cast(tf.equal(y, 1), tf.int32)) + 1\n",
    "    mean_ce = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits, weights=weights))\n",
    "\n",
    "    # Add in l2 loss\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "\n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize cross-entropy\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int64)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))\n",
    "\n",
    "    # calculate recall\n",
    "    if num_classes > 2:\n",
    "        recall = [0] * num_classes\n",
    "        rec_op = [[]] * num_classes\n",
    "\n",
    "        for k in range(num_classes):\n",
    "            recall[k], rec_op[k] = tf.metrics.recall(\n",
    "                labels=tf.equal(y, k),\n",
    "                predictions=tf.equal(predictions, k)\n",
    "            )\n",
    "    else:\n",
    "        recall, rec_op = tf.metrics.recall(labels=y, predictions=predictions, name=\"recall\")\n",
    "        precision, prec_op = tf.metrics.precision(labels=y, predictions=predictions, name=\"precision\")\n",
    "        f1_score = 2 * ( (precision * recall) / (precision + recall))\n",
    "\n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    tf.summary.scalar('recall_1', recall)\n",
    "    tf.summary.scalar('cross_entropy', mean_ce)\n",
    "\n",
    "    if num_classes == 2:\n",
    "        tf.summary.scalar('precision', precision)\n",
    "        tf.summary.scalar('f1_score', f1_score)\n",
    "\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "\n",
    "    # Merge all the summaries and write them out to /tmp/mnist_logs (by default)\n",
    "    merged = tf.summary.merge_all()\n",
    "\n",
    "    print(\"Graph created...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Graph created...\n"
     ]
    }
   ],
   "source": [
    "## Build the graph\n",
    "graph = tf.Graph()\n",
    "# whether to retrain model from scratch or use saved model\n",
    "init = True\n",
    "model_name = \"model_s0.2.04\"\n",
    "# 0.2.01 - trying to do more convolutions at first, and then rapidly decrease dimensionality\n",
    "\n",
    "with graph.as_default():\n",
    "    training = tf.placeholder(dtype=tf.bool, name=\"is_training\")\n",
    "    is_testing = tf.placeholder(dtype=bool, shape=(), name=\"is_testing\")\n",
    "\n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,\n",
    "                                               global_step,\n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,\n",
    "                                               staircase=staircase)\n",
    "\n",
    "    with tf.name_scope('inputs') as scope:\n",
    "        image, label = read_and_decode_single_example(train_files, label_type=\"label_normal\")\n",
    "\n",
    "        X_def, y_def = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=2000,\n",
    "                                              min_after_dequeue=1000)\n",
    "\n",
    "        # Placeholders\n",
    "        X = tf.placeholder_with_default(X_def, shape=[None, 299, 299, 1])\n",
    "        y = tf.placeholder_with_default(y_def, shape=[None])\n",
    "\n",
    "    # Convolutional layer 1\n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X,  # Input data\n",
    "            filters=64,  # 32 filters\n",
    "            kernel_size=(5, 5),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'\n",
    "        )\n",
    "\n",
    "        conv1 = tf.layers.batch_normalization(\n",
    "            conv1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(conv1, name='relu1')\n",
    "\n",
    "        if dropout:\n",
    "            conv1_bn_relu = tf.layers.dropout(conv1_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    with tf.name_scope('conv1.1') as scope:\n",
    "        conv11 = tf.layers.conv2d(\n",
    "            conv1_bn_relu,  # Input data\n",
    "            filters=64,  # 32 filters\n",
    "            kernel_size=(5, 5),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1.1'\n",
    "        )\n",
    "\n",
    "        conv11 = tf.layers.batch_normalization(\n",
    "            conv11,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn1.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv11_bn_relu = tf.nn.relu(conv11, name='relu1.1')\n",
    "\n",
    "        if dropout:\n",
    "            conv11_bn_relu = tf.layers.dropout(conv11_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 1\n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        pool1 = tf.layers.max_pooling2d(\n",
    "            conv11_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 3x3\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool1 = tf.layers.dropout(pool1, rate=0.1, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 2\n",
    "    with tf.name_scope('conv2') as scope:\n",
    "        conv2 = tf.layers.conv2d(\n",
    "            pool1,  # Input data\n",
    "            filters=96,  # 32 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 9x9\n",
    "            strides=(2, 2),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2'\n",
    "        )\n",
    "\n",
    "        conv2 = tf.layers.batch_normalization(\n",
    "            conv2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn2'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv2_bn_relu = tf.nn.relu(conv2, name='relu2')\n",
    "\n",
    "        if dropout:\n",
    "            conv2_bn_relu = tf.layers.dropout(conv2_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    with tf.name_scope('conv2.1') as scope:\n",
    "        conv21 = tf.layers.conv2d(\n",
    "            conv2_bn_relu,  # Input data\n",
    "            filters=96,  # 32 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 9x9\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2.1'\n",
    "        )\n",
    "\n",
    "        conv21 = tf.layers.batch_normalization(\n",
    "            conv21,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn2.1'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv21_bn_relu = tf.nn.relu(conv21, name='relu2.1')\n",
    "\n",
    "        if dropout:\n",
    "            conv21_bn_relu = tf.layers.dropout(conv21_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 2\n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv21_bn_relu,  # Input\n",
    "            pool_size=(3, 3),  # Pool size: 3x3\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool2 = tf.layers.dropout(pool2, rate=0.1, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 3\n",
    "    with tf.name_scope('conv3') as scope:\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool2,  # Input data\n",
    "            filters=128,  # 48 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3'\n",
    "        )\n",
    "\n",
    "        bn3 = tf.layers.batch_normalization(\n",
    "            conv3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn3'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv3_bn_relu = tf.nn.relu(bn3, name='relu3')\n",
    "\n",
    "        if dropout:\n",
    "            conv3_bn_relu = tf.layers.dropout(conv3_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 3\n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv3_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool3'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool3 = tf.layers.dropout(pool3, rate=0.1, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 4\n",
    "    with tf.name_scope('conv4') as scope:\n",
    "        conv4 = tf.layers.conv2d(\n",
    "            pool3,  # Input data\n",
    "            filters=192,  # 64 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 3x3\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4'\n",
    "        )\n",
    "\n",
    "        bn4 = tf.layers.batch_normalization(\n",
    "            conv4,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn4'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv4_bn_relu = tf.nn.relu(bn4, name='relu4')\n",
    "\n",
    "        if dropout:\n",
    "            conv4_bn_relu = tf.layers.dropout(conv4_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 4\n",
    "    with tf.name_scope('pool4') as scope:\n",
    "        pool4 = tf.layers.max_pooling2d(\n",
    "            conv4_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool4'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool4 = tf.layers.dropout(pool4, rate=0.25, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 5\n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool4,  # Input data\n",
    "            filters=256,  # 64 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'\n",
    "        )\n",
    "\n",
    "        bn5 = tf.layers.batch_normalization(\n",
    "            conv5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn5'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(bn5, name='relu5')\n",
    "\n",
    "        if dropout:\n",
    "            conv5_bn_relu = tf.layers.dropout(conv5_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    # Max pooling layer 5\n",
    "    with tf.name_scope('pool5') as scope:\n",
    "        pool5 = tf.layers.max_pooling2d(\n",
    "            conv5_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool5'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool5 = tf.layers.dropout(pool5, rate=0.1, seed=1, training=training)\n",
    "\n",
    "    # Convolutional layer 6\n",
    "    with tf.name_scope('conv6') as scope:\n",
    "        conv6 = tf.layers.conv2d(\n",
    "            pool5,  # Input data\n",
    "            filters=512,  # 48 filters\n",
    "            kernel_size=(3, 3),  # Kernel size: 5x5\n",
    "            strides=(1, 1),  # Stride: 1\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv6'\n",
    "        )\n",
    "\n",
    "        bn6 = tf.layers.batch_normalization(\n",
    "            conv6,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn6'\n",
    "        )\n",
    "\n",
    "        # apply relu\n",
    "        conv6_bn_relu = tf.nn.relu(bn6, name='relu6')\n",
    "\n",
    "        if dropout:\n",
    "            conv6_bn_relu = tf.layers.dropout(conv6_bn_relu, rate=0.1, seed=9, training=training)\n",
    "\n",
    "    with tf.name_scope('pool6') as scope:\n",
    "        # Max pooling layer 6\n",
    "        pool6 = tf.layers.max_pooling2d(\n",
    "            conv6_bn_relu,  # Input\n",
    "            pool_size=(2, 2),  # Pool size: 2x2\n",
    "            strides=(2, 2),  # Stride: 2\n",
    "            padding='SAME',  # \"same\" padding\n",
    "            name='pool6'\n",
    "        )\n",
    "\n",
    "        if dropout:\n",
    "            # dropout at 10%\n",
    "            pool6 = tf.layers.dropout(pool6, rate=0.1, seed=1, training=training)\n",
    "\n",
    "    # Flatten output\n",
    "    with tf.name_scope('flatten') as scope:\n",
    "        flat_output = tf.contrib.layers.flatten(pool6)\n",
    "\n",
    "        # dropout at 10%\n",
    "        flat_output = tf.layers.dropout(flat_output, rate=0.5, seed=5, training=training)\n",
    "\n",
    "    # Fully connected layer 1\n",
    "    with tf.name_scope('fc1') as scope:\n",
    "        fc1 = tf.layers.dense(\n",
    "            flat_output,  # input\n",
    "            1024,  # 2048 hidden units\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=4),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc1\"\n",
    "        )\n",
    "\n",
    "        bn_fc1 = tf.layers.batch_normalization(\n",
    "            fc1,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc1'\n",
    "        )\n",
    "\n",
    "        fc1_relu = tf.nn.relu(bn_fc1, name='fc1_relu')\n",
    "\n",
    "        # dropout at 25%\n",
    "        fc1_relu = tf.layers.dropout(fc1_relu, rate=0.75, seed=10, training=training)\n",
    "\n",
    "    # Fully connected layer 2\n",
    "    with tf.name_scope('fc2') as scope:\n",
    "        fc2 = tf.layers.dense(\n",
    "            fc1_relu,  # input\n",
    "            1024,  # 1024 hidden units\n",
    "            activation=None,  # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=5),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc2\"\n",
    "        )\n",
    "\n",
    "        bn_fc2 = tf.layers.batch_normalization(\n",
    "            fc2,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc2'\n",
    "        )\n",
    "\n",
    "        fc2_relu = tf.nn.relu(bn_fc2, name='fc2_relu')\n",
    "\n",
    "        # dropout at 10%\n",
    "        fc2_relu = tf.layers.dropout(fc2_relu, rate=0.75, seed=11, training=training)\n",
    "\n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        fc2_relu,  # input\n",
    "        num_classes,  # One output unit per category\n",
    "        activation=None,  # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=6),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        name=\"logits\"\n",
    "    )\n",
    "\n",
    "    # This will weight the positive examples higher so as to improve recall\n",
    "    weights = tf.multiply(4, tf.cast(tf.equal(y, 1), tf.int32)) + 1\n",
    "    # onehot_labels = tf.one_hot(y, depth=num_classes)\n",
    "    # mean_ce = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(targets=tf.one_hot(y, depth=num_classes), logits=logits, pos_weight=classes_weights))\n",
    "    # mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    mean_ce = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits, weights=weights))\n",
    "\n",
    "\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "\n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # Minimize cross-entropy\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int64)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))\n",
    "\n",
    "    if num_classes > 2:\n",
    "        recall = [0] * num_classes\n",
    "        rec_op = [[]] * num_classes\n",
    "\n",
    "        for k in range(num_classes):\n",
    "            recall[k], rec_op[k] = tf.metrics.recall(\n",
    "                labels=tf.equal(y, k),\n",
    "                predictions=tf.equal(predictions, k)\n",
    "            )\n",
    "    else:\n",
    "        recall, rec_op = tf.metrics.recall(labels=y, predictions=predictions, name=\"recall\")\n",
    "        # precision, prec_op = tf.metrics.precision(labels=y, predictions=predictions, name=\"precision\")\n",
    "        # f1_score = 2 * ( (precision * recall) / (precision + recall))\n",
    "\n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    tf.summary.scalar('recall', recall)\n",
    "    tf.summary.scalar('cross_entropy', mean_ce)\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "\n",
    "    # Merge all the summaries and write them out to /tmp/mnist_logs (by default)\n",
    "    merged = tf.summary.merge_all()\n",
    "    \n",
    "    print(\"Graph created...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4VeKl2i85tdR"
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mrBLt89h5uNu"
   },
   "outputs": [],
   "source": [
    "## CONFIGURE OPTIONS\n",
    "init = True                   # whether to initialize the model or use a saved version\n",
    "crop = False                  # do random cropping of images?\n",
    "\n",
    "meta_data_every = 5\n",
    "log_to_tensorboard = True\n",
    "print_every = 3                # how often to print metrics\n",
    "checkpoint_every = 1           # how often to save model in epochs\n",
    "use_gpu = False                 # whether or not to use the GPU\n",
    "print_metrics = False          # whether to print or plot metrics, if False a plot will be created and updated every epoch\n",
    "evaluate = True               # whether to periodically evaluate on test data\n",
    "\n",
    "# Placeholders for metrics\n",
    "if init:\n",
    "    valid_acc_values = []\n",
    "    valid_recall_values = []\n",
    "    valid_cost_values = []\n",
    "    train_acc_values = []\n",
    "    train_recall_values = []\n",
    "    train_cost_values = []\n",
    "    train_lr_values = []\n",
    "    train_loss_values = []\n",
    "    \n",
    "config = tf.ConfigProto()\n",
    "if use_gpu:\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allocator_type = 'BFC'\n",
    "    config.gpu_options.per_process_gpu_memory_fraction = 0.7\n",
    "else:\n",
    "    config = tf.ConfigProto(device_count = {'GPU': 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "UqbZFC1wUZ9G"
   },
   "outputs": [],
   "source": [
    "#epochs = 20\n",
    "#init = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "id": "4rAC4bf64kjn",
    "outputId": "84544b45-4506-4f19-f4e0-dde75ba6866c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model model_s0.0.0.12 ...\n"
     ]
    }
   ],
   "source": [
    "## train the model\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    if log_to_tensorboard:\n",
    "        train_writer = tf.summary.FileWriter('./logs/tr_' + model_name, sess.graph)\n",
    "        test_writer = tf.summary.FileWriter('./logs/te_' + model_name)\n",
    "    \n",
    "    if not print_metrics:\n",
    "        # create a plot to be updated as model is trained\n",
    "        f, ax = plt.subplots(1,4,figsize=(24,5))\n",
    "    \n",
    "    # create the saver\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # If the model is new initialize variables, else restore the session\n",
    "    if init:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "    else:\n",
    "        saver.restore(sess, './model/'+model_name+'.ckpt')\n",
    "\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    \n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "    print(\"Training model\", model_name,\"...\")\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        for i in range(steps_per_epoch):\n",
    "            # Accuracy values (train) after each batch\n",
    "            batch_acc = []\n",
    "            batch_cost = []\n",
    "            batch_loss = []\n",
    "            batch_lr = []\n",
    "            batch_recall = []\n",
    "\n",
    "            # create the metadata\n",
    "            run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "            run_metadata = tf.RunMetadata()\n",
    "\n",
    "            # Run training and evaluate accuracy\n",
    "            _, _, summary, acc_value, cost_value, loss_value, recall_value, step, lr = sess.run([train_op, extra_update_ops, \n",
    "                     merged, accuracy, mean_ce, loss, rec_op, global_step,\n",
    "                     learning_rate], feed_dict={\n",
    "                        #X: X_batch,\n",
    "                        #y: y_batch,\n",
    "                        training: True,\n",
    "                        is_testing: False,\n",
    "                    },\n",
    "            options=run_options,\n",
    "            run_metadata=run_metadata)\n",
    "\n",
    "            # Save accuracy (current batch)\n",
    "            batch_acc.append(acc_value)\n",
    "            batch_cost.append(cost_value)\n",
    "            batch_lr.append(lr)\n",
    "            batch_loss.append(loss_value)\n",
    "            batch_recall.append(np.mean(recall_value))\n",
    "            \n",
    "            # write the summary\n",
    "            if log_to_tensorboard:\n",
    "                train_writer.add_summary(summary, step)\n",
    "                # only log the meta data once per epoch\n",
    "                if i == (steps_per_epoch - 1):\n",
    "                    train_writer.add_run_metadata(run_metadata, 'step %d' % step)\n",
    "                \n",
    "        # save checkpoint every nth epoch\n",
    "        if(epoch % checkpoint_every == 0):\n",
    "            print(\"Saving checkpoint\")\n",
    "            save_path = saver.save(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "            # Now that model is saved set init to false so we reload it next time\n",
    "            init = False\n",
    "        \n",
    "        # init batch arrays\n",
    "        batch_cv_acc = []\n",
    "        batch_cv_cost = []\n",
    "        batch_cv_loss = []\n",
    "        batch_cv_recall = []\n",
    "        \n",
    "        ## evaluate on test data if it exists, otherwise ignore this step\n",
    "        if evaluate:\n",
    "            print(\"Evaluating model...\")\n",
    "            # load the test data\n",
    "            X_cv, y_cv = load_validation_data(percentage=1, how=\"normal\")\n",
    "            \n",
    "            # evaluate the test data\n",
    "            for X_batch, y_batch in get_batches(X_cv, y_cv, batch_size // 2, distort=False):\n",
    "                summary, valid_acc, valid_recall, valid_cost, valid_loss = sess.run([merged, accuracy, rec_op, mean_ce, loss], \n",
    "                    feed_dict={\n",
    "                        X: X_batch,\n",
    "                        y: y_batch,\n",
    "                        is_testing: True,\n",
    "                        training: False\n",
    "                    })\n",
    "\n",
    "                batch_cv_acc.append(valid_acc)\n",
    "                batch_cv_cost.append(valid_cost)\n",
    "                batch_cv_loss.append(valid_loss)\n",
    "                batch_cv_recall.append(np.mean(valid_recall))\n",
    "    \n",
    "            # Write average of validation data to summary logs\n",
    "            if log_to_tensorboard:\n",
    "                summary = tf.Summary(value=[tf.Summary.Value(tag=\"accuracy\", simple_value=np.mean(batch_cv_acc)),\n",
    "                                            tf.Summary.Value(tag=\"cross_entropy\", simple_value=np.mean(batch_cv_cost)),\n",
    "                                            tf.Summary.Value(tag=\"recall_1\", simple_value=np.mean(batch_cv_recall)), ])\n",
    "                test_writer.add_summary(summary, step)\n",
    "                step += 1\n",
    "            \n",
    "            # delete the test data to save memory\n",
    "            del(X_cv)\n",
    "            del(y_cv)\n",
    "\n",
    "            print(\"Done evaluating...\")\n",
    "        else:\n",
    "            batch_cv_acc.append(0)\n",
    "            batch_cv_cost.append(0)\n",
    "            batch_cv_loss.append(0)\n",
    "            batch_cv_recall.append(0)\n",
    "          \n",
    "        # take the mean of the values to add to the metrics\n",
    "        valid_acc_values.append(np.mean(batch_cv_acc))\n",
    "        valid_cost_values.append(np.mean(batch_cv_cost))\n",
    "        train_acc_values.append(np.mean(batch_acc))\n",
    "        train_cost_values.append(np.mean(batch_cost))\n",
    "        train_lr_values.append(np.mean(batch_lr))\n",
    "        train_loss_values.append(np.mean(batch_loss))\n",
    "        train_recall_values.append(np.mean(batch_recall))\n",
    "        valid_recall_values.append(np.mean(batch_cv_recall))\n",
    "\n",
    "        # Print progress every nth epoch to keep output to reasonable amount\n",
    "        if(epoch % print_every == 0):\n",
    "            print('Epoch {:02d} - step {} - cv acc: {:.3f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "            ))\n",
    "\n",
    "        # Print data every 50th epoch so I can write it down to compare models\n",
    "        if (not print_metrics) and (epoch % 50 == 0) and (epoch > 1):\n",
    "            if(epoch % print_every == 0):\n",
    "                print('Epoch {:02d} - step {} - cv acc: {:.4f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "                ))  \n",
    "      \n",
    "    # stop the coordinator\n",
    "    coord.request_stop()\n",
    "    \n",
    "    # Wait for threads to stop\n",
    "    coord.join(threads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GxOkYDfgS0El"
   },
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 2394,
     "output_extras": [
      {},
      {},
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 931,
     "status": "error",
     "timestamp": 1521660386411,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "71fxoeAPS15s",
    "outputId": "641378a9-1452-48ae-d14b-f8d38aea3102"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/model_s0.0.01.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/model_s0.0.01.ckpt\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 516\u001b[0;31m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[1;32m    517\u001b[0m     \u001b[0;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Unsuccessful TensorSliceReader constructor: Failed to get matching files on ./model/model_s0.0.01.ckpt: Not found: ./model; No such file or directory\n\t [[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_INT32, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save_1/Const_0_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-d3c64d5160fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'./model/'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.ckpt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     yhat, test_acc, test_recall = sess.run([predictions, accuracy, rec_op], feed_dict = \n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, sess, save_path)\u001b[0m\n\u001b[1;32m   1753\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_graph_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1754\u001b[0m       sess.run(self.saver_def.restore_op_name,\n\u001b[0;32m-> 1755\u001b[0;31m                {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[1;32m   1756\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1757\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_eager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_save\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_restore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1372\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1373\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1374\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1375\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1376\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Unsuccessful TensorSliceReader constructor: Failed to get matching files on ./model/model_s0.0.01.ckpt: Not found: ./model; No such file or directory\n\t [[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_INT32, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save_1/Const_0_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]\n\nCaused by op 'save_1/RestoreV2', defined at:\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-10-d3c64d5160fb>\", line 3, in <module>\n    saver = tf.train.Saver()\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1293, in __init__\n    self.build()\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1302, in build\n    self._build(self._filename, build_save=True, build_restore=True)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 1339, in _build\n    build_save=build_save, build_restore=build_restore)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 796, in _build_internal\n    restore_sequentially, reshape)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 449, in _AddRestoreOps\n    restore_sequentially)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py\", line 847, in bulk_restore\n    return io_ops.restore_v2(filename_tensor, names, slices, dtypes)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_io_ops.py\", line 1030, in restore_v2\n    shape_and_slices=shape_and_slices, dtypes=dtypes, name=name)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 3271, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\", line 1650, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInvalidArgumentError (see above for traceback): Unsuccessful TensorSliceReader constructor: Failed to get matching files on ./model/model_s0.0.01.ckpt: Not found: ./model; No such file or directory\n\t [[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_INT32, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, ..., DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](_arg_save_1/Const_0_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    # create the saver\n",
    "    saver = tf.train.Saver()\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    saver.restore(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "    yhat, test_acc, test_recall = sess.run([predictions, accuracy, rec_op], feed_dict = \n",
    "                              {\n",
    "                                  X: X_cv[0:128],\n",
    "                                  y: y_cv[0:128],\n",
    "                                  training:False\n",
    "                              })\n",
    "    \n",
    "i = 0\n",
    "\n",
    "print(\"Recall:\", test_recall)\n",
    "print(\"Accuracy:\", test_acc)\n",
    "\n",
    "for item in yhat:\n",
    "  if item == y_cv[i]:\n",
    "    found = \"*\"\n",
    "  else: \n",
    "    found = \"\"\n",
    "    \n",
    "  print(\"Prediction:\", item, \" Actual:\", y_cv[i], found)\n",
    "  i += 1\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "C70T7ponaHvL"
   },
   "source": [
    "## Old Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "glIFXdpI2_rf"
   },
   "outputs": [],
   "source": [
    "## Big Images\n",
    "if image_type == 'large':\n",
    "    if not os.path.exists(os.path.join(\"data\",\"data/all-mias.tar.gz\")):\n",
    "        # get the files from AWS\n",
    "        download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/all-mias.tar.gz', 'all-mias.tar.gz')\n",
    "\n",
    "        # extract the images\n",
    "        extract_tar(os.path.join(\"data\",\"all-mias.tar.gz\"))\n",
    "\n",
    "    if not os.path.exists(os.path.join(\"data\",\"images.npy\")):\n",
    "        # read all pgms in\n",
    "        files = glob('./data/pgms/*.pgm')\n",
    "        data = []\n",
    "\n",
    "        for file in files:\n",
    "            # read each file in and convert it to a float\n",
    "            data.append(read_pgm(file) * 1.0)\n",
    "\n",
    "        images = np.array(data, dtype=np.float32)\n",
    "\n",
    "        # save the data to a file so we don't have to keep downloading it\n",
    "        np.save(os.path.join('data','images.npy'), images)\n",
    "\n",
    "    else:\n",
    "        images = np.load('images.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 68,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3637,
     "status": "ok",
     "timestamp": 1521540748530,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "FOesn2sKpl2E",
    "outputId": "d0b78621-842d-4886-a952-545b531ac052"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading labels.pkl ...\n",
      "Downloading names.npy ...\n",
      "Downloading all_cases_df.pkl ...\n"
     ]
    }
   ],
   "source": [
    "image_type = 'small'\n",
    "\n",
    "## Labels\n",
    "if not os.path.exists(os.path.join(\"data\",\"labels.pkl\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/labels.pkl', 'labels.pkl')\n",
    "\n",
    "labels = pd.read_pickle(os.path.join(\"data\",\"labels.pkl\"))\n",
    "\n",
    "if not os.path.exists(os.path.join(\"data\",\"train_labels.npy\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/train_labels.npy', 'train_labels.npy')\n",
    "\n",
    "train_labels = np.load(os.path.join(\"data\", \"train_labels.npy\"))\n",
    "\n",
    "if not os.path.exists(os.path.join(\"data\",\"test_labels.npy\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/test_labels.npy', 'test_labels.npy')\n",
    "\n",
    "test_labels = np.load(os.path.join(\"data\", \"test_labels.npy\"))\n",
    "\n",
    "if not os.path.exists(os.path.join(\"data\",\"names.npy\")):  \n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/names.npy', 'names.npy')\n",
    "  \n",
    "names = np.load(os.path.join(\"data\",\"names.npy\"))\n",
    "  \n",
    "if not os.path.exists(os.path.join(\"data\",\"all_cases_df.pkl\")):  \n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/all_cases_df.pkl', 'all_cases_df.pkl')\n",
    "  \n",
    "all_cases_df = pd.read_pickle(os.path.join(\"data\",\"all_cases_df.pkl\"))\n",
    "\n",
    "## Small JPEG Images\n",
    "if image_type == 'small':\n",
    "  if not os.path.exists(os.path.join(\"data\",\"train_images299.npy\")):\n",
    "      # get the files from AWS\n",
    "      _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/train_images299.npy', 'train_images299.npy')\n",
    "      \n",
    "  if not os.path.exists(os.path.join(\"data\",\"test_images299.npy\")):\n",
    "      _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/test_images299.npy', 'test_images299.npy')\n",
    "      \n",
    "  images = np.load(os.path.join(\"data\",\"train_images299.npy\"))\n",
    "  test_images = np.load(os.path.join(\"data\",\"test_images299.npy\"))\n",
    "  \n",
    "  #images = images.reshape([-1,299,299,1])\n",
    "  #images = np.array(images)\n",
    "  \n",
    "\n",
    "## Small PGM Images\n",
    "if image_type == 'small_pgm':\n",
    "  if not os.path.exists(os.path.join(\"data\",\"small_images.npy\")):\n",
    "      # get the files from AWS\n",
    "      _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/small_images.npy', 'small_images.npy')\n",
    "\n",
    "  images = np.load(os.path.join(\"data\",\"small_images.npy\"))\n",
    "  images = images.reshape([-1,299,299,1])\n",
    "  images = np.array(images)\n",
    "\n",
    "## Medium Images\n",
    "if image_type == 'medium':\n",
    "  if not os.path.exists(os.path.join(\"data\",\"medium_images.npy\")):\n",
    "      # get the files from AWS\n",
    "      _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/medium_images.npy', 'medium_images.npy')\n",
    "\n",
    "  images = np.load(os.path.join(\"data\",\"medium_images.npy\"))\n",
    "  images = images.reshape([-1,512,512,1])\n",
    "  images = np.array(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 71,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3788,
     "status": "ok",
     "timestamp": 1521654586776,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "yHR3qyccaSMn",
    "outputId": "7108d6c5-ffa5-459a-f31c-0a0d7f56f074"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_all (7197, 299, 299, 1)\n",
      "y_all (7197,)\n"
     ]
    }
   ],
   "source": [
    "## Labels\n",
    "# cbis-ddsm labels\n",
    "if not os.path.exists(os.path.join(\"data\",\"all_labels4.npy\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/all_labels.npy', 'all_labels4.npy')\n",
    "\n",
    "# ddsm normal labels\n",
    "if not os.path.exists(os.path.join(\"data\",\"label_batch_0.npy\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/label_batch_0.npy', 'label_batch_0.npy')\n",
    "\n",
    "#if not os.path.exists(os.path.join(\"data\",\"label_batch_1.npy\")):\n",
    "    # get the files from AWS\n",
    "#    _ = download_file('', 'label_batch_1.npy')\n",
    "    \n",
    "## 299x299 JPEG Slices\n",
    "# cbis-ddsm slices\n",
    "if not os.path.exists(os.path.join(\"data\",\"all_slices.npy\")):\n",
    "    # get the files from AWS\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/all_slices.npy', 'all_slices.npy')\n",
    "\n",
    "# ddsm normal slices\n",
    "if not os.path.exists(os.path.join(\"data\",\"image_batch_0.npy\")):\n",
    "    _ = download_file('https://s3.eu-central-1.amazonaws.com/aws.skoo.ch/files/image_batch_0.npy', 'image_batch_0.npy')\n",
    "\n",
    "#if not os.path.exists(os.path.join(\"data\",\"image_batch_1.npy\")):\n",
    "#    _ = download_file('', 'image_batch_1.npy')\n",
    "\n",
    "\n",
    "## import the data    \n",
    "def load_data():\n",
    "  all_labels = np.load(os.path.join(\"data\", \"all_labels4.npy\"))\n",
    "  normal_labels_0 = np.load(os.path.join(\"data\", \"label_batch_0.npy\"))\n",
    "  #normal_labels_1 = np.load(os.path.join(\"data\", \"label_batch_1.npy\"))\n",
    "  \n",
    "  train_images = np.load(os.path.join(\"data\",\"all_slices.npy\"))\n",
    "  normal_images_0 = np.load(os.path.join(\"data\",\"image_batch_0.npy\"))\n",
    "  #normal_images_1 = np.load(os.path.join(\"data\",\"image_batch_1.npy\"))\n",
    "\n",
    "  # combine the data\n",
    "  y_all = np.concatenate([all_labels, normal_labels_0], axis=0)\n",
    "  X_all = np.concatenate([train_images, normal_images_0], axis=0)\n",
    "  \n",
    "  print(\"X_all\", X_all.shape)\n",
    "  print(\"y_all\", y_all.shape)\n",
    "  \n",
    "  # delete unused variables to save memory\n",
    "  del(train_images)\n",
    "  del(normal_images_0)\n",
    "  del(all_labels)\n",
    "  del(normal_labels_0)\n",
    "\n",
    "  return X_all, y_all\n",
    "  \n",
    "X_all, y_all = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 306,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6538,
     "status": "ok",
     "timestamp": 1521654593367,
     "user": {
      "displayName": "Eric Scuccimarra",
      "photoUrl": "//lh4.googleusercontent.com/-J0UnGJi90uc/AAAAAAAAAAI/AAAAAAAAJoI/epXN4nVk6CU/s50-c-k-no/photo.jpg",
      "userId": "116239815002878777210"
     },
     "user_tz": -60
    },
    "id": "s32N9yLr3HcZ",
    "outputId": "a71bf963-58b0-40b3-9e6b-331ef89ccf8e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr: (5757, 299, 299, 1)\n",
      "y_tr: (5757,)\n",
      "X_cv: (1440, 299, 299, 1)\n",
      "y_cv: (1440,)\n",
      "\n",
      "Validation distribution:\n",
      " 1.0    0.507639\n",
      "0.0    0.492361\n",
      "dtype: float64\n",
      "\n",
      "Training distribution:\n",
      " 1.0    0.511551\n",
      "0.0    0.488449\n",
      "dtype: float64\n",
      "\n",
      "Classes: ['NORMAL', 'ABNORMAL']\n",
      "Num Classes: 2\n"
     ]
    }
   ],
   "source": [
    "classify_by = \"normal\"\n",
    "\n",
    "# encode the labels\n",
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "\n",
    "#le.fit(y_all)\n",
    "#y_all_enc = le.transform(y_all)\n",
    "#print(\"Classes:\", le.classes_)\n",
    "\n",
    "if 'X_all' not in locals():\n",
    "    X_all, y_all = load_data()\n",
    "    \n",
    "# scale the data\n",
    "X_all = (X_all - 128) / 255.0\n",
    "\n",
    "## classify by normal or abnormal\n",
    "if classify_by == \"normal\":\n",
    "    y_enc = np.zeros(len(y_all))\n",
    "    y_enc[y_all != 'NORMAL'] = 1\n",
    "    classes = ['NORMAL','ABNORMAL']\n",
    "\n",
    "# classify by the Normal, Malignant, Mass, Calcification\n",
    "elif classify_by == \"class\":\n",
    "    # encode the labels\n",
    "    le.fit(y_all)\n",
    "    y_enc = le.transform(y_all)\n",
    "    classes = le.classes_\n",
    "\n",
    "# classify by Normal, Mass or Calcification\n",
    "elif classify_by == \"mass\":\n",
    "    y_enc = np.zeros(len(y_all))\n",
    "    y_enc[y_all == \"BENIGN_calcification\"] = 1\n",
    "    y_enc[y_all == \"MALIGNANT_calcification\"] = 1\n",
    "    y_enc[y_all == \"BENIGN_mass\"] = 2\n",
    "    y_enc[y_all == \"MALIGNANT_mass\"] = 2\n",
    "    classes = ['NORMAL','CALCIFICATION','MASS']\n",
    "\n",
    "# classify by Normal, Benign or Malignant\n",
    "elif classify_by == \"malignant\":\n",
    "    y_enc = np.zeros(len(y_all))\n",
    "    y_enc[y_all == \"BENIGN_mass\"] = 1\n",
    "    y_enc[y_all == \"BENIGN_calcification\"] = 1\n",
    "    y_enc[y_all == \"MALIGNANT_mass\"] = 2\n",
    "    y_enc[y_all == \"MALIGNANT_calcification\"] = 2\n",
    "    classes = ['NORMAL','BENIGNS','MALIGNANT']\n",
    "    \n",
    "num_classes = len(np.unique(classes))\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_cv, y_tr, y_cv = train_test_split(X_all, y_enc, test_size=0.2, random_state=3)\n",
    "\n",
    "# delete unused variables to save memory\n",
    "del(X_all)\n",
    "del(y_all)\n",
    "\n",
    "print(\"X_tr:\", X_tr.shape)\n",
    "print(\"y_tr:\", y_tr.shape)\n",
    "print(\"X_cv:\", X_cv.shape)\n",
    "print(\"y_cv:\", y_cv.shape)\n",
    "\n",
    "print(\"\\nValidation distribution:\\n\",pd.value_counts(y_cv, normalize=True))\n",
    "print(\"\\nTraining distribution:\\n\",pd.value_counts(y_tr, normalize=True))\n",
    "\n",
    "print(\"\\nClasses:\", classes)\n",
    "print(\"Num Classes:\", num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IuG290bY13qL"
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "# whether to retrain model from scratch or use saved model\n",
    "init = True\n",
    "model_name = \"model_s0.0.01\"\n",
    "train_path_0 = os.path.join(\"data\", \"training_0.tfrecords\")\n",
    "train_path_1 = os.path.join(\"data\", \"training_1.tfrecords\")\n",
    "train_path_2 = os.path.join(\"data\", \"training_2.tfrecords\")\n",
    "train_path_3 = os.path.join(\"data\", \"training_3.tfrecords\")\n",
    "test_path = os.path.join(\"data\", \"test.tfrecords\")\n",
    "train_files = [train_path_0, train_path_1, train_path_2, train_path_3]\n",
    "\n",
    "with graph.as_default():\n",
    "    training = tf.placeholder(dtype=tf.bool, name=\"is_training\")\n",
    "    is_testing = tf.placeholder(dtype=bool, shape=(), name=\"is_testing\")\n",
    "    \n",
    "    # create global step for decaying learning rate\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    \n",
    "    learning_rate = tf.train.exponential_decay(starting_rate,                 \n",
    "                                               global_step, \n",
    "                                               steps_per_epoch * epochs_per_decay,\n",
    "                                               decay_factor,                  \n",
    "                                               staircase=staircase) \n",
    "    \n",
    "    with tf.name_scope('inputs') as scope:\n",
    "        #train_images, train_labels = read_and_decode_single_example([test_path])\n",
    "        #test_images, test_labels = read_and_decode_single_example([train_path_0])\n",
    "        \n",
    "        #image = tf.cond(\n",
    "        #    is_testing,\n",
    "        #    true_fn=lambda: train_images,\n",
    "        #    false_fn=lambda:  test_images,\n",
    "        #    strict=False,\n",
    "        #    name=\"input_image_cond\",\n",
    "        #)\n",
    "        \n",
    "        #label = tf.cond(\n",
    "        #    is_testing,\n",
    "        #    true_fn=lambda: train_labels,\n",
    "        #    false_fn=lambda:  test_labels,\n",
    "        #    strict=False,\n",
    "        #    name=\"input_label_cond\",\n",
    "        #)\n",
    "        \n",
    "        image, label = read_and_decode_single_example(train_files, label_type=\"label_normal\")\n",
    "\n",
    "        X_def, y_def = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=2000, min_after_dequeue=1000)\n",
    "\n",
    "        # Placeholders\n",
    "        X = tf.placeholder_with_default(X_def, shape=[None, 299, 299, 1])\n",
    "        y = tf.placeholder_with_default(y_def, shape=[None])\n",
    "    \n",
    "    with tf.name_scope('conv1') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv1 = tf.layers.conv2d(\n",
    "            X,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(5, 5),          # Kernel size: 9x9\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv1'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        conv1 = tf.layers.batch_normalization(\n",
    "            conv1,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn1'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv1_bn_relu = tf.nn.relu(conv1, name='relu1')\n",
    "      \n",
    "        if dropout:\n",
    "            conv1_bn_relu = tf.layers.dropout(conv1_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    \n",
    "    with tf.name_scope('conv2') as scope:\n",
    "        # Convolutional layer 1 \n",
    "        conv2 = tf.layers.conv2d(\n",
    "            conv1_bn_relu,                           # Input data\n",
    "            filters=32,                  # 32 filters\n",
    "            kernel_size=(5, 5),          # Kernel size: 9x9\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv2'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn2 = tf.layers.batch_normalization(\n",
    "            conv2,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn2'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv2_bn_relu = tf.nn.relu(bn2, name='relu2')\n",
    "      \n",
    "        if dropout:\n",
    "            conv2_bn_relu = tf.layers.dropout(conv2_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool1') as scope:\n",
    "        # Max pooling layer 1\n",
    "        pool1 = tf.layers.max_pooling2d(\n",
    "            conv2_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 3x3\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool1'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool1 = tf.layers.dropout(pool1, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv3') as scope:\n",
    "        # Convolutional layer 3\n",
    "        conv3 = tf.layers.conv2d(\n",
    "            pool1,                       # Input data\n",
    "            filters=64,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv3'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn3 = tf.layers.batch_normalization(\n",
    "            conv3,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn3'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv3_bn_relu = tf.nn.relu(bn3, name='relu3')\n",
    "      \n",
    "        if dropout:\n",
    "            conv3_bn_relu = tf.layers.dropout(conv3_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv4') as scope:\n",
    "        # Convolutional layer 4\n",
    "        conv4 = tf.layers.conv2d(\n",
    "            conv3_bn_relu,                       # Input data\n",
    "            filters=64,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv4'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn4 = tf.layers.batch_normalization(\n",
    "            conv4,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn4'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv4_bn_relu = tf.nn.relu(bn4, name='relu4')\n",
    "      \n",
    "        if dropout:\n",
    "            conv4_bn_relu = tf.layers.dropout(conv4_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('pool2') as scope:\n",
    "        # Max pooling layer 2\n",
    "        pool2 = tf.layers.max_pooling2d(\n",
    "            conv4_bn_relu,               # Input\n",
    "            pool_size=(3, 3),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool2'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool2 = tf.layers.dropout(pool2, rate=0.1, seed=1, training=training)\n",
    "          \n",
    "    with tf.name_scope('conv5') as scope:\n",
    "        # Convolutional layer 5\n",
    "        conv5 = tf.layers.conv2d(\n",
    "            pool2,                       # Input data\n",
    "            filters=128,                  # 64 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv5'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn5 = tf.layers.batch_normalization(\n",
    "            conv5,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn5'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv5_bn_relu = tf.nn.relu(bn5, name='relu5')\n",
    "      \n",
    "        if dropout:\n",
    "            conv5_bn_relu = tf.layers.dropout(conv5_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv6') as scope:\n",
    "        # Convolutional layer 3\n",
    "        conv6 = tf.layers.conv2d(\n",
    "            conv5_bn_relu,                       # Input data\n",
    "            filters=128,                  # 48 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv6'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn6 = tf.layers.batch_normalization(\n",
    "            conv6,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn6'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv6_bn_relu = tf.nn.relu(bn6, name='relu6')\n",
    "      \n",
    "        if dropout:\n",
    "            conv6_bn_relu = tf.layers.dropout(conv6_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool3') as scope:\n",
    "        # Average pooling layer 3\n",
    "        pool3 = tf.layers.max_pooling2d(\n",
    "            conv6_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool3'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool3 = tf.layers.dropout(pool3, rate=0.25, seed=1, training=training)\n",
    "    \n",
    "    with tf.name_scope('conv7') as scope:\n",
    "        # Convolutional layer 7\n",
    "        conv7 = tf.layers.conv2d(\n",
    "            pool3,                       # Input data\n",
    "            filters=256,                  # 64 filters\n",
    "            kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "            strides=(1, 1),              # Stride: 1\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "            name='conv7'                 \n",
    "        )\n",
    "\n",
    "        # try batch normalization\n",
    "        bn7 = tf.layers.batch_normalization(\n",
    "            conv7,\n",
    "            axis=-1,\n",
    "            momentum=0.99,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn7'\n",
    "        )\n",
    "\n",
    "        #apply relu\n",
    "        conv7_bn_relu = tf.nn.relu(bn7, name='relu7')\n",
    "      \n",
    "        if dropout:\n",
    "            conv7_bn_relu = tf.layers.dropout(conv7_bn_relu, rate=0.1, seed=9, training=training)\n",
    "    \n",
    "    #with tf.name_scope('conv8') as scope:\n",
    "        # Convolutional layer 8\n",
    "    #    conv8 = tf.layers.conv2d(\n",
    "    #        conv7_bn_relu,                       # Input data\n",
    "    #        filters=256,                  # 48 filters\n",
    "    #        kernel_size=(3, 3),          # Kernel size: 5x5\n",
    "    #        strides=(1, 1),              # Stride: 1\n",
    "    #        padding='SAME',              # \"same\" padding\n",
    "    #        activation=None,             # None\n",
    "    #        kernel_initializer=tf.truncated_normal_initializer(stddev=5e-2, seed=10),\n",
    "    #        kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamC),\n",
    "    #        name='conv8'                 \n",
    "    #    )\n",
    "\n",
    "        # try batch normalization\n",
    "    #    bn8 = tf.layers.batch_normalization(\n",
    "    #        conv8,\n",
    "    #        axis=-1,\n",
    "    #        momentum=0.99,\n",
    "    #        epsilon=epsilon,\n",
    "    #        center=True,\n",
    "    #        scale=True,\n",
    "    #        beta_initializer=tf.zeros_initializer(),\n",
    "    #        gamma_initializer=tf.ones_initializer(),\n",
    "    #        moving_mean_initializer=tf.zeros_initializer(),\n",
    "    #        moving_variance_initializer=tf.ones_initializer(),\n",
    "    #        training=training,\n",
    "    #        name='bn8'\n",
    "    #    )\n",
    "\n",
    "        #apply relu\n",
    "    #    conv8_bn_relu = tf.nn.relu(bn8, name='relu8')\n",
    "      \n",
    "    #    if dropout:\n",
    "    #        conv8_bn_relu = tf.layers.dropout(conv8_bn_relu, rate=0.1, seed=9, training=training)\n",
    "            \n",
    "    with tf.name_scope('pool4') as scope:\n",
    "        # Average pooling layer 4\n",
    "        pool4 = tf.layers.max_pooling2d(\n",
    "            conv7_bn_relu,               # Input\n",
    "            pool_size=(2, 2),            # Pool size: 2x2\n",
    "            strides=(2, 2),              # Stride: 2\n",
    "            padding='SAME',              # \"same\" padding\n",
    "            name='pool4'\n",
    "        )\n",
    "        \n",
    "        if dropout:\n",
    "          # dropout at 10%\n",
    "          pool4 = tf.layers.dropout(pool4, rate=0.1, seed=1, training=training)\n",
    "    \n",
    "    # Flatten output\n",
    "    with tf.name_scope('flatten') as scope:\n",
    "        flat_output = tf.contrib.layers.flatten(pool4)\n",
    "\n",
    "        # dropout at 10%\n",
    "        flat_output = tf.layers.dropout(flat_output, rate=0.5, seed=5, training=training)\n",
    "   \n",
    "    # Fully connected layer 1\n",
    "    with tf.name_scope('fc1') as scope:\n",
    "        fc1 = tf.layers.dense(\n",
    "            flat_output,                 # input\n",
    "            1024,                        # 2048 hidden units\n",
    "            activation=None,             # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=4),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc1\"\n",
    "        )\n",
    "        \n",
    "        bn_fc1 = tf.layers.batch_normalization(\n",
    "            fc1,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc1'\n",
    "        )\n",
    "        \n",
    "        fc1_relu = tf.nn.relu(bn_fc1, name='fc1_relu')\n",
    "      \n",
    "        # dropout at 25%\n",
    "        fc1_relu = tf.layers.dropout(fc1_relu, rate=0.75, seed=10, training=training)\n",
    "\n",
    "    # Fully connected layer 2\n",
    "    with tf.name_scope('fc2') as scope:\n",
    "        fc2 = tf.layers.dense(\n",
    "            fc1_relu,                     # input\n",
    "            1024,                       # 1024 hidden units\n",
    "            activation=None,            # None\n",
    "            kernel_initializer=tf.variance_scaling_initializer(scale=2, seed=5),\n",
    "            bias_initializer=tf.zeros_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=lamF),\n",
    "            name=\"fc2\"\n",
    "        )\n",
    "        \n",
    "        bn_fc2 = tf.layers.batch_normalization(\n",
    "            fc2,\n",
    "            axis=-1,\n",
    "            momentum=0.9,\n",
    "            epsilon=epsilon,\n",
    "            center=True,\n",
    "            scale=True,\n",
    "            beta_initializer=tf.zeros_initializer(),\n",
    "            gamma_initializer=tf.ones_initializer(),\n",
    "            moving_mean_initializer=tf.zeros_initializer(),\n",
    "            moving_variance_initializer=tf.ones_initializer(),\n",
    "            training=training,\n",
    "            name='bn_fc2'\n",
    "        )\n",
    "        \n",
    "        fc2_relu = tf.nn.relu(bn_fc2, name='fc2_relu')\n",
    "        \n",
    "        # dropout at 10%\n",
    "        fc2_relu = tf.layers.dropout(fc2_relu, rate=0.75, seed=11, training=training)\n",
    "\n",
    "    # Output layer\n",
    "    logits = tf.layers.dense(\n",
    "        fc2_relu,                      # input\n",
    "        num_classes,                 # One output unit per category\n",
    "        activation=None,             # No activation function\n",
    "        kernel_initializer=tf.variance_scaling_initializer(scale=1, seed=6),\n",
    "        bias_initializer=tf.zeros_initializer(),\n",
    "        name=\"logits\"\n",
    "    )\n",
    "    \n",
    "    # Weighted mean cross-entropy to trade-off precision for recall\n",
    "    #mean_ce = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(targets=y, logits=logits, pos_weight=1.5))\n",
    "    mean_ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "    loss = mean_ce + tf.losses.get_regularization_loss()\n",
    "    \n",
    "    # Adam optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    \n",
    "    # Minimize cross-entropy\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Compute predictions and accuracy\n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int64)\n",
    "    is_correct = tf.equal(y, predictions)\n",
    "    accuracy = tf.reduce_mean(tf.cast(is_correct, dtype=tf.float32))\n",
    "    \n",
    "    if num_classes > 2:\n",
    "      recall = [0] * num_classes\n",
    "      rec_op = [[]] * num_classes\n",
    "\n",
    "      for k in range(num_classes):\n",
    "        recall[k], rec_op[k] = tf.metrics.recall(\n",
    "            labels=tf.equal(y, k),\n",
    "            predictions=tf.equal(predictions, k)\n",
    "        )\n",
    "    else:\n",
    "      recall, rec_op = tf.metrics.recall(labels=y, predictions=predictions, name=\"recall\")\n",
    "      #precision, prec_op = tf.metrics.precision(labels=y, predictions=predictions, name=\"precision\")\n",
    "      #f1_score = 2 * ( (precision * recall) / (precision + recall))\n",
    "    \n",
    "    # add this so that the batch norm gets run\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    \n",
    "    # Create summary hooks\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    tf.summary.scalar('cross_entropy', mean_ce)\n",
    "    tf.summary.scalar('learning_rate', learning_rate)\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    \n",
    "    # Merge all the summaries and write them out to /tmp/mnist_logs (by default)\n",
    "    merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "tD012JPf5zJv"
   },
   "outputs": [],
   "source": [
    "# train the model\n",
    "with tf.Session(graph=graph, config=config) as sess:\n",
    "    if log_to_tensorboard:\n",
    "        train_writer = tf.summary.FileWriter('./logs/tr_' + model_name, sess.graph)\n",
    "        test_writer = tf.summary.FileWriter('./logs/te_' + model_name)\n",
    "    \n",
    "    if not print_metrics:\n",
    "        # create a plot to be updated as model is trained\n",
    "        f, ax = plt.subplots(1,4,figsize=(24,5))\n",
    "    \n",
    "    # create the saver\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # If the model is new initialize variables, else restore the session\n",
    "    if init:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "    else:\n",
    "        saver.restore(sess, './model/'+model_name+'.ckpt')\n",
    "\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    \n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "    \n",
    "    # Set seed\n",
    "    np.random.seed(0)\n",
    "    \n",
    "    print(\"Training\", model_name, \"...\")\n",
    "    \n",
    "    # Train several epochs\n",
    "    for epoch in range(epochs):\n",
    "        # Accuracy values (train) after each batch\n",
    "        batch_acc = []\n",
    "        batch_cost = []\n",
    "        batch_loss = []\n",
    "        batch_lr = []\n",
    "        batch_recall = []\n",
    "        \n",
    "        # only log run metadata once per epoch\n",
    "        write_meta_data = False\n",
    "            \n",
    "        for X_batch, y_batch in get_batches(X_tr, y_tr, batch_size, distort=False):\n",
    "            if write_meta_data and log_to_tensboard:\n",
    "                # create the metadata\n",
    "                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "                run_metadata = tf.RunMetadata()\n",
    "            \n",
    "                # Run training and evaluate accuracy\n",
    "                _, _, summary, acc_value, recall_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, rec_op, mean_ce, loss, global_step, learning_rate], feed_dict={\n",
    "                    X: X_batch,\n",
    "                    y: y_batch,\n",
    "                    training: True\n",
    "                },\n",
    "                options=run_options,\n",
    "                run_metadata=run_metadata)\n",
    "\n",
    "                # Save accuracy (current batch)\n",
    "                batch_acc.append(acc_value)\n",
    "                batch_cost.append(cost_value)\n",
    "                batch_lr.append(lr)\n",
    "                batch_loss.append(loss_value)\n",
    "                batch_recall.append(np.mean(recall_value))\n",
    "  \n",
    "                # write the summary\n",
    "                train_writer.add_run_metadata(run_metadata, 'step %d' % step)\n",
    "                train_writer.add_summary(summary, step)\n",
    "                write_meta_data = False\n",
    "                \n",
    "            else:\n",
    "                # Run training without meta data\n",
    "                _, _, summary, acc_value, recall_value, cost_value, loss_value, step, lr = sess.run([train_op, extra_update_ops, merged, accuracy, rec_op, mean_ce, loss, global_step, learning_rate], feed_dict={\n",
    "                    X: X_batch,\n",
    "                    y: y_batch,\n",
    "                    training: True\n",
    "                })\n",
    "                \n",
    "                # Save accuracy (current batch)\n",
    "                batch_acc.append(acc_value)\n",
    "                batch_cost.append(cost_value)\n",
    "                batch_lr.append(lr)\n",
    "                batch_loss.append(loss_value)\n",
    "                batch_recall.append(np.mean(recall_value))\n",
    "                \n",
    "                # write the summary\n",
    "                if log_to_tensorboard:\n",
    "                    train_writer.add_summary(summary, step)\n",
    "\n",
    "        # save checkpoint every nth epoch\n",
    "        if(epoch % checkpoint_every == 0):\n",
    "            print(\"Saving checkpoint\")\n",
    "            # save the model\n",
    "            save_path = saver.save(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "            # Now that model is saved set init to false so we reload it\n",
    "            init = False\n",
    "        \n",
    "        # init batch arrays\n",
    "        batch_cv_acc = []\n",
    "        batch_cv_cost = []\n",
    "        batch_cv_loss = []\n",
    "        batch_cv_recall = []\n",
    "        \n",
    "        #sess.run(tf.local_variables_initializer())\n",
    "        \n",
    "        # Evaluate validation accuracy with batches so as to not crash the GPU\n",
    "        for X_batch, y_batch in get_batches(X_cv, y_cv, batch_size, distort=False):\n",
    "            summary, valid_acc, valid_recall, valid_cost, valid_loss = sess.run([merged, accuracy, rec_op, mean_ce, loss], feed_dict={\n",
    "                X: X_batch,\n",
    "                y: y_batch,\n",
    "                training: False\n",
    "            })\n",
    "\n",
    "            batch_cv_acc.append(valid_acc)\n",
    "            batch_cv_cost.append(valid_cost)\n",
    "            batch_cv_loss.append(valid_loss)\n",
    "            batch_cv_recall.append(np.mean(valid_recall))\n",
    "\n",
    "        # Write average of validation data to summary logs\n",
    "        if log_to_tensorboard:\n",
    "            summary = tf.Summary(value=[tf.Summary.Value(tag=\"accuracy\", simple_value=np.mean(batch_cv_acc)),tf.Summary.Value(tag=\"cross_entropy\", simple_value=np.mean(batch_cv_cost)),])\n",
    "            test_writer.add_summary(summary, step)\n",
    "            step += 1\n",
    "            \n",
    "        # take the mean of the values to add to the metrics\n",
    "        valid_acc_values.append(np.mean(batch_cv_acc))\n",
    "        valid_cost_values.append(np.mean(batch_cv_cost))\n",
    "        train_acc_values.append(np.mean(batch_acc))\n",
    "        train_cost_values.append(np.mean(batch_cost))\n",
    "        train_lr_values.append(np.mean(batch_lr))\n",
    "        train_loss_values.append(np.mean(batch_loss))\n",
    "        train_recall_values.append(np.mean(batch_recall))\n",
    "        valid_recall_values.append(np.mean(batch_cv_recall))\n",
    "        \n",
    "        if print_metrics:\n",
    "            # Print progress every nth epoch to keep output to reasonable amount\n",
    "            if(epoch % print_every == 0):\n",
    "                print('Epoch {:02d} - step {} - cv acc: {:.3f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "                ))\n",
    "        else:\n",
    "            # update the plot\n",
    "            ax[0].cla()\n",
    "            ax[0].plot(valid_acc_values, color=\"red\", label=\"Validation\")\n",
    "            ax[0].plot(train_acc_values, color=\"blue\", label=\"Training\")\n",
    "            ax[0].set_title('Validation accuracy: {:.4f} (mean last 3)'.format(np.mean(valid_acc_values[-4:])))\n",
    "            \n",
    "            # since we can't zoom in on plots like in tensorboard, scale y axis to give a decent amount of detail\n",
    "            #if np.mean(valid_acc_values[-3:]) > 0.90:\n",
    "            #    ax[0].set_ylim([0.80,1.0])\n",
    "            #elif np.mean(valid_acc_values[-3:]) > 0.85:\n",
    "            #    ax[0].set_ylim([0.75,1.0])\n",
    "            #elif np.mean(valid_acc_values[-3:]) > 0.75:\n",
    "            #    ax[0].set_ylim([0.65,1.0])\n",
    "            #elif np.mean(valid_acc_values[-3:]) > 0.65:\n",
    "            #    ax[0].set_ylim([0.55,1.0])\n",
    "            #elif np.mean(valid_acc_values[-3:]) > 0.55:\n",
    "            #    ax[0].set_ylim([0.45,1.0])           \n",
    "            \n",
    "            ax[0].set_xlabel('Epoch')\n",
    "            ax[0].set_ylabel('Accuracy')\n",
    "            ax[0].legend()\n",
    "            \n",
    "            ax[1].cla()\n",
    "            ax[1].plot(valid_recall_values, color=\"red\", label=\"Validation\")\n",
    "            ax[1].plot(train_recall_values, color=\"blue\", label=\"Training\")\n",
    "            ax[1].set_title('Validation recall: {:.3f} (mean last 3)'.format(np.mean(valid_recall_values[-4:])))\n",
    "            ax[1].set_xlabel('Epoch')\n",
    "            ax[1].set_ylabel('Recall')\n",
    "            #ax[1].set_ylim([0,2.0])\n",
    "            ax[1].legend()\n",
    "            \n",
    "            ax[2].cla()\n",
    "            ax[2].plot(valid_cost_values, color=\"red\", label=\"Validation\")\n",
    "            ax[2].plot(train_cost_values, color=\"blue\", label=\"Training\")\n",
    "            ax[2].set_title('Validation xentropy: {:.3f} (mean last 3)'.format(np.mean(valid_cost_values[-4:])))\n",
    "            ax[2].set_xlabel('Epoch')\n",
    "            ax[2].set_ylabel('Cross Entropy')\n",
    "            ax[2].set_ylim([0,3.0])\n",
    "            ax[2].legend()\n",
    "            \n",
    "            ax[3].cla()\n",
    "            ax[3].plot(train_lr_values)\n",
    "            ax[3].set_title(\"Learning rate: {:.6f}\".format(np.mean(train_lr_values[-1:])))\n",
    "            ax[3].set_xlabel(\"Epoch\")\n",
    "            ax[3].set_ylabel(\"Learning Rate\")\n",
    "            \n",
    "            display.display(plt.gcf())\n",
    "            display.clear_output(wait=True)\n",
    "            \n",
    "        # Print data every 50th epoch so I can write it down to compare models\n",
    "        if (not print_metrics) and (epoch % 50 == 0) and (epoch > 1):\n",
    "            if(epoch % print_every == 0):\n",
    "                print('Epoch {:02d} - step {} - cv acc: {:.4f} - train acc: {:.3f} (mean) - cv cost: {:.3f} - lr: {:.5f}'.format(\n",
    "                    epoch, step, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost), lr\n",
    "                ))  \n",
    "    \n",
    "    \n",
    "    # stop queue coordinator\n",
    "    coord.request_stop()\n",
    "    coord.join(threads)\n",
    "    \n",
    "    # print results of last epoch\n",
    "    print('Epoch {} - cv acc: {:.4f} - train acc: {:.4f} (mean) - cv cost: {:.3f}'.format(\n",
    "                epochs, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost)\n",
    "            ))\n",
    "    \n",
    "    # save the session\n",
    "    save_path = saver.save(sess, './model/'+model_name+'.ckpt')\n",
    "    \n",
    "    # init the test data array\n",
    "    test_acc_values = []\n",
    "    \n",
    "    # Check on the test data\n",
    "    #for X_batch, y_batch in get_batches(X_te, y_te, batch_size, distort=False):\n",
    "    #    test_accuracy = sess.run(accuracy, feed_dict={\n",
    "    #        X: X_batch,\n",
    "    #        y: y_batch,\n",
    "    #        training: False\n",
    "    #    })\n",
    "    #    test_acc_values.append(test_accuracy)\n",
    "    \n",
    "    # average test accuracy across batches\n",
    "    #test_acc = np.mean(test_acc_values)\n",
    "    \n",
    "# show the plot\n",
    "plt.show()\n",
    "\n",
    "# print results of last epoch\n",
    "print('Epoch {} - cv acc: {:.4f} - train acc: {:.4f} (mean) - cv cost: {:.3f}'.format(\n",
    "      epochs, np.mean(batch_cv_acc), np.mean(batch_acc), np.mean(batch_cv_cost)\n",
    "    ))\n",
    "    \n",
    "# print test accuracy\n",
    "#print(\"Convolutional network accuracy (test set):\", test_acc, \" Validation set:\", valid_acc_values[-1])"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "GxOkYDfgS0El",
    "C70T7ponaHvL"
   ],
   "default_view": {},
   "name": "DDSM_ROI_Slices_299x299.ipynb",
   "provenance": [
    {
     "file_id": "1eLHyQIu3WRkmUWx1ZKupzOlaA0HIPQu0",
     "timestamp": 1521635705642
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}