res34.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "\n",
    "from keras.models import Model, load_model, save_model\n",
    "from keras.layers import Input,Dropout,BatchNormalization,Activation,Add,merge,Flatten,ZeroPadding2D,GlobalAveragePooling2D\\\n",
    ",Dense,ZeroPadding2D\n",
    "from keras.engine import get_source_inputs\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Conv2D, Conv2DTranspose, UpSampling2D\n",
    "from keras.layers.pooling import MaxPooling2D,GlobalAveragePooling2D,AveragePooling2D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras import backend as K\n",
    "from keras import optimizers\n",
    "from keras.applications.imagenet_utils import _obtain_input_shape\n",
    "import keras\n",
    "from keras.utils import get_file\n",
    "from keras.losses import binary_crossentropy\n",
    "\n",
    "from keras.preprocessing.image import load_img\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD, adam\n",
    "\n",
    "from skimage.transform import resize\n",
    "import tensorflow as tf\n",
    "import glob\n",
    "import copy\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47e7ca42cea64462a790a361a5a40af0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5977855eca64e1ab4e0a9685fa3a9fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "img_size_target = 256\n",
    "    \n",
    "def downsample(img):\n",
    "    if img_size_ori == img_size_target:\n",
    "        return img\n",
    "    return resize(img, (img_size_ori, img_size_ori), mode='constant', preserve_range=True)\n",
    "\n",
    "classes = pd.read_csv(\"classes-trainable.csv\", engine='python')\n",
    "training = pd.read_csv(\"training.csv\", engine='python')\n",
    "valid = pd.read_csv(\"valid.csv\", engine='python')\n",
    "test = glob.glob(\"test/*\")\n",
    "\n",
    "labels_all = list(classes.label_code)\n",
    "\n",
    "ids = list(training.ImageID)\n",
    "x_train = [np.array(load_img(\"train/{}.png\".format(idx))) / 255 \n",
    "                      for idx in tqdm_notebook(ids)]\n",
    "y_train = training.labels\n",
    "\n",
    "ids = list(valid.ImageID)\n",
    "x_valid = [np.array(load_img(\"valid/{}.png\".format(idx))) / 255 \n",
    "                      for idx in tqdm_notebook(ids)]\n",
    "y_valid = valid.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encoding(y, all_labels):\n",
    "    all_labels = list(all_labels)\n",
    "    output = []\n",
    "    for i in y:\n",
    "        i = i.split(\" \")\n",
    "        one_label = np.zeros(len(all_labels))\n",
    "        for j in i:\n",
    "            inde = all_labels.index(j)\n",
    "            one_label[inde] = 1\n",
    "        output.append(i)\n",
    "    return np.array(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = one_hot_encoding(y_train)\n",
    "y_valid = one_hot_encoding(y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model_weights(weights_collection, model, dataset, classes, include_top):\n",
    "    weights = find_weights(weights_collection, model.name, dataset, include_top)\n",
    "\n",
    "    if weights:\n",
    "        weights = weights[0]\n",
    "\n",
    "        if include_top and weights['classes'] != classes:\n",
    "            raise ValueError('If using `weights` and `include_top`'\n",
    "                             ' as true, `classes` should be {}'.format(weights['classes']))\n",
    "\n",
    "        weights_path = get_file(weights['name'],\n",
    "                                weights['url'],\n",
    "                                cache_subdir='models',\n",
    "                                md5_hash=weights['md5'])\n",
    "\n",
    "        model.load_weights(weights_path)\n",
    "\n",
    "    else:\n",
    "        raise ValueError('There is no weights for such configuration: ' +\n",
    "                         'model = {}, dataset = {}, '.format(model.name, dataset) +\n",
    "                         'classes = {}, include_top = {}.'.format(classes, include_top))\n",
    "\n",
    "def find_weights(weights_collection, model_name, dataset, include_top):\n",
    "    w = list(filter(lambda x: x['model'] == model_name, weights_collection))\n",
    "    w = list(filter(lambda x: x['dataset'] == dataset, w))\n",
    "    w = list(filter(lambda x: x['include_top'] == include_top, w))\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_collection = [\n",
    "    # ResNet34\n",
    "    {\n",
    "        'model': 'resnet34',\n",
    "        'dataset': 'imagenet',\n",
    "        'classes': 1000,\n",
    "        'include_top': True,\n",
    "        'url': 'https://github.com/qubvel/classification_models/releases/download/0.0.1/resnet34_imagenet_1000.h5',\n",
    "        'name': 'resnet34_imagenet_1000.h5',\n",
    "        'md5': '2ac8277412f65e5d047f255bcbd10383',\n",
    "    },\n",
    "\n",
    "    {\n",
    "        'model': 'resnet34',\n",
    "        'dataset': 'imagenet',\n",
    "        'classes': 1000,\n",
    "        'include_top': False,\n",
    "        'url': 'https://github.com/qubvel/classification_models/releases/download/0.0.1/resnet34_imagenet_1000_no_top.h5',\n",
    "        'name': 'resnet34_imagenet_1000_no_top.h5',\n",
    "        'md5': '8caaa0ad39d927cb8ba5385bf945d582',\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.losses import binary_crossentropy\n",
    "from keras import backend as K\n",
    "\n",
    "def dice_coef(y_true, y_pred):\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred = K.cast(y_pred, 'float32')\n",
    "    y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')\n",
    "    intersection = y_true_f * y_pred_f\n",
    "    score = 2. * K.sum(intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))\n",
    "    return score\n",
    "\n",
    "def dice_loss(y_true, y_pred):\n",
    "    smooth = 1.\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = y_true_f * y_pred_f\n",
    "    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "    return 1. - score\n",
    "\n",
    "def bce_dice_loss(y_true, y_pred):\n",
    "    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n",
    "\n",
    "def bce_no_empty(y_true, y_pred):\n",
    "    return K.max(K.max(y_true,axis=0),axis=1)*binary_crossentropy(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def focal_loss(gamma=2., alpha=.25):\n",
    "    def focal_loss_fixed(y_true, y_pred):\n",
    "        eps = 1e-14\n",
    "        y_pred=K.clip(y_pred, eps, 1.-eps)#improve the stability of the focal loss and see issues 1 for more information\n",
    "        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n",
    "        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n",
    "        pt_1 = K.clip(pt_1, eps, 1.-eps)\n",
    "        pt_0 = K.clip(pt_0, eps, 1.-eps)\n",
    "        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1))-K.sum((1-alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n",
    "    return focal_loss_fixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lovasz_grad(gt_sorted):\n",
    "    \"\"\"\n",
    "    Computes gradient of the Lovasz extension w.r.t sorted errors\n",
    "    See Alg. 1 in paper\n",
    "    \"\"\"\n",
    "    gts = tf.reduce_sum(gt_sorted)\n",
    "    intersection = gts - tf.cumsum(gt_sorted)\n",
    "    union = gts + tf.cumsum(1. - gt_sorted)\n",
    "    jaccard = 1. - intersection / union\n",
    "    jaccard = tf.concat((jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)\n",
    "    return jaccard\n",
    "\n",
    "\n",
    "# --------------------------- BINARY LOSSES ---------------------------\n",
    "\n",
    "def lovasz_hinge(logits, labels, per_image=True, ignore=None):\n",
    "    \"\"\"\n",
    "    Binary Lovasz hinge loss\n",
    "      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n",
    "      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n",
    "      per_image: compute the loss per image instead of per batch\n",
    "      ignore: void class id\n",
    "    \"\"\"\n",
    "    if per_image:\n",
    "        def treat_image(log_lab):\n",
    "            log, lab = log_lab\n",
    "            log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)\n",
    "            log, lab = flatten_binary_scores(log, lab, ignore)\n",
    "            return lovasz_hinge_flat(log, lab)\n",
    "        losses = tf.map_fn(treat_image, (logits, labels), dtype=tf.float32)\n",
    "        loss = tf.reduce_mean(losses)\n",
    "    else:\n",
    "        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n",
    "    return loss\n",
    "\n",
    "\n",
    "def lovasz_hinge_flat(logits, labels):\n",
    "    \"\"\"\n",
    "    Binary Lovasz hinge loss\n",
    "      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n",
    "      labels: [P] Tensor, binary ground truth labels (0 or 1)\n",
    "      ignore: label to ignore\n",
    "    \"\"\"\n",
    "\n",
    "    def compute_loss():\n",
    "        labelsf = tf.cast(labels, logits.dtype)\n",
    "        signs = 2. * labelsf - 1.\n",
    "        errors = 1. - logits * tf.stop_gradient(signs)\n",
    "        errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name=\"descending_sort\")\n",
    "        gt_sorted = tf.gather(labelsf, perm)\n",
    "        grad = lovasz_grad(gt_sorted)\n",
    "        loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name=\"loss_non_void\")\n",
    "        return loss\n",
    "\n",
    "    # deal with the void prediction case (only void pixels)\n",
    "    loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),\n",
    "                   lambda: tf.reduce_sum(logits) * 0.,\n",
    "                   compute_loss,\n",
    "                   strict=True,\n",
    "                   name=\"loss\"\n",
    "                   )\n",
    "    return loss\n",
    "\n",
    "\n",
    "def flatten_binary_scores(scores, labels, ignore=None):\n",
    "    \"\"\"\n",
    "    Flattens predictions in the batch (binary case)\n",
    "    Remove labels equal to 'ignore'\n",
    "    \"\"\"\n",
    "    scores = tf.reshape(scores, (-1,))\n",
    "    labels = tf.reshape(labels, (-1,))\n",
    "    if ignore is None:\n",
    "        return scores, labels\n",
    "    valid = tf.not_equal(labels, ignore)\n",
    "    vscores = tf.boolean_mask(scores, valid, name='valid_scores')\n",
    "    vlabels = tf.boolean_mask(labels, valid, name='valid_labels')\n",
    "    return vscores, vlabels\n",
    "\n",
    "def lovasz_loss(y_true, y_pred):\n",
    "    y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')\n",
    "    #logits = K.log(y_pred / (1. - y_pred))\n",
    "    logits = y_pred #Jiaxin\n",
    "    loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)\n",
    "    return loss\n",
    "\n",
    "def lovasz_no_empty(y_true, y_pred):\n",
    "    return K.max(K.max(y_true,axis=1),axis=1)*lovasz_loss(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iou_loss_core(y_true, y_pred, smooth=1):\n",
    "    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n",
    "    union = K.sum(y_true,-1) + K.sum(y_pred,-1) - intersection\n",
    "    iou = (intersection + smooth) / (union + smooth)\n",
    "    return iou"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_iou_vector(A, B):\n",
    "    batch_size = A.shape[0]\n",
    "    metric = []\n",
    "    for batch in range(batch_size):\n",
    "        t, p = A[batch]>0, B[batch]>0\n",
    "#         if np.count_nonzero(t) == 0 and np.count_nonzero(p) > 0:\n",
    "#             metric.append(0)\n",
    "#             continue\n",
    "#         if np.count_nonzero(t) >= 1 and np.count_nonzero(p) == 0:\n",
    "#             metric.append(0)\n",
    "#             continue\n",
    "#         if np.count_nonzero(t) == 0 and np.count_nonzero(p) == 0:\n",
    "#             metric.append(1)\n",
    "#             continue\n",
    "        \n",
    "        intersection = np.logical_and(t, p)\n",
    "        union = np.logical_or(t, p)\n",
    "        iou = (np.sum(intersection > 0) + 1e-10 )/ (np.sum(union > 0) + 1e-10)\n",
    "        thresholds = np.arange(0.5, 1, 0.05)\n",
    "        s = []\n",
    "        for thresh in thresholds:\n",
    "            s.append(iou > thresh)\n",
    "        metric.append(np.mean(s))\n",
    "\n",
    "    return np.mean(metric)\n",
    "\n",
    "def my_iou_metric(label, pred):\n",
    "    return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)\n",
    "\n",
    "def my_iou_metric_2(label, pred):\n",
    "    return tf.py_func(get_iou_vector, [label, pred >0], tf.float64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conv_params(**params):\n",
    "    default_conv_params = {\n",
    "        'kernel_initializer': 'glorot_uniform',\n",
    "        'use_bias': False,\n",
    "        'padding': 'valid',\n",
    "    }\n",
    "    default_conv_params.update(params)\n",
    "    return default_conv_params\n",
    "\n",
    "\n",
    "def get_bn_params(**params):\n",
    "    default_bn_params = {\n",
    "        'axis': 3,\n",
    "        'momentum': 0.99,\n",
    "        'epsilon': 2e-5,\n",
    "        'center': True,\n",
    "        'scale': True,\n",
    "    }\n",
    "    default_bn_params.update(params)\n",
    "    return default_bn_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_block_names(stage, block):\n",
    "    name_base = 'stage{}_unit{}_'.format(stage + 1, block + 1)\n",
    "    conv_name = name_base + 'conv'\n",
    "    bn_name = name_base + 'bn'\n",
    "    relu_name = name_base + 'relu'\n",
    "    sc_name = name_base + 'sc'\n",
    "    return conv_name, bn_name, relu_name, sc_name\n",
    "\n",
    "def basic_identity_block(filters, stage, block):\n",
    "    def layer(input_tensor):\n",
    "        conv_params = get_conv_params()\n",
    "        bn_params = get_bn_params()\n",
    "        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n",
    "        x = Activation('relu', name=relu_name + '1')(x)\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), name=conv_name + '1', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '2')(x)\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n",
    "\n",
    "        x = Add()([x, input_tensor])\n",
    "        return x\n",
    "\n",
    "    return layer\n",
    "\n",
    "\n",
    "def basic_conv_block(filters, stage, block, strides=(2, 2)):\n",
    "    def layer(input_tensor):\n",
    "        conv_params = get_conv_params()\n",
    "        bn_params = get_bn_params()\n",
    "        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n",
    "        x = Activation('relu', name=relu_name + '1')(x)\n",
    "        shortcut = x\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), strides=strides, name=conv_name + '1', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '2')(x)\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n",
    "\n",
    "        shortcut = Conv2D(filters, (1, 1), name=sc_name, strides=strides, **conv_params)(shortcut)\n",
    "        x = Add()([x, shortcut])\n",
    "        return x\n",
    "\n",
    "    return layer\n",
    "\n",
    "def usual_conv_block(filters, stage, block, strides=(2, 2)):\n",
    "    def layer(input_tensor):\n",
    "        conv_params = get_conv_params()\n",
    "        bn_params = get_bn_params()\n",
    "        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n",
    "        x = Activation('relu', name=relu_name + '1')(x)\n",
    "        shortcut = x\n",
    "        x = Conv2D(filters, (1, 1), name=conv_name + '1', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '2')(x)\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), strides=strides, name=conv_name + '2', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '3', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '3')(x)\n",
    "        x = Conv2D(filters*4, (1, 1), name=conv_name + '3', **conv_params)(x)\n",
    "\n",
    "        shortcut = Conv2D(filters*4, (1, 1), name=sc_name, strides=strides, **conv_params)(shortcut)\n",
    "        x = Add()([x, shortcut])\n",
    "        return x\n",
    "\n",
    "    return layer\n",
    "\n",
    "\n",
    "def usual_identity_block(filters, stage, block):\n",
    "    def layer(input_tensor):\n",
    "        conv_params = get_conv_params()\n",
    "        bn_params = get_bn_params()\n",
    "        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n",
    "        x = Activation('relu', name=relu_name + '1')(x)\n",
    "        x = Conv2D(filters, (1, 1), name=conv_name + '1', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '2')(x)\n",
    "        x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n",
    "\n",
    "        x = BatchNormalization(name=bn_name + '3', **bn_params)(x)\n",
    "        x = Activation('relu', name=relu_name + '3')(x)\n",
    "        x = Conv2D(filters*4, (1, 1), name=conv_name + '3', **conv_params)(x)\n",
    "\n",
    "        x = Add()([x, input_tensor])\n",
    "        return x\n",
    "\n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_resnet(\n",
    "     repetitions=(2, 2, 2, 2),\n",
    "     include_top=True,\n",
    "     input_tensor=None,\n",
    "     input_shape=None,\n",
    "     classes=1000,\n",
    "     block_type='usual'):\n",
    "    # Determine proper input shape\n",
    "    input_shape = _obtain_input_shape(input_shape,\n",
    "                                      default_size=224,\n",
    "                                      min_size=128,\n",
    "                                      data_format='channels_last',\n",
    "                                      require_flatten=include_top)\n",
    "\n",
    "    if input_tensor is None:\n",
    "        img_input = Input(shape=input_shape, name='data')\n",
    "    else:\n",
    "        if not K.is_keras_tensor(input_tensor):\n",
    "            img_input = Input(tensor=input_tensor, shape=input_shape)\n",
    "        else:\n",
    "            img_input = input_tensor\n",
    "    \n",
    "    # get parameters for model layers\n",
    "    no_scale_bn_params = get_bn_params(scale=False)\n",
    "    bn_params = get_bn_params()\n",
    "    conv_params = get_conv_params()\n",
    "    init_filters = 64\n",
    "\n",
    "    if block_type == 'basic':\n",
    "        conv_block = basic_conv_block\n",
    "        identity_block = basic_identity_block\n",
    "    else:\n",
    "        conv_block = usual_conv_block\n",
    "        identity_block = usual_identity_block\n",
    "    \n",
    "    # resnet bottom\n",
    "    x = BatchNormalization(name='bn_data', **no_scale_bn_params)(img_input)\n",
    "    x = ZeroPadding2D(padding=(3, 3))(x)\n",
    "    x = Conv2D(init_filters, (7, 7), strides=(2, 2), name='conv0', **conv_params)(x)\n",
    "    x = BatchNormalization(name='bn0', **bn_params)(x)\n",
    "    x = Activation('relu', name='relu0')(x)\n",
    "    x = ZeroPadding2D(padding=(1, 1))(x)\n",
    "    x = MaxPooling2D((3, 3), strides=(2, 2), padding='valid', name='pooling0')(x)\n",
    "    \n",
    "    # resnet body\n",
    "    for stage, rep in enumerate(repetitions):\n",
    "        for block in range(rep):\n",
    "            \n",
    "            filters = init_filters * (2**stage)\n",
    "            \n",
    "            # first block of first stage without strides because we have maxpooling before\n",
    "            if block == 0 and stage == 0:\n",
    "                x = conv_block(filters, stage, block, strides=(1, 1))(x)\n",
    "                \n",
    "            elif block == 0:\n",
    "                x = conv_block(filters, stage, block, strides=(2, 2))(x)\n",
    "                \n",
    "            else:\n",
    "                x = identity_block(filters, stage, block)(x)\n",
    "                \n",
    "    x = BatchNormalization(name='bn1', **bn_params)(x)\n",
    "    x = Activation('relu', name='relu1')(x)\n",
    "\n",
    "    # resnet top\n",
    "    if include_top:\n",
    "        x = GlobalAveragePooling2D(name='pool1')(x)\n",
    "        x = Dense(classes, name='fc1')(x)\n",
    "        x = Activation('softmax', name='softmax')(x)\n",
    "\n",
    "    # Ensure that the model takes into account any potential predecessors of `input_tensor`.\n",
    "    if input_tensor is not None:\n",
    "        inputs = get_source_inputs(input_tensor)\n",
    "    else:\n",
    "        inputs = img_input\n",
    "        \n",
    "    # Create model.\n",
    "    model = Model(inputs, x)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ResNet34(input_shape, input_tensor=None, weights=None, classes=1000, include_top=True):\n",
    "    model = build_resnet(input_tensor=input_tensor,\n",
    "                         input_shape=input_shape,\n",
    "                         repetitions=(3, 4, 6, 3),\n",
    "                         classes=classes,\n",
    "                         include_top=include_top,\n",
    "                         block_type='basic')\n",
    "    model.name = 'resnet34'\n",
    "\n",
    "    if weights:\n",
    "        load_model_weights(weights_collection, model, weights, classes, include_top)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ResNet101(input_shape, input_tensor=None, weights=None, classes=1000, include_top=True):\n",
    "    model = build_resnet(input_tensor=input_tensor,\n",
    "                         input_shape=input_shape,\n",
    "                         repetitions=(3, 4, 23, 3),\n",
    "                         classes=classes,\n",
    "                         include_top=include_top)\n",
    "    model.name = 'resnet101'\n",
    "\n",
    "    if weights:\n",
    "        load_model_weights(weights_collection, model, weights, classes, include_top)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "enco = ResNet34(input_shape=(256, 256, 3), weights=None, include_top=True, classes=class_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_memory_usage(batch_size, model):\n",
    "    import numpy as np\n",
    "    from keras import backend as K\n",
    "\n",
    "    shapes_mem_count = 0\n",
    "    for l in model.layers:\n",
    "        single_layer_mem = 1\n",
    "        for s in l.output_shape:\n",
    "            if s is None:\n",
    "                continue\n",
    "            single_layer_mem *= s\n",
    "        shapes_mem_count += single_layer_mem\n",
    "\n",
    "    trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)])\n",
    "    non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])\n",
    "\n",
    "    total_memory = 4.0*batch_size*(shapes_mem_count + trainable_count + non_trainable_count)\n",
    "    gbytes = np.round(total_memory / (1024.0 ** 3), 3)\n",
    "    return gbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20.381"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_model_memory_usage(100, enco)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "enco.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=adam(lr=0.001),\n",
    "              loss='binary_crossentropy', \n",
    "              metrics=[my_iou_metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_process(model, epoch=50, batch_size=16, x_train, y_train, x_valid, y_valid, load=3000):\n",
    "    loss = np.inf\n",
    "    for i in range(epoch):\n",
    "        splits = x_train.shape[0]//load+1\n",
    "        for split in range(splits):\n",
    "            model.fit(x_train[split*load:split*load+load], y_train[split*load:split*load+load], \n",
    "                      epochs = epochs, \n",
    "                      batch_size = batch_size, \n",
    "                      verbose = 1)\n",
    "        preds = model.predict(x_valid)\n",
    "        new_loss = binary_crossentropy(y_valid, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(\"stage_1.yaml\")\n",
    "outputs_ = model.layers[-1].input\n",
    "final_fusion = Conv2D(1, 1, padding='same', activation=None, name=\"non_sigmoid_fusion\")(outputs_)\n",
    "final_fusion = Activation(\"sigmoid\")(final_fusion)\n",
    "inputs_ = model.layers[0].input\n",
    "model_fine = Model(inputs_, final_fusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fine.compile(optimizer=adam(lr=0.0001),\n",
    "              loss=\"binary_crossentropy\", \n",
    "              metrics=[my_iou_metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6400 samples, validate on 800 samples\n",
      "Epoch 1/100\n",
      " - 165s - loss: 11.9371 - my_iou_metric: 0.1379 - val_loss: 12.2717 - val_my_iou_metric: 0.1194\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 12.27170, saving model to stage_2.model\n",
      "Epoch 2/100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-77-b79d930edec7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     15\u001b[0m                      \u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m                      \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mearly_stopping\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel_checkpoint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreduce_lr\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m                      verbose = 2)\n\u001b[0m",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1705\u001b[1;33m                               validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1706\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[1;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m   1233\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1234\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1235\u001b[1;33m                     \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1236\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1237\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[1;32m-> 2478\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2479\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2480\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    875\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    876\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 877\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    878\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    879\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1098\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1099\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1100\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1101\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1270\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1271\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1272\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1273\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1274\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1276\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1277\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1278\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1279\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1280\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1261\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1262\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1263\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1264\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1265\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\zheng\\anaconda3\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1348\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1349\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1350\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1351\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1352\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "early_stopping = EarlyStopping(monitor='my_iou_metric', mode ='max', patience=15, verbose=1)\n",
    "save_model_name = 'stage_2.model'\n",
    "model_checkpoint = ModelCheckpoint(save_model_name, monitor='val_loss', mode='min',\n",
    "                                   save_best_only=True, verbose=1)\n",
    "\n",
    "reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode='max', factor=0.5, patience=5,\n",
    "                              min_lr=1e-7, verbose=1)\n",
    "\n",
    "epochs = 100\n",
    "batch_size = 10\n",
    "\n",
    "history = model_fine.fit(x_train, y_train,\n",
    "                     validation_data = [x_valid, y_valid], \n",
    "                     epochs = epochs, \n",
    "                     batch_size = batch_size, \n",
    "                     callbacks = [early_stopping, model_checkpoint, reduce_lr],\n",
    "                     verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_ = model_fine.layers[-1].input\n",
    "model_fine = Model(model_fine.layers[0].input, output_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fine.compile(optimizer=adam(lr=0.0001),\n",
    "              loss=lovasz_loss, \n",
    "              metrics=[my_iou_metric])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}