Baseline transfer learning ResNet50.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import time\n",
    "import glob, os, sys\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "from keras.preprocessing import image\n",
    "import keras\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from keras import optimizers\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.applications.resnet50 import ResNet50 as resnet\n",
    "from keras.applications.resnet50 import preprocess_input as resnet_pp\n",
    "\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_path = os.path.abspath(\"d://data/POETdataset/\")\n",
    "classes = ['aeroplane', 'boat', 'dog', 'bicycle', 'cat', 'cow', 'diningtable', 'horse', 'motorbike','sofa']\n",
    "IMG_SIZE = 224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_object(obj, filename):\n",
    "    with open(filename, 'wb') as output:  # Overwrites any existing file.\n",
    "        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)\n",
    "        \n",
    "def load_object(filename):\n",
    "    with open(filename, 'rb') as fp:\n",
    "        return pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a model by adding preprocessing before the pretrained CNN\n",
    "def get_feature_extraction_model():\n",
    "    cnn_object, pp_function, img_size = _get_pretrained_model()\n",
    "    model = keras.models.Sequential()\n",
    "    cnn_model = cnn_object(weights='imagenet', include_top=False, pooling='max')\n",
    "    model.add(keras.layers.Lambda(pp_function, name='preprocessing', input_shape=(img_size, img_size, 3)))\n",
    "    model.add(cnn_model)\n",
    "    return model\n",
    "\n",
    "# Unpacking information from the models dictionary\n",
    "def _get_pretrained_model():\n",
    "    cnn_object = resnet\n",
    "    pp_function = resnet_pp\n",
    "    img_size = IMG_SIZE\n",
    "    return cnn_object, pp_function, img_size\n",
    "\n",
    "def get_features(files, model):\n",
    "    # Load images based on the size of the Lambda layer \n",
    "    # provided as the first layer before the pretrained CNN\n",
    "    x = np.array([image.img_to_array(image.load_img(file, target_size=(model.layers[0].input_shape[1], model.layers[0].input_shape[1]))) for file in files])\n",
    "    return model.predict(x, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_feature_extraction_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aeroplane\n",
      "666/666 [==============================] - 9s 14ms/step\n",
      "(666, 2048)\n",
      "boat\n",
      "504/504 [==============================] - 5s 9ms/step\n",
      "(504, 2048)\n",
      "dog\n",
      "1257/1257 [==============================] - 11s 9ms/step\n",
      "(1257, 2048)\n",
      "bicycle\n",
      "536/536 [==============================] - 5s 9ms/step\n",
      "(536, 2048)\n",
      "cat\n",
      "1051/1051 [==============================] - 9s 9ms/step\n",
      "(1051, 2048)\n",
      "cow\n",
      "301/301 [==============================] - 3s 10ms/step\n",
      "(301, 2048)\n",
      "diningtable\n",
      "498/498 [==============================] - 5s 9ms/step\n",
      "(498, 2048)\n",
      "horse\n",
      "480/480 [==============================] - 4s 9ms/step\n",
      "(480, 2048)\n",
      "motorbike\n",
      "510/510 [==============================] - 5s 10ms/step\n",
      "(510, 2048)\n",
      "sofa\n",
      "467/467 [==============================] - 4s 9ms/step\n",
      "(467, 2048)\n"
     ]
    }
   ],
   "source": [
    "files_list = [glob.glob(os.path.join(main_path, 'PascalImages', '%s*' %class_)) for class_ in classes]\n",
    "\n",
    "for files in files_list:\n",
    "    assert len(files) > 0\n",
    "    \n",
    "new_dir = 'features'\n",
    "if not os.path.exists(new_dir):\n",
    "    os.makedirs(new_dir)\n",
    "\n",
    "files_dict = {class_name.replace('*', ''): class_files for class_name, class_files in zip(classes, files_list)}\n",
    "for class_number, (class_name, files) in enumerate(files_dict.items()):\n",
    "    print(class_name)\n",
    "    features = get_features(files, model)\n",
    "    print(features.shape)\n",
    "    save_object(features, new_dir + os.path.sep + class_name+'_features.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\data\\cogsci3\\features\\aeroplane_features.pkl\n",
      "(666, 2048)\n",
      "D:\\data\\cogsci3\\features\\boat_features.pkl\n",
      "(504, 2048)\n",
      "D:\\data\\cogsci3\\features\\dog_features.pkl\n",
      "(1257, 2048)\n",
      "D:\\data\\cogsci3\\features\\bicycle_features.pkl\n",
      "(536, 2048)\n",
      "D:\\data\\cogsci3\\features\\cat_features.pkl\n",
      "(1051, 2048)\n",
      "D:\\data\\cogsci3\\features\\cow_features.pkl\n",
      "(301, 2048)\n",
      "D:\\data\\cogsci3\\features\\diningtable_features.pkl\n",
      "(498, 2048)\n",
      "D:\\data\\cogsci3\\features\\horse_features.pkl\n",
      "(480, 2048)\n",
      "D:\\data\\cogsci3\\features\\motorbike_features.pkl\n",
      "(510, 2048)\n",
      "D:\\data\\cogsci3\\features\\sofa_features.pkl\n",
      "(467, 2048)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((6270, 2048), (6270, 10))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = []\n",
    "y = []\n",
    "for class_number, (class_name, files) in enumerate(files_dict.items()):\n",
    "    classfeatures_path = os.path.abspath(os.path.join('.', 'features', class_name + '_features.pkl'))\n",
    "    print(classfeatures_path)\n",
    "    features = load_object(classfeatures_path).reshape(-1,2048)\n",
    "    print(features.shape)\n",
    "    X.append(features)\n",
    "    y.append([class_number] * features.shape[0])\n",
    "    \n",
    "X = np.concatenate(X)\n",
    "y = np.concatenate(y)\n",
    "y = keras.utils.to_categorical(y)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5643, 2048), (5643, 10), (627, 2048), (627, 10))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, stratify=y)\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5643 samples, validate on 627 samples\n",
      "Epoch 1/100\n",
      "5643/5643 [==============================] - 2s 332us/step - loss: 5.9879 - acc: 0.4916 - val_loss: 0.9175 - val_acc: 0.6794\n",
      "Epoch 2/100\n",
      "5643/5643 [==============================] - 1s 123us/step - loss: 0.9088 - acc: 0.7163 - val_loss: 0.7247 - val_acc: 0.8038\n",
      "Epoch 3/100\n",
      "5643/5643 [==============================] - 1s 123us/step - loss: 0.7188 - acc: 0.7618 - val_loss: 0.7199 - val_acc: 0.7831\n",
      "Epoch 4/100\n",
      "5643/5643 [==============================] - 1s 124us/step - loss: 0.6217 - acc: 0.7948 - val_loss: 0.6747 - val_acc: 0.8214\n",
      "Epoch 5/100\n",
      "5643/5643 [==============================] - 1s 125us/step - loss: 0.5866 - acc: 0.8005 - val_loss: 0.6801 - val_acc: 0.8166\n",
      "Epoch 6/100\n",
      "5643/5643 [==============================] - 1s 126us/step - loss: 0.5295 - acc: 0.8182 - val_loss: 0.6621 - val_acc: 0.8293\n",
      "Epoch 7/100\n",
      "5643/5643 [==============================] - 1s 123us/step - loss: 0.4930 - acc: 0.8347 - val_loss: 0.6819 - val_acc: 0.8262\n",
      "Epoch 8/100\n",
      "5643/5643 [==============================] - 1s 124us/step - loss: 0.4527 - acc: 0.8435 - val_loss: 0.6491 - val_acc: 0.8357\n",
      "Epoch 9/100\n",
      "3552/5643 [=================>............] - ETA: 0s - loss: 0.4046 - acc: 0.8612"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-25e4adf9bbf9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     11\u001b[0m                     \u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m                     \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m                     validation_data=(X_test, y_test))\n\u001b[0m",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1039\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2713\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2714\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2674\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2675\u001b[1;33m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2676\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1397\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1398\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1399\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1400\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1401\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dense_model = Sequential()\n",
    "# model.add(Flatten(input_shape=(1,1,2048)))\n",
    "dense_model.add(Dense(units=128, activation='relu', input_shape=(2048,)))\n",
    "dense_model.add(Dropout(0.25))\n",
    "dense_model.add(Dense(units=10, activation='softmax'))\n",
    "\n",
    "dense_model.compile(loss='categorical_crossentropy',\n",
    "             optimizer=Adam(),\n",
    "             metrics=['accuracy'])\n",
    "history = dense_model.fit(X_train,y_train,\n",
    "                    batch_size = 32,\n",
    "                    epochs = 100,\n",
    "                    validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}