Evaluation - CIFAR - Traditional Filters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load the prepared dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cifar10', 'mnist']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_DIR = 'C:/Users/MINH LE/Desktop/TraditionalvsCNN/dataset/'\n",
    "\n",
    "list_files = os.listdir(DATA_DIR)\n",
    "number_files = len(list_files)\n",
    "list_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. CIFAR10 Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**CLTP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob, cv2\n",
    "cltp = []\n",
    "\n",
    "CLTP_DIR = DATA_DIR + 'cifar10/cltp/test/'\n",
    "for img in glob.glob(CLTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    cltp.append(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cltp = np.asarray(cltp, dtype=np.float32)\n",
    "cltp.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**CSLBP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cslbp = []\n",
    "\n",
    "CSLBP_DIR = DATA_DIR + 'cifar10/cslbp/test/'\n",
    "for img in glob.glob(CSLBP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    cslbp.append(n)\n",
    "    \n",
    "cslbp = np.asarray(cslbp, dtype=np.float32)\n",
    "cslbp.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**GAUSSIAN FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gauss_filter = []\n",
    "\n",
    "GAUSSIAN_DIR = DATA_DIR + 'cifar10/gaussian/test/'\n",
    "for img in glob.glob(GAUSSIAN_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    gauss_filter.append(n)\n",
    "    \n",
    "gauss_filter = np.asarray(gauss_filter, dtype=np.float32)\n",
    "gauss_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**GLTP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gltp_filter = []\n",
    "\n",
    "GLTP_DIR = DATA_DIR + 'cifar10/gltp/test/'\n",
    "for img in glob.glob(GLTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    gltp_filter.append(n)\n",
    "    \n",
    "gltp_filter = np.asarray(gltp_filter, dtype=np.float32)\n",
    "gltp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LAPLACIAN FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "laplacian_filter = []\n",
    "\n",
    "LAPLACIAN_DIR = DATA_DIR + 'cifar10/laplacian/test/'\n",
    "for img in glob.glob(LAPLACIAN_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    laplacian_filter.append(n)\n",
    "    \n",
    "laplacian_filter = np.asarray(laplacian_filter, dtype=np.float32)\n",
    "laplacian_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LOCAL BINARY PATTERN FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lbp_filter = []\n",
    "\n",
    "LBP_DIR = DATA_DIR + 'cifar10/lbp/test/'\n",
    "for img in glob.glob(LBP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    lbp_filter.append(n)\n",
    "    \n",
    "lbp_filter = np.asarray(lbp_filter, dtype=np.float32)\n",
    "lbp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LOCAL TERNARY PATTERN**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ltp_filter = []\n",
    "\n",
    "LTP_DIR = DATA_DIR + 'cifar10/ltp/test/'\n",
    "for img in glob.glob(LTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    ltp_filter.append(n)\n",
    "    \n",
    "ltp_filter = np.asarray(ltp_filter, dtype=np.float32)\n",
    "ltp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**MC-LBP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mclbp_filter = []\n",
    "\n",
    "MCLBP_DIR = DATA_DIR + 'cifar10/mclbp/test/'\n",
    "for img in glob.glob(MCLBP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    mclbp_filter.append(n)\n",
    "    \n",
    "mclbp_filter = np.asarray(mclbp_filter, dtype=np.float32)\n",
    "mclbp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**PREWITT X FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prewittx_filter = []\n",
    "\n",
    "PREWITTX_DIR = DATA_DIR + 'cifar10/prewittx/test/'\n",
    "for img in glob.glob(PREWITTX_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    prewittx_filter.append(n)\n",
    "    \n",
    "prewittx_filter = np.asarray(prewittx_filter, dtype=np.float32)\n",
    "prewittx_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**PREWITT Y FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prewitty_filter = []\n",
    "\n",
    "PREWITTY_DIR = DATA_DIR + 'cifar10/prewitty/test/'\n",
    "for img in glob.glob(PREWITTY_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    prewitty_filter.append(n)\n",
    "    \n",
    "prewitty_filter = np.asarray(prewitty_filter, dtype=np.float32)\n",
    "prewitty_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SCBP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scbp_filter = []\n",
    "\n",
    "SCBP_DIR = DATA_DIR + 'cifar10/scbp/test/'\n",
    "for img in glob.glob(SCBP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    scbp_filter.append(n)\n",
    "    \n",
    "scbp_filter = np.asarray(scbp_filter, dtype=np.float32)\n",
    "scbp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SCS-LTP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scsltp_filter = []\n",
    "\n",
    "SCSLTP_DIR = DATA_DIR + 'cifar10/scsltp/test/'\n",
    "for img in glob.glob(SCSLTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    scsltp_filter.append(n)\n",
    "    \n",
    "scsltp_filter = np.asarray(scsltp_filter, dtype=np.float32)\n",
    "scsltp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SILTP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "siltp_filter = []\n",
    "\n",
    "SILTP_DIR = DATA_DIR + 'cifar10/siltp/test/'\n",
    "for img in glob.glob(SILTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    siltp_filter.append(n)\n",
    "    \n",
    "siltp_filter = np.asarray(siltp_filter, dtype=np.float32)\n",
    "siltp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SOBEL X FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sobelx_filter = []\n",
    "\n",
    "SOBELX_DIR = DATA_DIR + 'cifar10/sobelx/test/'\n",
    "for img in glob.glob(SOBELX_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    sobelx_filter.append(n)\n",
    "    \n",
    "sobelx_filter = np.asarray(sobelx_filter, dtype=np.float32)\n",
    "sobelx_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SOBEL Y FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sobely_filter = []\n",
    "\n",
    "SOBELY_DIR = DATA_DIR + 'cifar10/sobely/test/'\n",
    "for img in glob.glob(SOBELY_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    sobely_filter.append(n)\n",
    "    \n",
    "sobely_filter = np.asarray(sobely_filter, dtype=np.float32)\n",
    "sobely_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**XCSLTP FILTER**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 30, 30, 3)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xcsltp_filter = []\n",
    "\n",
    "XCSLTP_DIR = DATA_DIR + 'cifar10/xcsltp/test/'\n",
    "for img in glob.glob(XCSLTP_DIR + '*.jpg'):\n",
    "    n = cv2.imread(img)\n",
    "    xcsltp_filter.append(n)\n",
    "    \n",
    "xcsltp_filter = np.asarray(xcsltp_filter, dtype=np.float32)\n",
    "xcsltp_filter.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Test Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import MaxPooling2D\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([10000, 30, 30, 48])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = tf.keras.layers.Concatenate()([cltp, \n",
    "                                           cslbp, \n",
    "                                           gauss_filter, \n",
    "                                           gltp_filter,\n",
    "                                           laplacian_filter,\n",
    "                                           lbp_filter,\n",
    "                                           ltp_filter,\n",
    "                                           mclbp_filter,\n",
    "                                           prewittx_filter,\n",
    "                                           prewitty_filter,\n",
    "                                           scbp_filter,\n",
    "                                           scsltp_filter,\n",
    "                                           siltp_filter,\n",
    "                                           sobelx_filter,\n",
    "                                           sobely_filter,\n",
    "                                           xcsltp_filter])\n",
    "test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10 = keras.datasets.cifar10\n",
    "(_, _), (_, test_labels) = cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras in c:\\users\\minh le\\anaconda3\\lib\\site-packages (2.4.3)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\users\\minh le\\anaconda3\\lib\\site-packages (from keras) (1.19.5)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\minh le\\anaconda3\\lib\\site-packages (from keras) (5.3.1)\n",
      "Requirement already satisfied: h5py in c:\\users\\minh le\\anaconda3\\lib\\site-packages (from keras) (2.10.0)\n",
      "Requirement already satisfied: scipy>=0.14 in c:\\users\\minh le\\anaconda3\\lib\\site-packages (from keras) (1.6.0)\n",
      "Requirement already satisfied: six in c:\\users\\minh le\\anaconda3\\lib\\site-packages (from h5py->keras) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "# Load model 1\n",
    "model1 = load_model('C://Users/MINH LE/Desktop/TraditionalvsCNN/models/Experiment 1/Exp1-CIFAR-TraditionalFilter.h5')\n",
    "\n",
    "# Load model 2\n",
    "model2 = load_model('C://Users/MINH LE/Desktop/TraditionalvsCNN/models/Experiment 2/Exp2-CIFAR-TraditionalFilter.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "def eval_model(model, test_images, test_labels):\n",
    "  predictions = np.argmax(model.predict(test_images), axis=1)\n",
    "  print(str(accuracy_score(y_pred=predictions, y_true=test_labels)))\n",
    "  print(str(recall_score(y_pred=predictions, y_true=test_labels, average='macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1001\n",
      "0.1001\n"
     ]
    }
   ],
   "source": [
    "eval_model(model1, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0943\n",
      "0.09430000000000001\n"
     ]
    }
   ],
   "source": [
    "eval_model(model2, test_data, test_labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}