Nearest Neighbors - Features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\eric\\Anaconda2\\envs\\exts-aml2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import h5py\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "% matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import the data\n",
    "X = np.load(os.path.join('data','small_images.npy'))\n",
    "labels = pd.read_pickle(os.path.join(\"data\",\"labels.pkl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decide what we want to classify on, let's start with normal or abnormal\n",
    "y = labels.ABNORMAL.values.astype(np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calculate_recall(y, yhat):\n",
    "    tp = np.sum(y & yhat)\n",
    "    fn = np.sum(y & ~yhat)\n",
    "    recall = tp / (tp + fn)\n",
    "    return recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fc6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 4096)\n",
      "X_te (65, 4096)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# vgg pool 5 features\n",
    "with h5py.File('data/vgg_fc6_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_fc6_features = features['vgg_19']['fc6'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_fc6_flat = vgg_fc6_features.reshape([322,-1])\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_fc6_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.579767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "8        30               1  0.626459\n",
       "9        30               2  0.626459\n",
       "5        20               2  0.614786\n",
       "7        25               2  0.610895\n",
       "6        25               1  0.607004\n",
       "4        20               1  0.603113\n",
       "1        10               2  0.591440\n",
       "0        10               1  0.587549\n",
       "2        15               1  0.583658\n",
       "3        15               2  0.579767"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.7076923076923077\n",
      "Positive Predictions: 0\n",
      "Recall: 0.0\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 25088)\n",
      "X_te (65, 25088)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# vgg pool 5 features\n",
    "with h5py.File('data/vgg_p5_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_features = features['vgg_19']['pool5'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_p5_flat = vgg_features.reshape([322,-1])\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_p5_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test baseline 0.7076923076923077\n",
      "Train baseline 0.6264591439688716\n"
     ]
    }
   ],
   "source": [
    "# distribution in y_te\n",
    "print(\"Test baseline\", np.sum(y_te == 0) / y_te.shape[0])\n",
    "print(\"Train baseline\", np.sum(y_tr == 0) / y_tr.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.630350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "9        30               2  0.630350\n",
       "7        25               2  0.626459\n",
       "8        30               1  0.626459\n",
       "0        10               1  0.622568\n",
       "4        20               1  0.622568\n",
       "6        25               1  0.622568\n",
       "5        20               2  0.618677\n",
       "2        15               1  0.610895\n",
       "1        10               2  0.607004\n",
       "3        15               2  0.607004"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.7076923076923077\n",
      "Recall: 0.0\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# vgg pool 3 features\n",
    "with h5py.File('data/vgg_p4_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_p4_features = features['vgg_19']['pool4'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_p4_flat = vgg_p4_features.reshape([322,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 100352)\n",
      "X_te (65, 100352)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n",
      "Most frequent baseline: 0.7076923076923077\n"
     ]
    }
   ],
   "source": [
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_p4_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)\n",
    "\n",
    "print(\"Most frequent baseline:\", np.sum(y_te == 0) / y_te.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('scaler', None), ('knn', KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'))]),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'knn__n_neighbors': [10, 15, 20, 25, 30], 'knn__p': [1, 2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "9        30               2  0.622568\n",
       "0        10               1  0.618677\n",
       "8        30               1  0.618677\n",
       "1        10               2  0.607004\n",
       "2        15               1  0.607004\n",
       "3        15               2  0.607004\n",
       "6        25               1  0.607004\n",
       "4        20               1  0.599222\n",
       "5        20               2  0.599222\n",
       "7        25               2  0.591440"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.676923076923077\n",
      "Positive Predictions: 4\n",
      "True Positives: 19\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"True Positives:\", np.sum(y_te))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# vgg pool 3 features\n",
    "with h5py.File('data/vgg_p3_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_p3_features = features['vgg_19']['pool3'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_p3_flat = vgg_p3_features.reshape([322,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 200704)\n",
      "X_te (65, 200704)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n",
      "Most frequent baseline: 0.7076923076923077\n"
     ]
    }
   ],
   "source": [
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_p3_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)\n",
    "\n",
    "print(\"Most frequent baseline:\", np.sum(y_te == 0) / y_te.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "0        10               1  0.626459\n",
       "3        15               2  0.622568\n",
       "8        30               1  0.622568\n",
       "9        30               2  0.618677\n",
       "4        20               1  0.614786\n",
       "2        15               1  0.610895\n",
       "1        10               2  0.603113\n",
       "7        25               2  0.595331\n",
       "5        20               2  0.591440\n",
       "6        25               1  0.587549"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.676923076923077\n",
      "Positive Predictions: 4\n",
      "True Positives: 19\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"True Positives:\", np.sum(y_te))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# vgg pool 2 features\n",
    "with h5py.File('data/vgg_p2_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_p2_features = features['vgg_19']['pool2'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_p2_flat = vgg_p2_features.reshape([322,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 401408)\n",
      "X_te (65, 401408)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_p2_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.630350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.579767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "9        30               2  0.630350\n",
       "0        10               1  0.622568\n",
       "4        20               1  0.622568\n",
       "8        30               1  0.614786\n",
       "2        15               1  0.610895\n",
       "5        20               2  0.610895\n",
       "6        25               1  0.610895\n",
       "3        15               2  0.603113\n",
       "7        25               2  0.599222\n",
       "1        10               2  0.579767"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.7076923076923077\n",
      "Positive Predictions: 0\n",
      "True Positives: 19\n",
      "Recall: 0.0\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"True Positives:\", np.sum(y_te))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# vgg pool 2 features\n",
    "with h5py.File('data/vgg_p1_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    vgg_p1_features = features['vgg_19']['pool1'].value\n",
    "    \n",
    "# flatten the array\n",
    "vgg_p1_flat = vgg_p1_features.reshape([322,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 802816)\n",
      "X_te (65, 802816)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(vgg_p1_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "2        15               1  0.622568\n",
       "4        20               1  0.618677\n",
       "6        25               1  0.618677\n",
       "9        30               2  0.618677\n",
       "0        10               1  0.614786\n",
       "1        10               2  0.610895\n",
       "7        25               2  0.610895\n",
       "8        30               1  0.610895\n",
       "3        15               2  0.603113\n",
       "5        20               2  0.603113"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.5846153846153846\n",
      "Positive Predictions: 10\n",
      "True Positives: 19\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"True Positives:\", np.sum(y_te))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Block 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# resnet features\n",
    "with h5py.File('data/rn_b1_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    rn_b1_features = features['resnet_v2_152']['block1'].value\n",
    "    \n",
    "# flatten the array\n",
    "rn_b1_flat = rn_b1_features.reshape([322,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 200704)\n",
      "X_te (65, 200704)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(rn_b1_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('scaler', None), ('knn', KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'))]),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'knn__n_neighbors': [10, 15, 20, 25, 30], 'knn__p': [1, 2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "0        10               1  0.618677\n",
       "8        30               1  0.614786\n",
       "2        15               1  0.610895\n",
       "4        20               1  0.607004\n",
       "9        30               2  0.607004\n",
       "1        10               2  0.595331\n",
       "5        20               2  0.595331\n",
       "6        25               1  0.587549\n",
       "7        25               2  0.587549\n",
       "3        15               2  0.583658"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6153846153846154"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.score(X_te, y_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.predict(X_te)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Block 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 50176)\n",
      "X_te (65, 50176)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# resnet features\n",
    "with h5py.File('data/rn_b3_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    rn_b3_features = features['resnet_v2_152']['block3'].value\n",
    "    \n",
    "# flatten the array\n",
    "rn_b3_flat = rn_b3_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(rn_b3_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "8        30               1  0.618677\n",
       "0        10               1  0.607004\n",
       "9        30               2  0.607004\n",
       "3        15               2  0.603113\n",
       "4        20               1  0.603113\n",
       "6        25               1  0.599222\n",
       "2        15               1  0.595331\n",
       "5        20               2  0.595331\n",
       "1        10               2  0.587549\n",
       "7        25               2  0.583658"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6615384615384615"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.score(X_te, y_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.predict(X_te)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Block 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 100352)\n",
      "X_te (65, 100352)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# resnet features\n",
    "with h5py.File('data/rn_b4_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    rn_b4_features = features['resnet_v2_152']['block4'].value\n",
    "    \n",
    "# flatten the array\n",
    "rn_b4_flat = rn_b4_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(rn_b4_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "2        15               1  0.622568\n",
       "9        30               2  0.622568\n",
       "4        20               1  0.618677\n",
       "5        20               2  0.618677\n",
       "7        25               2  0.614786\n",
       "8        30               1  0.614786\n",
       "3        15               2  0.610895\n",
       "6        25               1  0.607004\n",
       "1        10               2  0.603113\n",
       "0        10               1  0.599222"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.676923076923077"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.score(X_te, y_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.predict(X_te)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 1001)\n",
      "X_te (65, 1001)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# resnet features\n",
    "with h5py.File('data/rn_l_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    rn_l_features = features['resnet_v2_152']['logits'].value\n",
    "    \n",
    "# flatten the array\n",
    "rn_l_flat = rn_l_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(rn_l_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.630350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.571984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "6        25               1  0.630350\n",
       "5        20               2  0.626459\n",
       "8        30               1  0.626459\n",
       "4        20               1  0.622568\n",
       "7        25               2  0.618677\n",
       "9        30               2  0.614786\n",
       "1        10               2  0.607004\n",
       "2        15               1  0.603113\n",
       "3        15               2  0.603113\n",
       "0        10               1  0.571984"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7076923076923077"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_cv.score(X_te, y_te)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inception"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mixed_3a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 852640)\n",
      "X_te (65, 852640)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_3a_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_3a_features = features['Mixed_3a'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_3a_flat = inc_3a_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_3a_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "0        10               1  0.626459\n",
       "4        20               1  0.626459\n",
       "1        10               2  0.614786\n",
       "6        25               1  0.614786\n",
       "7        25               2  0.614786\n",
       "2        15               1  0.610895\n",
       "5        20               2  0.610895\n",
       "8        30               1  0.610895\n",
       "3        15               2  0.607004\n",
       "9        30               2  0.607004"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6615384615384615\n",
      "Positive Predictions: 5\n",
      "True Positives: 19\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Positive Predictions:\", np.sum(yhat))\n",
    "print(\"True Positives:\", np.sum(y_te))\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed_5e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 470400)\n",
      "X_te (65, 470400)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_5e_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_5e_features = features['Mixed_5e'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_5e_flat = inc_5e_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_5e_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.630350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "0        10               1  0.630350\n",
       "8        30               1  0.626459\n",
       "9        30               2  0.626459\n",
       "4        20               1  0.610895\n",
       "3        15               2  0.607004\n",
       "6        25               1  0.607004\n",
       "2        15               1  0.603113\n",
       "5        20               2  0.603113\n",
       "7        25               2  0.599222\n",
       "1        10               2  0.595331"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6615384615384615\n",
      "Recall: 0.10526315789473684\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed_6a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 295936)\n",
      "X_te (65, 295936)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_6a_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_6a_features = features['Mixed_6a'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_6a_flat = inc_6a_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_6a_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "4        20               1  0.618677\n",
       "8        30               1  0.618677\n",
       "9        30               2  0.618677\n",
       "2        15               1  0.614786\n",
       "0        10               1  0.610895\n",
       "3        15               2  0.603113\n",
       "1        10               2  0.599222\n",
       "5        20               2  0.599222\n",
       "6        25               1  0.599222\n",
       "7        25               2  0.599222"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.676923076923077\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed_6h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 295936)\n",
      "X_te (65, 295936)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_6h_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_6h_features = features['Mixed_6h'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_6h_flat = inc_6h_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_6h_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "2        15               1  0.626459\n",
       "4        20               1  0.614786\n",
       "6        25               1  0.610895\n",
       "8        30               1  0.607004\n",
       "9        30               2  0.607004\n",
       "3        15               2  0.603113\n",
       "0        10               1  0.595331\n",
       "5        20               2  0.595331\n",
       "7        25               2  0.595331\n",
       "1        10               2  0.591440"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6461538461538462\n",
      "Recall: 0.15789473684210525\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed 7d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 98304)\n",
      "X_te (65, 98304)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_7d_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_7d_features = features['Mixed_7d'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_7d_flat = inc_7d_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_7d_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.610895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.571984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.564202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.544747</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "8        30               1  0.610895\n",
       "0        10               1  0.607004\n",
       "9        30               2  0.599222\n",
       "4        20               1  0.591440\n",
       "6        25               1  0.587549\n",
       "5        20               2  0.583658\n",
       "7        25               2  0.571984\n",
       "1        10               2  0.564202\n",
       "2        15               1  0.548638\n",
       "3        15               2  0.544747"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6923076923076923\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed 7a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 98304)\n",
      "X_te (65, 98304)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_7a_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_7a_features = features['Mixed_7a'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_7a_flat = inc_7a_features.reshape([322,-1])\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_7a_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.595331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "8        30               1  0.618677\n",
       "9        30               2  0.614786\n",
       "0        10               1  0.599222\n",
       "4        20               1  0.595331\n",
       "6        25               1  0.595331\n",
       "7        25               2  0.591440\n",
       "1        10               2  0.587549\n",
       "2        15               1  0.587549\n",
       "3        15               2  0.583658\n",
       "5        20               2  0.583658"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6923076923076923\n",
      "Recall: 0.0\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PreLogits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 1536)\n",
      "X_te (65, 1536)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_flat_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_flat_features = features['PreLogitsFlatten'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_flat_flat = inc_flat_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_flat_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.591440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.583658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.579767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.571984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "9        30               2  0.618677\n",
       "8        30               1  0.614786\n",
       "4        20               1  0.607004\n",
       "5        20               2  0.599222\n",
       "6        25               1  0.591440\n",
       "1        10               2  0.587549\n",
       "7        25               2  0.587549\n",
       "2        15               1  0.583658\n",
       "0        10               1  0.579767\n",
       "3        15               2  0.571984"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.7076923076923077\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed 4a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 967872)\n",
      "X_te (65, 967872)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_4a_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_4a_features = features['Mixed_4a'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_4a_flat = inc_4a_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_4a_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.630350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.626459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.614786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.587549</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "1        10               2  0.630350\n",
       "5        20               2  0.626459\n",
       "8        30               1  0.626459\n",
       "9        30               2  0.626459\n",
       "3        15               2  0.622568\n",
       "4        20               1  0.622568\n",
       "7        25               2  0.622568\n",
       "2        15               1  0.614786\n",
       "6        25               1  0.614786\n",
       "0        10               1  0.587549"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6615384615384615\n",
      "Recall: 0.05263157894736842\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed_5a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_tr (257, 470400)\n",
      "X_te (65, 470400)\n",
      "y_tr (257,)\n",
      "y_te (65,)\n"
     ]
    }
   ],
   "source": [
    "# inception features\n",
    "with h5py.File('data/inc_m_5a_features.h5') as features:\n",
    "    filenames = features['filenames'].value\n",
    "    inc_5a_features = features['Mixed_5a'].value\n",
    "    \n",
    "# flatten the array\n",
    "inc_5a_flat = inc_5a_features.reshape([322,-1])\n",
    "\n",
    "# split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split data into training and test\n",
    "X_tr, X_te, y_tr,y_te = train_test_split(inc_5a_flat, y, test_size=0.20, random_state=0)\n",
    "print(\"X_tr\", X_tr.shape)\n",
    "print(\"X_te\", X_te.shape)\n",
    "print(\"y_tr\", y_tr.shape)\n",
    "print(\"y_te\", y_te.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbors</th>\n",
       "      <th>distance_metric</th>\n",
       "      <th>mean_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0.634241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>0.622568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.618677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.599222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbors distance_metric   mean_te\n",
       "8        30               1  0.634241\n",
       "9        30               2  0.622568\n",
       "1        10               2  0.618677\n",
       "2        15               1  0.618677\n",
       "3        15               2  0.618677\n",
       "4        20               1  0.618677\n",
       "5        20               2  0.618677\n",
       "6        25               1  0.618677\n",
       "7        25               2  0.618677\n",
       "0        10               1  0.599222"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('knn', knn)\n",
    "])\n",
    "\n",
    "grid = {\n",
    "    'knn__n_neighbors': [10, 15, 20, 25, 30],\n",
    "    'knn__p': [1, 2],\n",
    "}\n",
    "\n",
    "grid_cv = GridSearchCV(pipe, grid, cv=5)\n",
    "grid_cv.fit(X_tr, y_tr)\n",
    "\n",
    "# Collect results and sort them\n",
    "df = pd.DataFrame.from_items([\n",
    "    ('neighbors', grid_cv.cv_results_['param_knn__n_neighbors']),\n",
    "    ('distance_metric', grid_cv.cv_results_['param_knn__p']),\n",
    "    ('mean_te', grid_cv.cv_results_['mean_test_score'])\n",
    "])\n",
    "\n",
    "df.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.6923076923076923\n",
      "Recall: 0.0\n"
     ]
    }
   ],
   "source": [
    "test_acc = grid_cv.score(X_te, y_te)\n",
    "yhat = grid_cv.predict(X_te)\n",
    "recall = calculate_recall(y_te, yhat)\n",
    "print(\"Test accuracy:\", test_acc)\n",
    "print(\"Recall:\", recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}