starskope_3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cItRVIv8QpLR"
   },
   "source": [
    "# STARSKØPE\n",
    "\n",
    "## Part III: Restricted Boltzmann Machines\n",
    "\n",
    "\n",
    "WORK IN PROGRESS "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Qc_g3hRiPP3p"
   },
   "source": [
    "Bernoulli Restricted Boltzmann Machine (RBM).\n",
    "\n",
    "    A Restricted Boltzmann Machine with binary visible units and\n",
    "    binary hidden units. Parameters are estimated using Stochastic Maximum\n",
    "    Likelihood (SML), also known as Persistent Contrastive Divergence (PCD)\n",
    "    [2].\n",
    "\n",
    "    The time complexity of this implementation is ``O(d ** 2)`` assuming\n",
    "    d ~ n_features ~ n_components."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1ZRUudjsf0ofOMVjhzfF239Df3BaO2jqp#scrollTo=n0aJxtzZMCSI&uniqifier=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T00:19:49.103462Z",
     "start_time": "2020-08-26T00:19:49.086139Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "8B6xSvu5PG7j"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "%matplotlib inline\n",
    "from matplotlib.colors import LogNorm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-bright')\n",
    " \n",
    "\n",
    "font_dict={'family':'\"Titillium Web\", monospace','size':16}\n",
    "mpl.rc('font',**font_dict)\n",
    "\n",
    "\n",
    "#ignore pink warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "# Allow for large # columns\n",
    "pd.set_option('display.max_columns', 0)\n",
    "# pd.set_option('display.max_rows','')\n",
    "\n",
    "\n",
    "from scipy.ndimage import convolve\n",
    "from sklearn import linear_model, datasets, metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import BernoulliRBM\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use this instead if using Jupyter notebook/local\n",
    "# import os, glob, sys\n",
    "# home = os.path.abspath(os.curdir)\n",
    "# os.listdir(home)\n",
    "# %cd ../\n",
    "# %cd data/\n",
    "# %ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T00:20:32.373558Z",
     "start_time": "2020-08-26T00:20:32.353510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test', 'train']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os, glob, sys\n",
    "from os import path\n",
    "\n",
    "base_folder = r'specs/'\n",
    "os.listdir(base_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T00:20:33.830928Z",
     "start_time": "2020-08-26T00:20:33.824230Z"
    }
   },
   "outputs": [],
   "source": [
    "train_base_dir = base_folder+'train/'\n",
    "test_base_dir = base_folder+'test/'\n",
    "\n",
    "train_planets_dir = train_base_dir+'planets/'\n",
    "train_negatives_dir = train_base_dir+'negatives/'\n",
    "\n",
    "test_planets_dir = test_base_dir+'planets/'\n",
    "test_negatives_dir = test_base_dir+'negatives/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T00:20:37.578563Z",
     "start_time": "2020-08-26T00:20:37.567199Z"
    }
   },
   "outputs": [],
   "source": [
    "planet_train_files = glob.glob(train_planets_dir+'*.png')\n",
    "negative_train_files = glob.glob(train_negatives_dir+'*.png')\n",
    "all_train_files = [*planet_train_files, *negative_train_files]\n",
    "\n",
    "planet_test_files = glob.glob(test_planets_dir+'*.png')\n",
    "negative_test_files = glob.glob(test_negatives_dir+'*.png')\n",
    "all_test_files = [*planet_test_files, *negative_test_files]\n",
    "\n",
    "all_filename_vars = [planet_train_files, negative_train_files,\n",
    "                     planet_test_files, negative_test_files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:16:58.043305Z",
     "start_time": "2020-08-26T01:16:58.035253Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_img_cv2(filename, RGB=True):\n",
    "    import cv2\n",
    "    IMG = cv2.imread(filename)\n",
    "    \n",
    "    if RGB: \n",
    "        cmap = cv2.COLOR_BGR2RGB\n",
    "    \n",
    "    else:\n",
    "        cmap=cv2.COLOR_BGR2GRAY\n",
    "        \n",
    "    return cv2.cvtColor(IMG, cmap)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:42:42.102335Z",
     "start_time": "2020-08-26T01:42:42.092549Z"
    }
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from keras.preprocessing import image\n",
    "from imageio import imread\n",
    "from skimage.transform import resize\n",
    "from tqdm import tqdm\n",
    "\n",
    "# defining a function to read images and convert to array\n",
    "def read_img(img_path, target_size=(8,8)):\n",
    "    #img = load_img_cv2(img_path)\n",
    "    #img = image.img_to_array(img)\n",
    "    img = image.load_img(img_path, target_size=target_size, interpolation='bilinear')\n",
    "    img = image.img_to_array(img)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:42:44.240727Z",
     "start_time": "2020-08-26T01:42:44.228056Z"
    }
   },
   "outputs": [],
   "source": [
    "# Read in training and test filenames to produce X and y data splits.\n",
    "\n",
    "def xy_arrays(planet_train_filenames, negative_train_filenames, \n",
    "              planet_test_filenames, negative_test_filenames, \n",
    "              img_size=(8,8)):\n",
    "    # y labels are encoded as 0=negatives, 1=planets\n",
    "    # returns X_train, X_test, y_train, y_test, y_val\n",
    "    \n",
    "    import numpy as np\n",
    "    \n",
    "    display('[i] LOADING IMAGES')\n",
    "    X_img = []\n",
    "    y_label = []\n",
    "    \n",
    "    for img_path in tqdm(planet_train_files):\n",
    "        X_img.append(read_img(img_path, target_size=img_size))\n",
    "        y_label.append(1)\n",
    "        \n",
    "    for img_path in tqdm(negative_train_files):\n",
    "        X_img.append(read_img(img_path, target_size=img_size))\n",
    "        y_label.append(0)\n",
    "    \n",
    "    for img_path in tqdm(planet_test_files):\n",
    "        X_img.append(read_img(img_path, target_size=img_size))\n",
    "        y_label.append(1)\n",
    "        \n",
    "    for img_path in tqdm(negative_test_files):\n",
    "        X_img.append(read_img(img_path, target_size=img_size))\n",
    "        y_label.append(0)\n",
    "        \n",
    "    from sklearn.model_selection import train_test_split\n",
    "    X = np.array(X_img, np.float32)\n",
    "    y = np.array(y_label)\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:24.529979Z",
     "start_time": "2020-08-26T01:43:23.219553Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[i] LOADING IMAGES'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 37/37 [00:00<00:00, 55.01it/s]\n",
      "100%|██████████| 37/37 [00:00<00:00, 96.51it/s]\n",
      "100%|██████████| 5/5 [00:00<00:00, 54.55it/s]\n",
      "100%|██████████| 5/5 [00:00<00:00, 42.27it/s]\n"
     ]
    }
   ],
   "source": [
    "X, y = xy_arrays(planet_train_files, negative_train_files, \n",
    "              planet_test_files, negative_test_files, \n",
    "              img_size=(64,64,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:25.452360Z",
     "start_time": "2020-08-26T01:43:25.443229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((84, 64, 64, 3), (84,))"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:26.679487Z",
     "start_time": "2020-08-26T01:43:26.667173Z"
    }
   },
   "outputs": [],
   "source": [
    "# Authors: Yann N. Dauphin, Vlad Niculae, Gabriel Synnaeve\n",
    "# License: BSD\n",
    "\n",
    "# #############################################################################\n",
    "# Setting up\n",
    "\n",
    "def nudge_dataset(X, Y):\n",
    "    \"\"\"\n",
    "    This produces a dataset 5 times bigger than the original one,\n",
    "    by moving the 8x8 images in X around by 1px to left, right, down, up\n",
    "    \"\"\"\n",
    "    direction_vectors = [\n",
    "        [[0, 1, 0],\n",
    "         [0, 0, 0],\n",
    "         [0, 0, 0]],\n",
    "\n",
    "        [[0, 0, 0],\n",
    "         [1, 0, 0],\n",
    "         [0, 0, 0]],\n",
    "\n",
    "        [[0, 0, 0],\n",
    "         [0, 0, 1],\n",
    "         [0, 0, 0]],\n",
    "\n",
    "        [[0, 0, 0],\n",
    "         [0, 0, 0],\n",
    "         [0, 1, 0]]]\n",
    "\n",
    "    def shift(x, w):\n",
    "        return convolve(x.reshape((8, 8)), mode='constant', weights=w).ravel()\n",
    "\n",
    "    X = np.concatenate([X] +\n",
    "                       [np.apply_along_axis(shift, 1, X, vector)\n",
    "                        for vector in direction_vectors])\n",
    "    Y = np.concatenate([Y for _ in range(5)], axis=0)\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:30.484726Z",
     "start_time": "2020-08-26T01:43:27.760586Z"
    }
   },
   "outputs": [],
   "source": [
    "X, Y = nudge_dataset(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:44:36.409658Z",
     "start_time": "2020-08-26T01:44:36.400620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(420, 64, 64, 3)"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:30.532963Z",
     "start_time": "2020-08-26T01:43:30.505003Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1,\n",
       "       1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0,\n",
       "       0, 0])"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:32.338708Z",
     "start_time": "2020-08-26T01:43:32.259353Z"
    }
   },
   "outputs": [],
   "source": [
    "X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001)  # 0-1 scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:33.294011Z",
     "start_time": "2020-08-26T01:43:33.284307Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((420, 64, 64, 3), (84,))"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train,X_test,X_val,y_train,y_test,y_val=load_train_test_val(planet_train_files, \n",
    "#                                                               negative_train_files,\n",
    "#                                                               planet_test_files, \n",
    "#                                                               negative_test_files, \n",
    "#                                                               val_size=0.1, \n",
    "#                                                               img_size=(64,64,3))\n",
    "\n",
    "\n",
    "#     X_train,X_val, y_train, y_val = train_test_split(X,y,test_size=0.1)\n",
    "    \n",
    "#     print('\\n[i] Length of Splits:')\n",
    "#     print(f\"X_train={len(X_train)}, X_test={len(X_test)}, X_val={len(X_val)}\")\n",
    "    \n",
    "#     return X_train, X_test, X_val, y_train, y_test, y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:39.215920Z",
     "start_time": "2020-08-26T01:43:39.198212Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(\n",
    "    X, Y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:40.056805Z",
     "start_time": "2020-08-26T01:43:40.046668Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99215645"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[1][1][1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:41.791880Z",
     "start_time": "2020-08-26T01:43:41.785105Z"
    }
   },
   "outputs": [],
   "source": [
    "# Models we will use\n",
    "logistic = linear_model.LogisticRegression(solver='newton-cg', tol=1)\n",
    "rbm = BernoulliRBM(random_state=0, verbose=True)\n",
    "\n",
    "rbm_features_classifier = Pipeline(\n",
    "    steps=[('rbm', rbm), ('logistic', logistic)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:42.584085Z",
     "start_time": "2020-08-26T01:43:42.576895Z"
    }
   },
   "outputs": [],
   "source": [
    "# #############################################################################\n",
    "# Training\n",
    "\n",
    "# Hyper-parameters. These were set by cross-validation,\n",
    "# using a GridSearchCV. Here we are not performing cross-validation to\n",
    "# save time.\n",
    "rbm.learning_rate = 0.06\n",
    "rbm.n_iter = 10\n",
    "# More components tend to give better prediction performance, but larger\n",
    "# fitting time\n",
    "rbm.n_components = 100\n",
    "logistic.C = 6000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:43:51.486787Z",
     "start_time": "2020-08-26T01:43:51.375271Z"
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found array with dim 4. Estimator expected <= 2.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-169-dc5d0c13eb41>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Training RBM-Logistic Pipeline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrbm_features_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mThis\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    349\u001b[0m         \"\"\"\n\u001b[0;32m--> 350\u001b[0;31m         \u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m         with _print_elapsed_time('Pipeline',\n\u001b[1;32m    352\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    313\u001b[0m                 \u001b[0mmessage_clsname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Pipeline'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m                 \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m                 **fit_params_steps[name])\n\u001b[0m\u001b[1;32m    316\u001b[0m             \u001b[0;31m# Replace the transformer of the step with the fitted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m             \u001b[0;31m# transformer. This is necessary when loading the transformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/joblib/memory.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 355\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    356\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[0;34m(transformer, X, y, weight, message_clsname, message, **fit_params)\u001b[0m\n\u001b[1;32m    726\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0m_print_elapsed_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage_clsname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit_transform'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    729\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    572\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m             \u001b[0;31m# fit method of arity 2 (supervised transformation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 574\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    575\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/neural_network/_rbm.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    337\u001b[0m             \u001b[0mThe\u001b[0m \u001b[0mfitted\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m         \"\"\"\n\u001b[0;32m--> 339\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    340\u001b[0m         \u001b[0mn_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[0mrng\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_random_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    572\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mallow_nd\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m             raise ValueError(\"Found array with dim %d. %s expected <= 2.\"\n\u001b[0;32m--> 574\u001b[0;31m                              % (array.ndim, estimator_name))\n\u001b[0m\u001b[1;32m    575\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mforce_all_finite\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Found array with dim 4. Estimator expected <= 2."
     ]
    }
   ],
   "source": [
    "# Training RBM-Logistic Pipeline\n",
    "rbm_features_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T01:37:05.956711Z",
     "start_time": "2020-08-26T01:37:05.905283Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "ZeVDoXp0PDIs"
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found array with dim 4. Estimator expected <= 2.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-138-cab912481fe4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mraw_pixel_classifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogistic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mraw_pixel_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mC\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mraw_pixel_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m   1525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1526\u001b[0m         X, y = check_X_y(X, y, accept_sparse='csr', dtype=_dtype, order=\"C\",\n\u001b[0;32m-> 1527\u001b[0;31m                          accept_large_sparse=solver != 'liblinear')\n\u001b[0m\u001b[1;32m   1528\u001b[0m         \u001b[0mcheck_classification_targets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1529\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_X_y\u001b[0;34m(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    753\u001b[0m                     \u001b[0mensure_min_features\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensure_min_features\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m                     \u001b[0mwarn_on_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwarn_on_dtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 755\u001b[0;31m                     estimator=estimator)\n\u001b[0m\u001b[1;32m    756\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmulti_output\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    757\u001b[0m         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,\n",
      "\u001b[0;32m~/opt/anaconda3/envs/learn-env/lib/python3.6/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    572\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mallow_nd\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m             raise ValueError(\"Found array with dim %d. %s expected <= 2.\"\n\u001b[0;32m--> 574\u001b[0;31m                              % (array.ndim, estimator_name))\n\u001b[0m\u001b[1;32m    575\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mforce_all_finite\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Found array with dim 4. Estimator expected <= 2."
     ]
    }
   ],
   "source": [
    "# Training the Logistic regression classifier directly on the pixel\n",
    "raw_pixel_classifier = clone(logistic)\n",
    "raw_pixel_classifier.C = 100.\n",
    "raw_pixel_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #############################################################################\n",
    "# Evaluation\n",
    "\n",
    "Y_pred = rbm_features_classifier.predict(X_test)\n",
    "print(\"Logistic regression using RBM features:\\n%s\\n\" % (\n",
    "    metrics.classification_report(Y_test, Y_pred)))\n",
    "\n",
    "Y_pred = raw_pixel_classifier.predict(X_test)\n",
    "print(\"Logistic regression using raw pixel features:\\n%s\\n\" % (\n",
    "    metrics.classification_report(Y_test, Y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #############################################################################\n",
    "# Plotting\n",
    "\n",
    "plt.figure(figsize=(4.2, 4))\n",
    "for i, comp in enumerate(rbm.components_):\n",
    "    plt.subplot(10, 10, i + 1)\n",
    "    plt.imshow(comp.reshape((8, 8)), cmap=plt.cm.gray_r,\n",
    "               interpolation='nearest')\n",
    "    plt.xticks(())\n",
    "    plt.yticks(())\n",
    "plt.suptitle('100 components extracted by RBM', fontsize=16)\n",
    "plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "starskope-3.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}