Earth quake preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np \n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "import string\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc,accuracy_score,f1_score,precision_score,recall_score\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC,Ridge\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.metrics import precision_score,recall_score,auc\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading the data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"/Users/sarath/Documents/Final Project Data/train_values.csv\")\n",
    "train_labels = pd.read_csv(\"/Users/sarath/Documents/Final Project Data/train_labels.csv\")\n",
    "test_labels = pd.read_csv(\"/Users/sarath/Documents/Final Project Data/test_values.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['building_id', 'geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id',\n",
       "       'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
       "       'land_surface_condition', 'foundation_type', 'roof_type',\n",
       "       'ground_floor_type', 'other_floor_type', 'position',\n",
       "       'plan_configuration', 'has_superstructure_adobe_mud',\n",
       "       'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag',\n",
       "       'has_superstructure_cement_mortar_stone',\n",
       "       'has_superstructure_mud_mortar_brick',\n",
       "       'has_superstructure_cement_mortar_brick', 'has_superstructure_timber',\n",
       "       'has_superstructure_bamboo', 'has_superstructure_rc_non_engineered',\n",
       "       'has_superstructure_rc_engineered', 'has_superstructure_other',\n",
       "       'legal_ownership_status', 'count_families', 'has_secondary_use',\n",
       "       'has_secondary_use_agriculture', 'has_secondary_use_hotel',\n",
       "       'has_secondary_use_rental', 'has_secondary_use_institution',\n",
       "       'has_secondary_use_school', 'has_secondary_use_industry',\n",
       "       'has_secondary_use_health_post', 'has_secondary_use_gov_office',\n",
       "       'has_secondary_use_use_police', 'has_secondary_use_other'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260601, 39)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>geo_level_1_id</th>\n",
       "      <th>geo_level_2_id</th>\n",
       "      <th>geo_level_3_id</th>\n",
       "      <th>count_floors_pre_eq</th>\n",
       "      <th>age</th>\n",
       "      <th>area_percentage</th>\n",
       "      <th>height_percentage</th>\n",
       "      <th>land_surface_condition</th>\n",
       "      <th>foundation_type</th>\n",
       "      <th>...</th>\n",
       "      <th>has_secondary_use_agriculture</th>\n",
       "      <th>has_secondary_use_hotel</th>\n",
       "      <th>has_secondary_use_rental</th>\n",
       "      <th>has_secondary_use_institution</th>\n",
       "      <th>has_secondary_use_school</th>\n",
       "      <th>has_secondary_use_industry</th>\n",
       "      <th>has_secondary_use_health_post</th>\n",
       "      <th>has_secondary_use_gov_office</th>\n",
       "      <th>has_secondary_use_use_police</th>\n",
       "      <th>has_secondary_use_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>802906</td>\n",
       "      <td>6</td>\n",
       "      <td>487</td>\n",
       "      <td>12198</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>r</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   building_id  geo_level_1_id  geo_level_2_id  geo_level_3_id  \\\n",
       "0       802906               6             487           12198   \n",
       "\n",
       "   count_floors_pre_eq  age  area_percentage  height_percentage  \\\n",
       "0                    2   30                6                  5   \n",
       "\n",
       "  land_surface_condition foundation_type           ...             \\\n",
       "0                      t               r           ...              \n",
       "\n",
       "  has_secondary_use_agriculture has_secondary_use_hotel  \\\n",
       "0                             0                       0   \n",
       "\n",
       "  has_secondary_use_rental has_secondary_use_institution  \\\n",
       "0                        0                             0   \n",
       "\n",
       "  has_secondary_use_school  has_secondary_use_industry  \\\n",
       "0                        0                           0   \n",
       "\n",
       "   has_secondary_use_health_post  has_secondary_use_gov_office  \\\n",
       "0                              0                             0   \n",
       "\n",
       "   has_secondary_use_use_police  has_secondary_use_other  \n",
       "0                             0                        0  \n",
       "\n",
       "[1 rows x 39 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "building_id                               0\n",
       "geo_level_1_id                            0\n",
       "geo_level_2_id                            0\n",
       "geo_level_3_id                            0\n",
       "count_floors_pre_eq                       0\n",
       "age                                       0\n",
       "area_percentage                           0\n",
       "height_percentage                         0\n",
       "land_surface_condition                    0\n",
       "foundation_type                           0\n",
       "roof_type                                 0\n",
       "ground_floor_type                         0\n",
       "other_floor_type                          0\n",
       "position                                  0\n",
       "plan_configuration                        0\n",
       "has_superstructure_adobe_mud              0\n",
       "has_superstructure_mud_mortar_stone       0\n",
       "has_superstructure_stone_flag             0\n",
       "has_superstructure_cement_mortar_stone    0\n",
       "has_superstructure_mud_mortar_brick       0\n",
       "has_superstructure_cement_mortar_brick    0\n",
       "has_superstructure_timber                 0\n",
       "has_superstructure_bamboo                 0\n",
       "has_superstructure_rc_non_engineered      0\n",
       "has_superstructure_rc_engineered          0\n",
       "has_superstructure_other                  0\n",
       "legal_ownership_status                    0\n",
       "count_families                            0\n",
       "has_secondary_use                         0\n",
       "has_secondary_use_agriculture             0\n",
       "has_secondary_use_hotel                   0\n",
       "has_secondary_use_rental                  0\n",
       "has_secondary_use_institution             0\n",
       "has_secondary_use_school                  0\n",
       "has_secondary_use_industry                0\n",
       "has_secondary_use_health_post             0\n",
       "has_secondary_use_gov_office              0\n",
       "has_secondary_use_use_police              0\n",
       "has_secondary_use_other                   0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>damage_grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>802906</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   building_id  damage_grade\n",
       "0       802906             3"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels['building_id'].equals(train_data['building_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage1 = train_data.merge(train_labels, how = 'inner', on = 'building_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage1.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Unique Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_info(feature):\n",
    "    print('Unique values in the feature:',feature.unique())\n",
    "    print('No of Null values:',feature.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: ['r' 'w' 'i' 'u' 'h']\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "feature_info(building_damage1.foundation_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: [ 6  8 21 22 11  9 20  0 26 17 12 18  4 27 25 10  7  3 13 30 16  5 15 14\n",
      " 19  1 24 28 23  2 29]\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "feature_info(building_damage1.geo_level_1_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: [2 3 1 4 5 6 7 8 9]\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "feature_info(building_damage.count_floors_pre_eq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: [3 2 1]\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "feature_info(building_damage1.damage_grade)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: ['r' 'w' 'i' 'u' 'h']\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "feature_info(building_damage1.foundation_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#building_damage.isnull()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage1 = building_damage1.drop(columns =\"has_secondary_use\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage1 = building_damage1.drop(columns =\"has_secondary_use_agriculture\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "building_id                               0\n",
       "geo_level_1_id                            0\n",
       "geo_level_2_id                            0\n",
       "geo_level_3_id                            0\n",
       "count_floors_pre_eq                       0\n",
       "age                                       0\n",
       "area_percentage                           0\n",
       "height_percentage                         0\n",
       "land_surface_condition                    0\n",
       "foundation_type                           0\n",
       "roof_type                                 0\n",
       "ground_floor_type                         0\n",
       "other_floor_type                          0\n",
       "position                                  0\n",
       "plan_configuration                        0\n",
       "has_superstructure_adobe_mud              0\n",
       "has_superstructure_mud_mortar_stone       0\n",
       "has_superstructure_stone_flag             0\n",
       "has_superstructure_cement_mortar_stone    0\n",
       "has_superstructure_mud_mortar_brick       0\n",
       "has_superstructure_cement_mortar_brick    0\n",
       "has_superstructure_timber                 0\n",
       "has_superstructure_bamboo                 0\n",
       "has_superstructure_rc_non_engineered      0\n",
       "has_superstructure_rc_engineered          0\n",
       "has_superstructure_other                  0\n",
       "legal_ownership_status                    0\n",
       "count_families                            0\n",
       "has_secondary_use_hotel                   0\n",
       "has_secondary_use_rental                  0\n",
       "has_secondary_use_institution             0\n",
       "has_secondary_use_school                  0\n",
       "has_secondary_use_industry                0\n",
       "has_secondary_use_health_post             0\n",
       "has_secondary_use_gov_office              0\n",
       "has_secondary_use_use_police              0\n",
       "has_secondary_use_other                   0\n",
       "damage_grade                              0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "building_damage1.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage = building_damage1[building_damage1['age'] <= 250]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    147437\n",
       "3     86829\n",
       "1     24945\n",
       "Name: damage_grade, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "building_damage['damage_grade'].value_counts()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of final raw data matrix: (259211, 38)\n",
      "No of labels: 259211\n",
      "Unique values in the feature: [3 2 1]\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "print('Shape of final raw data matrix:',building_damage.shape)\n",
    "print('No of labels:',len(building_damage))\n",
    "feature_info(building_damage.damage_grade)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values in the feature: ['t' 'o' 'n']\n",
      "No of Null values: 0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "feature_info(building_damage.land_surface_condition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data : (259211, 38)\n",
      "Test data: (86868, 39)\n"
     ]
    }
   ],
   "source": [
    "print('Training data :',building_damage.shape)\n",
    "print('Test data:',test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data in Train dataset: 260601\n",
      "Data in Test dataset: 86868\n",
      "Data in CV dataset: 260601\n"
     ]
    }
   ],
   "source": [
    "print('Data in Train dataset:',len(train_data))\n",
    "print('Data in Test dataset:',len(test_labels))\n",
    "print('Data in CV dataset:',len(train_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### utility function to preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import Normalizer,LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "def preprocess_num_data(X_train,num_data,X_test):\n",
    "    std=Normalizer()\n",
    "    std.fit(train_data)\n",
    "    transformed_input=std.transform(building_damage)\n",
    "    transformed_test=std.transform(test_labels)\n",
    "    return transformed_input,transformed_test\n",
    "def preprocess_cat_data(building_damage,test_labels):\n",
    "    std=LabelEncoder()\n",
    "    for col in train_data.columns:\n",
    "        col_list_train=list(map(str,building_damage[col].values))\n",
    "        #print(col_list_train)\n",
    "        col_list_test=list(map(str,test_lables[col].values))\n",
    "        std.fit(col_list_train)\n",
    "        col_list_train=std.transform(col_list_train)\n",
    "        col_list_test=std.transform(col_list_test)\n",
    "        building_damage[col]=col_list_train\n",
    "        test_data[col]=col_list_test\n",
    "    return building_damage,test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_data=['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id',\n",
    "       'count_floors_pre_eq','area_percentage', 'age','height_percentage','count_families']\n",
    "cat_data=['land_surface_condition', 'foundation_type', 'roof_type',\n",
    "       'ground_floor_type', 'other_floor_type', 'position','legal_ownership_status',\n",
    "       'plan_configuration'] \n",
    "bin_data=['has_superstructure_adobe_mud',\n",
    "       'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag',\n",
    "       'has_superstructure_cement_mortar_stone',\n",
    "       'has_superstructure_mud_mortar_brick',\n",
    "       'has_superstructure_cement_mortar_brick', 'has_superstructure_timber',\n",
    "       'has_superstructure_bamboo', 'has_superstructure_rc_non_engineered',\n",
    "       'has_superstructure_rc_engineered', 'has_superstructure_other',\n",
    "         'has_secondary_use_hotel',\n",
    "       'has_secondary_use_rental', 'has_secondary_use_institution',\n",
    "       'has_secondary_use_school', 'has_secondary_use_industry',\n",
    "       'has_secondary_use_health_post', 'has_secondary_use_gov_office',\n",
    "       'has_secondary_use_use_police', 'has_secondary_use_other']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtering numerical data\n",
    "num_train=building_damage[num_data]\n",
    "num_test=test_labels[num_data]\n",
    "# filtering categorical data\n",
    "cat_train=building_damage[cat_data]\n",
    "cat_test=building_damage[cat_data]\n",
    "# filtering binary data\n",
    "bin_train=building_damage[bin_data]\n",
    "bin_test=test_labels[bin_data]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_feats = ['land_surface_condition', 'foundation_type', 'roof_type',\n",
    "       'ground_floor_type', 'other_floor_type', 'position','legal_ownership_status',\n",
    "       'plan_configuration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "building_damage['damage_grade'] = building_damage['damage_grade'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_final = pd.get_dummies(building_damage,columns=cat_feats,drop_first=True)\n",
    "test_final = pd.get_dummies(test_labels,columns=cat_feats,drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CategoricalDtype(categories=[1, 2, 3], ordered=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_final['damage_grade'].dtype\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['building_id', 'geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id',\n",
       "       'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
       "       'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone',\n",
       "       'has_superstructure_stone_flag',\n",
       "       'has_superstructure_cement_mortar_stone',\n",
       "       'has_superstructure_mud_mortar_brick',\n",
       "       'has_superstructure_cement_mortar_brick', 'has_superstructure_timber',\n",
       "       'has_superstructure_bamboo', 'has_superstructure_rc_non_engineered',\n",
       "       'has_superstructure_rc_engineered', 'has_superstructure_other',\n",
       "       'count_families', 'has_secondary_use_hotel', 'has_secondary_use_rental',\n",
       "       'has_secondary_use_institution', 'has_secondary_use_school',\n",
       "       'has_secondary_use_industry', 'has_secondary_use_health_post',\n",
       "       'has_secondary_use_gov_office', 'has_secondary_use_use_police',\n",
       "       'has_secondary_use_other', 'damage_grade', 'land_surface_condition_o',\n",
       "       'land_surface_condition_t', 'foundation_type_i', 'foundation_type_r',\n",
       "       'foundation_type_u', 'foundation_type_w', 'roof_type_q', 'roof_type_x',\n",
       "       'ground_floor_type_m', 'ground_floor_type_v', 'ground_floor_type_x',\n",
       "       'ground_floor_type_z', 'other_floor_type_q', 'other_floor_type_s',\n",
       "       'other_floor_type_x', 'position_o', 'position_s', 'position_t',\n",
       "       'legal_ownership_status_r', 'legal_ownership_status_v',\n",
       "       'legal_ownership_status_w', 'plan_configuration_c',\n",
       "       'plan_configuration_d', 'plan_configuration_f', 'plan_configuration_m',\n",
       "       'plan_configuration_n', 'plan_configuration_o', 'plan_configuration_q',\n",
       "       'plan_configuration_s', 'plan_configuration_u'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_final.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 86868 entries, 0 to 86867\n",
      "Data columns (total 61 columns):\n",
      "building_id                               86868 non-null int64\n",
      "geo_level_1_id                            86868 non-null int64\n",
      "geo_level_2_id                            86868 non-null int64\n",
      "geo_level_3_id                            86868 non-null int64\n",
      "count_floors_pre_eq                       86868 non-null int64\n",
      "age                                       86868 non-null int64\n",
      "area_percentage                           86868 non-null int64\n",
      "height_percentage                         86868 non-null int64\n",
      "has_superstructure_adobe_mud              86868 non-null int64\n",
      "has_superstructure_mud_mortar_stone       86868 non-null int64\n",
      "has_superstructure_stone_flag             86868 non-null int64\n",
      "has_superstructure_cement_mortar_stone    86868 non-null int64\n",
      "has_superstructure_mud_mortar_brick       86868 non-null int64\n",
      "has_superstructure_cement_mortar_brick    86868 non-null int64\n",
      "has_superstructure_timber                 86868 non-null int64\n",
      "has_superstructure_bamboo                 86868 non-null int64\n",
      "has_superstructure_rc_non_engineered      86868 non-null int64\n",
      "has_superstructure_rc_engineered          86868 non-null int64\n",
      "has_superstructure_other                  86868 non-null int64\n",
      "count_families                            86868 non-null int64\n",
      "has_secondary_use                         86868 non-null int64\n",
      "has_secondary_use_agriculture             86868 non-null int64\n",
      "has_secondary_use_hotel                   86868 non-null int64\n",
      "has_secondary_use_rental                  86868 non-null int64\n",
      "has_secondary_use_institution             86868 non-null int64\n",
      "has_secondary_use_school                  86868 non-null int64\n",
      "has_secondary_use_industry                86868 non-null int64\n",
      "has_secondary_use_health_post             86868 non-null int64\n",
      "has_secondary_use_gov_office              86868 non-null int64\n",
      "has_secondary_use_use_police              86868 non-null int64\n",
      "has_secondary_use_other                   86868 non-null int64\n",
      "land_surface_condition_o                  86868 non-null uint8\n",
      "land_surface_condition_t                  86868 non-null uint8\n",
      "foundation_type_i                         86868 non-null uint8\n",
      "foundation_type_r                         86868 non-null uint8\n",
      "foundation_type_u                         86868 non-null uint8\n",
      "foundation_type_w                         86868 non-null uint8\n",
      "roof_type_q                               86868 non-null uint8\n",
      "roof_type_x                               86868 non-null uint8\n",
      "ground_floor_type_m                       86868 non-null uint8\n",
      "ground_floor_type_v                       86868 non-null uint8\n",
      "ground_floor_type_x                       86868 non-null uint8\n",
      "ground_floor_type_z                       86868 non-null uint8\n",
      "other_floor_type_q                        86868 non-null uint8\n",
      "other_floor_type_s                        86868 non-null uint8\n",
      "other_floor_type_x                        86868 non-null uint8\n",
      "position_o                                86868 non-null uint8\n",
      "position_s                                86868 non-null uint8\n",
      "position_t                                86868 non-null uint8\n",
      "legal_ownership_status_r                  86868 non-null uint8\n",
      "legal_ownership_status_v                  86868 non-null uint8\n",
      "legal_ownership_status_w                  86868 non-null uint8\n",
      "plan_configuration_c                      86868 non-null uint8\n",
      "plan_configuration_d                      86868 non-null uint8\n",
      "plan_configuration_f                      86868 non-null uint8\n",
      "plan_configuration_m                      86868 non-null uint8\n",
      "plan_configuration_n                      86868 non-null uint8\n",
      "plan_configuration_o                      86868 non-null uint8\n",
      "plan_configuration_q                      86868 non-null uint8\n",
      "plan_configuration_s                      86868 non-null uint8\n",
      "plan_configuration_u                      86868 non-null uint8\n",
      "dtypes: int64(31), uint8(30)\n",
      "memory usage: 23.0 MB\n"
     ]
    }
   ],
   "source": [
    "test_final.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=train_final.damage_grade\n",
    "train=train_final.drop('damage_grade',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(259211, 60)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = train_final.drop('damage_grade',axis=1)\n",
    "y = train_final['damage_grade']\n",
    "\n",
    "train_final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature Scaling\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import linear_model\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import pickle\n",
    "from sklearn.externals import joblib \n",
    "import random\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=[1, 2,3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {'C':[0.01, 0.1, 1,5, 10], 'penalty':['l2']}\n",
    "log_reg = linear_model.LogisticRegression(class_weight='none')\n",
    "log_reg_grid = GridSearchCV(log_reg, param_grid=parameters, cv=3, verbose=0, n_jobs='none')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "logmodel = LogisticRegression(class_weight=None)\n",
    "p = logmodel.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predictions and Evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = logmodel.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### * Create a classification report for the model.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2703"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(predictions == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.60      0.26      0.36      6252\n",
      "           2       0.60      0.89      0.72     36960\n",
      "           3       0.55      0.19      0.28     21591\n",
      "\n",
      "   micro avg       0.59      0.59      0.59     64803\n",
      "   macro avg       0.58      0.44      0.45     64803\n",
      "weighted avg       0.58      0.59      0.54     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72571.5"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum (predictions)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1625  4446   181]\n",
      " [ 1012 32799  3149]\n",
      " [   66 17522  4003]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1032f99b0>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB29JREFUeJzt28FrnHUex/HPZ5uYi17KzEFqs9oiguxFGLwIexAWul7coz14EnoSFPZQ/wpveylYlgVRFvTgQZA9CGFBpLF4sAaXIl0MBjJDDrqnpPDdQwPbbgPzpJ3fPDPzeb8gkEkfnn540jfPTCZ1VQlAlt/0PQDA/BE+EIjwgUCEDwQifCAQ4QOBVj5825ds/2D7tu33+t6zqGxft71v+7u+tywy2+dtf2l7x/Yt2+/0velReJXfx7d9RtK/JP1B0q6kG5IuV9X3vQ5bQLZ/L+k/kv5WVb/re8+isv20pKer6qbtpyR9I+lPy/ZvatXv+C9Lul1VP1bVoaSPJb3e86aFVFVbkg763rHoqmqvqm4ef/6rpB1J5/pddXqrHv45ST/d93hXS/hNwmKy/ayklyR93e+S01v18H3C11b3tQ3mxvaTkj6R9G5V/dL3ntNa9fB3JZ2/7/Ezkn7uaQtWhO113Yv+w6r6tO89j2LVw78h6Xnbz9l+QtIbkj7reROWmG1L+kDSTlW93/eeR7XS4VfVXUlvS/pC934I8/equtXvqsVk+yNJX0l6wfau7bf63rSgXpH0pqRXbX97/PFa36NOa6XfzgNwspW+4wM4GeEDgQgfCET4QCDCBwLFhG/7St8blgHXqbtlvlYx4Uta2m/SnHGdulvaa5UUPoBjTX6BZzAY1Obm5szP+zgmk4kGg0HfMx5wdHTU94SHHBwc6OzZs33PeMjGxkbfEx4yHo81HA77nvGAO3fuaDKZnPSf0x6w1uIv39zc1NbWVotTr5T9/f2+JyyNCxcu9D1hKYxGo07H8VQfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgUKfwbV+y/YPt27bfaz0KQFtTw7d9RtJfJP1R0ouSLtt+sfUwAO10ueO/LOl2Vf1YVYeSPpb0ettZAFrqEv45ST/d93j3+GsAllSX8H3C1+qhg+wrtrdtb08mk8dfBqCZLuHvSjp/3+NnJP38/wdV1bWqGlXVaDAYzGofgAa6hH9D0vO2n7P9hKQ3JH3WdhaAltamHVBVd22/LekLSWckXa+qW82XAWhmaviSVFWfS/q88RYAc8Jv7gGBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCDQWouT2tb6+nqLU6+Uixcv9j1haezt7fU9YSkcHR11Oo47PhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhBoavi2r9vet/3dPAYBaK/LHf+vki413gFgjqaGX1Vbkg7msAXAnPAaHwg0s/BtX7G9bXt7PB7P6rQAGphZ+FV1rapGVTUaDoezOi2ABniqDwTq8nbeR5K+kvSC7V3bb7WfBaCltWkHVNXleQwBMD881QcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwi01uKktrWxsdHi1Cvl6tWrfU9YGoeHh31PWApV1ek47vhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAoKnh2z5v+0vbO7Zv2X5nHsMAtLPW4Zi7kv5cVTdtPyXpG9v/qKrvG28D0MjUO35V7VXVzePPf5W0I+lc62EA2jnVa3zbz0p6SdLXLcYAmI/O4dt+UtInkt6tql9O+PMrtrdtb4/H41luBDBjncK3va570X9YVZ+edExVXauqUVWNhsPhLDcCmLEuP9W3pA8k7VTV++0nAWityx3/FUlvSnrV9rfHH6813gWgoalv51XVPyV5DlsAzAm/uQcEInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFArqrZn9QeS/r3zE/8eAaSJn2PWAJcp+4W8Vr9tqqG0w5qEv4isr1dVaO+dyw6rlN3y3yteKoPBCJ8IFBS+Nf6HrAkuE7dLe21inmND+B/ku74AI4RPhCI8IFAhA8EInwg0H8BPdJFEkWzOLAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = metrics.confusion_matrix(y_test, predictions)\n",
    "print(cm)\n",
    "plt.matshow(cm, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.5929818063978519\n",
      "Kappa:     0.14538424478346168\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,predictions))\n",
    "print('Kappa:    ', cohen_kappa_score(predictions,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gridsearch CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-6c22016fa929>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0my_train\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sc' is not defined"
     ]
    }
   ],
   "source": [
    "y_train  = list(y_train)\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "grid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"]}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03]), 'penalty': ['l1', 'l2']},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg=LogisticRegression()\n",
    "logreg_cv=GridSearchCV(logreg,grid,cv=10)\n",
    "logreg_cv.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tuned hpyerparameters :(best parameters)  {'C': 0.1, 'penalty': 'l1'}\n",
      "accuracy : 0.5921361260853463\n"
     ]
    }
   ],
   "source": [
    "print(\"tuned hpyerparameters :(best parameters) \",logreg_cv.best_params_)\n",
    "print(\"accuracy :\",logreg_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.592950943629153\n"
     ]
    }
   ],
   "source": [
    "logreg2=LogisticRegression(C=0.1,penalty=\"l1\")\n",
    "logreg2.fit(X_train,y_train)\n",
    "print(\"score\",logreg2.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = logreg2.fit(X_train,y_train).predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 1, ..., 2, 2, 2])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.60      0.26      0.36      6252\n",
      "           2       0.60      0.89      0.72     36960\n",
      "           3       0.55      0.18      0.28     21591\n",
      "\n",
      "   micro avg       0.59      0.59      0.59     64803\n",
      "   macro avg       0.58      0.44      0.45     64803\n",
      "weighted avg       0.58      0.59      0.53     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1616  4456   180]\n",
      " [ 1012 32822  3126]\n",
      " [   64 17540  3987]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot confusin matrix\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.grid(b=False)\n",
    "#plot_confusion_matrix(cm, normalize=True, title='Normalized confusion matrix')\n",
    "#plt.show()\n",
    "\n",
    "plot_confusion_matrix(cm,classes= labels normalize=True,\n",
    "                      title='Normalized confusion matrix')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.592950943629153\n",
      "Kappa:     0.14491218912090742\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,predictions))\n",
    "print('Kappa:    ', cohen_kappa_score(predictions,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a KNN model instance with n_neighbors=5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit this KNN model to the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=5, weights='uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train  = list(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions and Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the predict method to predict values using your KNN model and X_test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = knn.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(pred == 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2213  3723   316]\n",
      " [ 1456 29880  5624]\n",
      " [  119 12407  9065]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.58      0.35      0.44      6252\n",
      "           2       0.65      0.81      0.72     36960\n",
      "           3       0.60      0.42      0.50     21591\n",
      "\n",
      "   micro avg       0.64      0.64      0.64     64803\n",
      "   macro avg       0.61      0.53      0.55     64803\n",
      "weighted avg       0.63      0.64      0.62     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.6351249170563091\n",
      "Kappa:     0.28772972087265325\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,pred))\n",
    "print('Kappa:    ', cohen_kappa_score(pred,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Choosing a K Value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's go ahead and use the elbow method to pick a good K Value!\n",
    "\n",
    "#### ** Create a for loop that trains various KNN models with different k values, then keep track of the error_rate for each of these models with a list. Refer to the lecture if you are confused on this step.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_rate = []\n",
    "\n",
    "for i in range(1,40):\n",
    "    \n",
    "    knn = KNeighborsClassifier(n_neighbors=i)\n",
    "    knn.fit(X_train,y_train)\n",
    "    pred_i = knn.predict(X_test)\n",
    "    error_rate.append(np.mean(pred_i != y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now create the following plot using the information from your for loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',\n",
    "         markerfacecolor='red', markersize=10)\n",
    "plt.title('Error Rate vs. K Value')\n",
    "plt.xlabel('K')\n",
    "plt.ylabel('Error Rate')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Retrain with new K Value\n",
    "Retrain your model with the best K value (up to you to decide what you want) and re-do the classification report and the confusion matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WITH K=30\n",
      "\n",
      "\n",
      "[[ 2213  3723   316]\n",
      " [ 1456 29880  5624]\n",
      " [  119 12407  9065]]\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'classification_report' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-13d7df97ac51>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfusion_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'classification_report' is not defined"
     ]
    }
   ],
   "source": [
    "# NOW WITH K=30\n",
    "knn = KNeighborsClassifier(n_neighbors=30, weights='uniform')\n",
    "\n",
    "knn.fit(X_train,y_train)\n",
    "pred = knn.predict(X_test)\n",
    "\n",
    "print('WITH K=30')\n",
    "print('\\n')\n",
    "print(confusion_matrix(y_test,pred))\n",
    "print('\\n')\n",
    "print(classification_report(y_test,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,pred))\n",
    "print('Kappa:    ', cohen_kappa_score(pred,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### kNN in a GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=5, weights='uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train  = list(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_params = {\n",
    "    'n_neighbors':[3,5,11,19],\n",
    "    'weights':['uniform', 'distance'],\n",
    "    'metric': ['manhattan']\n",
    "}\n",
    "\n",
    "gs=       GridSearchCV(\n",
    "          KNeighborsClassifier(),\n",
    "          grid_params,\n",
    "          verbose =1,\n",
    "           cv=3,\n",
    "           n_jobs = -1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.3891018 , -0.98317386, -1.24320225, ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599],\n",
       "       [-1.62043234, -0.48496037,  0.0839259 , ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599],\n",
       "       [ 0.66942421, -0.11130026, -0.95743378, ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599],\n",
       "       ...,\n",
       "       [-1.21444032,  0.88512671, -0.54088991, ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599],\n",
       "       [ 0.41961428, -0.360407  , -1.37882118, ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599],\n",
       "       [ 1.662828  , -0.360407  , -1.30616818, ..., -0.14888611,\n",
       "        -0.0361696 , -0.11927599]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:    2.0s finished\n"
     ]
    }
   ],
   "source": [
    "gs_results = gs.fit(X_train[1:1000,],  y_train[1:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5675675675675675"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_results.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='manhattan',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=11, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_results.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'n_neighbors': [3, 5, 11, 19], 'weights': ['uniform', 'distance'], 'metric': ['manhattan']},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=1)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = gs_results.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.45      0.22      0.30      6252\n",
      "           2       0.59      0.81      0.69     36960\n",
      "           3       0.48      0.24      0.32     21591\n",
      "\n",
      "   micro avg       0.57      0.57      0.57     64803\n",
      "   macro avg       0.51      0.43      0.44     64803\n",
      "weighted avg       0.54      0.57      0.53     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1398  4521   333]\n",
      " [ 1455 30047  5458]\n",
      " [  231 16114  5246]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.5661929231671373\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'cohen_kappa_score' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-76-9d14b7745c78>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Accuracy: '\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Kappa:    '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcohen_kappa_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'cohen_kappa_score' is not defined"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,predictions))\n",
    "print('Kappa:    ', cohen_kappa_score(predictions,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a Decision Tree Model\n",
    "Let's start by training a single decision tree first!\n",
    "\n",
    "** Import DecisionTreeClassifier**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create an instance of DecisionTreeClassifier() called dtree and fit it to the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtree = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train  = list(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtree.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions and Evaluation of Decision Tree\n",
    "Create predictions from the test set and create a classification report and a confusion matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = dtree.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.48      0.49      0.48      6252\n",
      "           2       0.70      0.69      0.70     36960\n",
      "           3       0.59      0.61      0.60     21591\n",
      "\n",
      "   micro avg       0.64      0.64      0.64     64803\n",
      "   macro avg       0.59      0.60      0.59     64803\n",
      "weighted avg       0.65      0.64      0.64     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3056  2728   468]\n",
      " [ 2911 25525  8524]\n",
      " [  433  8008 13150]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3056  2728   468]\n",
      " [ 2911 25525  8524]\n",
      " [  433  8008 13150]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1a19da5f28>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB2hJREFUeJzt28FrnHUex/HPZ5N40kuyoQ11WD2IIHsRBi/CHoSFrhf3aA+ehJ4Ehb34V3jbS8GyLIiyEA85WMSDIAsiHYsHa3Ap0sWgYDqh6NKDFL57aGDrNjBP6vzmmWc+7xcEMunD0w9P+uaZyaSuKgHI8pu+BwBYPMIHAhE+EIjwgUCEDwQifCDQyodv+7ztr23fsP1m33uWle3Ltn+w/WXfW5aZ7ZHtj23v275u+/W+Nz0Mr/L7+LbXJP1L0h8lHUi6KulCVX3V67AlZPsPkv4j6e9V9fu+9ywr2zuSdqrqmu3HJH0u6c9D+ze16nf85yTdqKpvqupnSe9JeqnnTUupqj6RdNT3jmVXVd9X1bXjz3+StC/pXL+rTm/Vwz8n6dv7Hh9ogN8kLCfbT0h6VtJn/S45vVUP3yd8bXVf22BhbD8qaVfSG1X1Y997TmvVwz+QNLrv8eOSvutpC1aE7Q3di/6dqnq/7z0PY9XDvyrpKdtP2n5E0suS9nrehAGzbUlvS9qvqrf63vOwVjr8qror6TVJH+reD2H+UVXX+121nGy/K+lTSU/bPrD9at+bltTzkl6R9ILtL44/Xux71Gmt9Nt5AE620nd8ACcjfCAQ4QOBCB8IRPhAoJjwbV/se8MQcJ26G/K1iglf0mC/SQvGdepusNcqKXwAx5r8As/m5maNRqPZBy7QdDrV1tZW3zOW3tHRkTY3N/ue8YCNjY2+Jzzg8PBQ29vbfc/4hZs3b+rWrVsn/ee0X1hv8ZePRiNduXKlxalXytraWt8TBuPMmTN9TxiE8Xjc6Tie6gOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwTqFL7t87a/tn3D9putRwFoa2b4ttck/VXSnyQ9I+mC7WdaDwPQTpc7/nOSblTVN1X1s6T3JL3UdhaAlrqEf07St/c9Pjj+GoCB6hK+T/haPXCQfdH2xPZkOp3++mUAmukS/oGk0X2PH5f03f8fVFWXqmpcVeOtra157QPQQJfwr0p6yvaTth+R9LKkvbazALS0PuuAqrpr+zVJH0pak3S5qq43XwagmZnhS1JVfSDpg8ZbACwIv7kHBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQOutTmy71alXxtmzZ/ueMBi7u7t9TxiE27dvdzqOOz4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QaGb4ti/b/sH2l4sYBKC9Lnf8v0k633gHgAWaGX5VfSLpaAFbACwIr/GBQHML3/ZF2xPbk+l0Oq/TAmhgbuFX1aWqGlfVeGtra16nBdAAT/WBQF3ezntX0qeSnrZ9YPvV9rMAtLQ+64CqurCIIQAWh6f6QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgdZbnHRjY0M7OzstTr1S9vb2+p4wGJPJpO8Jg3Dnzp1Ox3HHBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwLNDN/2yPbHtvdtX7f9+iKGAWhnvcMxdyX9paqu2X5M0ue2P6qqrxpvA9DIzDt+VX1fVdeOP/9J0r6kc62HAWjnVK/xbT8h6VlJn7UYA2AxOodv+1FJu5LeqKofT/jzi7YntieHh4fz3AhgzjqFb3tD96J/p6reP+mYqrpUVeOqGm9vb89zI4A56/JTfUt6W9J+Vb3VfhKA1rrc8Z+X9IqkF2x/cfzxYuNdABqa+XZeVf1TkhewBcCC8Jt7QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCET4QCDCBwIRPhCI8IFAhA8EInwgEOEDgQgfCOSqmv9J7UNJ/577iX+d30q61feIAeA6dbeM1+p3VbU966Am4S8j25OqGve9Y9lxnbob8rXiqT4QiPCBQEnhX+p7wEBwnbob7LWKeY0P4H+S7vgAjhE+EIjwgUCEDwQifCDQfwEUbT5n12QMJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = metrics.confusion_matrix(y_test, predictions)\n",
    "print(cm)\n",
    "plt.matshow(cm, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.6439671002885669\n",
      "Kappa:     0.3613650440364855\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,predictions))\n",
    "print('Kappa:    ', cohen_kappa_score(predictions,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the Random Forest model\n",
    "Create an instance of the RandomForestClassifier class and fit it to our training data from the previous step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF = RandomForestClassifier()\n",
    "def RF_param_selection(X, y, nfolds):\n",
    "   trees = [50,55,60,65,70,75,80,85,90,95,100]\n",
    "   param_grid = {'n_estimators': trees}\n",
    "   grid_search = GridSearchCV(RF, param_grid, cv=nfolds)\n",
    "   grid_search.fit(X, y)\n",
    "   grid_search.best_params_\n",
    "   return grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 95}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RF_param_selection(X_train,y_train, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=95, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 1, ..., 2, 3, 2])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predictions and Evaluation\n",
    "Let's predict off the y_test values and evaluate our model.\n",
    "\n",
    "** Predict the class of not.fully.paid for the X_test data.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now create a classification report from the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.67      0.45      0.54      6252\n",
      "           2       0.72      0.84      0.77     36960\n",
      "           3       0.73      0.58      0.65     21591\n",
      "\n",
      "   micro avg       0.72      0.72      0.72     64803\n",
      "   macro avg       0.70      0.62      0.65     64803\n",
      "weighted avg       0.72      0.72      0.71     64803\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Show the Confusion Matrix for the predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2787  3343   122]\n",
      " [ 1247 31102  4611]\n",
      " [  125  8927 12539]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2787  3343   122]\n",
      " [ 1247 31102  4611]\n",
      " [  125  8927 12539]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1a19a97d68>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB2xJREFUeJzt28FrnHUex/HPJ02EFnOa5mDroB5EkL0IgxdhD8JC14se7cGT0JOgsBf/Cm97KViWBVEW9ODBYvcQkAWRxuLBGixFuhgUTOIh7qUS+O6hgW3XwDyp85tnnvm8XxDIpA9PPzzpmyeTmbqqBCDLSt8DAMwf4QOBCB8IRPhAIMIHAhE+EGjpw7d9wfa3tm/bfrvvPYvK9hXbP9n+uu8ti8z22Pam7W3bN22/2femh+Flfh3f9ilJtyT9SdKOpOuSLlbVN70OW0C2/yjpP5L+XlV/6HvPorL9mKTHquqG7XVJX0p6ZWj/ppb9jv+8pNtV9V1V/SrpA0kv97xpIVXVZ5J+7nvHoquqH6vqxtHnv0jalnS+31Unt+zhn5f0/X2PdzTAbxIWk+0nJT0n6Yt+l5zcsofvY762vM9tMDe2H5X0oaS3quqg7z0ntezh70ga3/f4cUk/9LQFS8L2mu5F/15VfdT3noex7OFfl/S07adsPyLpVUkf97wJA2bbkt6VtF1V7/S952EtdfhVdSjpDUmf6t4vYf5RVTf7XbWYbL8v6XNJz9jesf1635sW1AuSXpP0ou2vjj5e6nvUSS31y3kAjrfUd3wAxyN8IBDhA4EIHwhE+ECgmPBtX+p7wxBwnbob8rWKCV/SYL9Jc8Z16m6w1yopfABHmryBZzQa1Xg8nn7gHO3v72s0GvU94wGL+OapRbxOkrS2ttb3hN/Y3d3VxsZG3zMecOfOHe3t7R33n9MesNriLx+Px7p27VqLUy+Vw8PDvicMxrlz5/qeMAiTyaTTcfyoDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EKhT+LYv2P7W9m3bb7ceBaCtqeHbPiXpr5L+LOlZSRdtP9t6GIB2utzxn5d0u6q+q6pfJX0g6eW2swC01CX885K+v+/xztHXAAxUl/B9zNfqNwfZl2xv2d7a39///csANNMl/B1J4/sePy7ph/8/qKouV9Wkqiaj0WhW+wA00CX865Ketv2U7UckvSrp47azALS0Ou2Aqjq0/YakTyWdknSlqm42XwagmanhS1JVfSLpk8ZbAMwJ79wDAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAoNUWJ11ZWdH6+nqLUy+VM2fO9D1hMG7dutX3hEG4e/dup+O44wOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBCB8IRPhAIMIHAhE+EIjwgUCEDwQifCAQ4QOBpoZv+4rtn2x/PY9BANrrcsf/m6QLjXcAmKOp4VfVZ5J+nsMWAHPCc3wg0MzCt33J9pbtrb29vVmdFkADMwu/qi5X1aSqJmfPnp3VaQE0wI/6QKAuL+e9L+lzSc/Y3rH9evtZAFpanXZAVV2cxxAA88OP+kAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4FWW5x0ZWVFp0+fbnHqpXL16tW+JwzG5uZm3xMG4eDgoNNx3PGBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQFPDtz22vWl72/ZN22/OYxiAdlY7HHMo6S9VdcP2uqQvbf+zqr5pvA1AI1Pv+FX1Y1XdOPr8F0nbks63HgagnRM9x7f9pKTnJH3RYgyA+egcvu1HJX0o6a2qOjjmzy/Z3rK9tbu7O8uNAGasU/i213Qv+veq6qPjjqmqy1U1qarJxsbGLDcCmLEuv9W3pHclbVfVO+0nAWityx3/BUmvSXrR9ldHHy813gWgoakv51XVvyR5DlsAzAnv3AMCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgwgcCET4QiPCBQIQPBCJ8IBDhA4EIHwhE+EAgV9XsT2rvSvr3zE/8+5yVtNf3iAHgOnW3iNfqiaramHZQk/AXke2tqpr0vWPRcZ26G/K14kd9IBDhA4GSwr/c94CB4Dp1N9hrFfMcH8D/JN3xARwhfCAQ4QOBCB8IRPhAoP8CW+VEqUNelOEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = metrics.confusion_matrix(y_test, predictions)\n",
    "print(cm)\n",
    "plt.matshow(cm, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7164483125781214\n",
      "Kappa:     0.45812645405986574\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',accuracy_score(y_test,predictions))\n",
    "print('Kappa:    ', cohen_kappa_score(predictions,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_values = pd.read_csv(DATA_DIR / 'test_values.csv', index_col='building_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train  = list(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "classifier = SVC(kernel = 'linear', random_state = 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " #Predicting the Test set results\n",
    "y_pred = classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy=accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = svc_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions1= clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_test,predictions1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'predictions1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-62-143481bfc343>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpredictions1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'predictions1' is not defined"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predictions1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gridsearch Practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'C': [0.1,1, 10, 100],'kernel': ['linear'], 'gamma': [1,0.1,0.01,0.001]} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = GridSearchCV(SVC(),param_grid,refit=True,verbose=2)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Applying Grid Search to find the best model and the best parameters\n",
    "grid_search = GridSearchCV(estimator = classifier,\n",
    "                           param_grid = parameters,\n",
    "                           scoring = 'accuracy',\n",
    "                           cv = 10,\n",
    "                           n_jobs = -1)\n",
    "grid_search = grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = grid_search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = SVC(kernel = 'rbf', gamma=0.7)\n",
    "classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting the Test set results\n",
    "y_pred = classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy=accuracy_score(y_test,y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}