Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning\n",
    "\n",
    "Now that the EDA and the preprocessing steps are done in R, we can move on to building the proper ML model. We will be trying out three different algorithms, decision tree, random forest and XGBoost. We will evaluate their performance and decide which model performs best for the particular dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>...</th>\n",
       "      <th>OccupationEntrepreneur</th>\n",
       "      <th>OccupationJournalist</th>\n",
       "      <th>OccupationLawyer</th>\n",
       "      <th>OccupationManager</th>\n",
       "      <th>OccupationMechanic</th>\n",
       "      <th>OccupationMedia_Manager</th>\n",
       "      <th>OccupationMusician</th>\n",
       "      <th>OccupationScientist</th>\n",
       "      <th>OccupationTeacher</th>\n",
       "      <th>OccupationWriter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>809.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>809.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>809.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>809.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>809.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>502.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>502.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>502.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>25</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>502.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>502.38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  Interest_Rate  \\\n",
       "0       23       19114.12                  3                4              3   \n",
       "1       23       19114.12                  3                4              3   \n",
       "2       23       19114.12                  3                4              3   \n",
       "3       23       19114.12                  3                4              3   \n",
       "4       23       19114.12                  3                4              3   \n",
       "...    ...            ...                ...              ...            ...   \n",
       "99995   25       39628.99                  4                6              7   \n",
       "99996   25       39628.99                  4                6              7   \n",
       "99997   25       39628.99                  4                6           5729   \n",
       "99998   25       39628.99                  4                6              7   \n",
       "99999   25       39628.99                  4                6              7   \n",
       "\n",
       "       Num_of_Loan  Delay_from_due_date  Num_Credit_Inquiries  Credit_Mix  \\\n",
       "0                4                    3                   4.0           2   \n",
       "1                4                   -1                   4.0           2   \n",
       "2                4                    3                   4.0           2   \n",
       "3                4                    5                   4.0           2   \n",
       "4                4                    6                   4.0           2   \n",
       "...            ...                  ...                   ...         ...   \n",
       "99995            2                   23                   3.0           2   \n",
       "99996            2                   18                   3.0           2   \n",
       "99997            2                   27                   3.0           2   \n",
       "99998            2                   20                   3.0           2   \n",
       "99999            2                   18                   3.0           2   \n",
       "\n",
       "       Outstanding_Debt  ...  OccupationEntrepreneur  OccupationJournalist  \\\n",
       "0                809.98  ...                       0                     0   \n",
       "1                809.98  ...                       0                     0   \n",
       "2                809.98  ...                       0                     0   \n",
       "3                809.98  ...                       0                     0   \n",
       "4                809.98  ...                       0                     0   \n",
       "...                 ...  ...                     ...                   ...   \n",
       "99995            502.38  ...                       0                     0   \n",
       "99996            502.38  ...                       0                     0   \n",
       "99997            502.38  ...                       0                     0   \n",
       "99998            502.38  ...                       0                     0   \n",
       "99999            502.38  ...                       0                     0   \n",
       "\n",
       "       OccupationLawyer  OccupationManager  OccupationMechanic  \\\n",
       "0                     0                  0                   0   \n",
       "1                     0                  0                   0   \n",
       "2                     0                  0                   0   \n",
       "3                     0                  0                   0   \n",
       "4                     0                  0                   0   \n",
       "...                 ...                ...                 ...   \n",
       "99995                 0                  0                   1   \n",
       "99996                 0                  0                   1   \n",
       "99997                 0                  0                   1   \n",
       "99998                 0                  0                   1   \n",
       "99999                 0                  0                   1   \n",
       "\n",
       "       OccupationMedia_Manager  OccupationMusician  OccupationScientist  \\\n",
       "0                            0                   0                    1   \n",
       "1                            0                   0                    1   \n",
       "2                            0                   0                    1   \n",
       "3                            0                   0                    1   \n",
       "4                            0                   0                    1   \n",
       "...                        ...                 ...                  ...   \n",
       "99995                        0                   0                    0   \n",
       "99996                        0                   0                    0   \n",
       "99997                        0                   0                    0   \n",
       "99998                        0                   0                    0   \n",
       "99999                        0                   0                    0   \n",
       "\n",
       "       OccupationTeacher  OccupationWriter  \n",
       "0                      0                 0  \n",
       "1                      0                 0  \n",
       "2                      0                 0  \n",
       "3                      0                 0  \n",
       "4                      0                 0  \n",
       "...                  ...               ...  \n",
       "99995                  0                 0  \n",
       "99996                  0                 0  \n",
       "99997                  0                 0  \n",
       "99998                  0                 0  \n",
       "99999                  0                 0  \n",
       "\n",
       "[100000 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('./archive/train.csv')\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = train.drop('Credit_Score', axis = 1)\n",
    "y = train['Credit_Score']\n",
    "x_train, x_test, y_train, y_test= train_test_split(x, y, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90000, 30)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51994    2\n",
       "77540    1\n",
       "16382    1\n",
       "83439    1\n",
       "61618    2\n",
       "        ..\n",
       "6265     2\n",
       "54886    0\n",
       "76820    1\n",
       "860      1\n",
       "15795    0\n",
       "Name: Credit_Score, Length: 90000, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Credit_Score\n",
       "1    47867\n",
       "0    26053\n",
       "2    16080\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Building\n",
    "\n",
    "The cells below actually build the Decision Tree, Random Forest and the XGBoost models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = DecisionTreeClassifier()\n",
    "rf = RandomForestClassifier()\n",
    "xgb = XGBClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              objective=&#x27;multi:softprob&#x27;, predictor=None, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              objective=&#x27;multi:softprob&#x27;, predictor=None, ...)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              objective='multi:softprob', predictor=None, ...)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.fit(x_train, y_train)\n",
    "rf.fit(x_train, y_train)\n",
    "xgb.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtprediction = dt.predict(x_test)\n",
    "rfprediction = rf.predict(x_test)\n",
    "xgbprediction = xgb.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation\n",
    "\n",
    "Here we will judge the performance of these models through 4 metrics, the accuracy, the precision, the recall and the f1 score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Random Forest</th>\n",
       "      <td>80.23</td>\n",
       "      <td>79.334735</td>\n",
       "      <td>78.983727</td>\n",
       "      <td>79.136580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost</th>\n",
       "      <td>74.56</td>\n",
       "      <td>72.601110</td>\n",
       "      <td>73.758800</td>\n",
       "      <td>73.138585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Decision Tree</th>\n",
       "      <td>72.62</td>\n",
       "      <td>71.238225</td>\n",
       "      <td>70.939036</td>\n",
       "      <td>71.085906</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Accuracy  Precision     Recall   F1_Score\n",
       "Random Forest     80.23  79.334735  78.983727  79.136580\n",
       "XGBoost           74.56  72.601110  73.758800  73.138585\n",
       "Decision Tree     72.62  71.238225  70.939036  71.085906"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = [dtprediction, rfprediction, xgbprediction]\n",
    "\n",
    "accuracy, precision, recall, f1 = [], [], [], []\n",
    "\n",
    "for prediction in predictions:\n",
    "    accuracy.append(accuracy_score(y_test, prediction) * 100)\n",
    "    precision.append(precision_score(y_test, prediction, average='macro') * 100)\n",
    "    recall.append(recall_score(y_test, prediction, average='macro') * 100)\n",
    "    f1.append(f1_score(y_test, prediction, average='macro') * 100)\n",
    "\n",
    "finaldf = pd.DataFrame({'Accuracy': accuracy, 'Precision': precision, 'Recall': recall, 'F1_Score': f1}, index = ['Decision Tree', 'Random Forest', 'XGBoost'])\n",
    "\n",
    "finaldf.sort_values('Accuracy', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "\n",
    "Thus we can conclude that Random Forest gives us the best result for this particular dataset."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}