audio-data-and-RF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using signals to obtain MFCC Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Volume in drive C is Windows\n",
      " Volume Serial Number is 0C09-7045\n",
      "\n",
      " Directory of C:\\Users\\Jonathan\\Desktop\\STA141C_Project\n",
      "\n",
      "06/09/2019  12:54 PM    <DIR>          .\n",
      "06/09/2019  12:54 PM    <DIR>          ..\n",
      "06/09/2019  12:38 PM    <DIR>          .ipynb_checkpoints\n",
      "06/09/2019  08:54 AM        25,423,950 21_Savage_-_Partments_Slaughter_King_2015_+_DOWNLOAD-VbCY_3mWXec.wav\n",
      "06/08/2019  03:40 PM             5,441 artists.txt\n",
      "06/08/2019  03:40 PM        30,863,438 AtTheClub.wav\n",
      "06/09/2019  12:54 PM            62,689 audio-data-and-RF.ipynb\n",
      "06/09/2019  08:59 AM            12,659 audio-RF.ipynb\n",
      "06/08/2019  03:40 PM           106,820 BillboardAPI_EDA.ipynb\n",
      "06/08/2019  03:40 PM         7,326,487 Cleaned_data.pkl\n",
      "06/08/2019  03:40 PM               236 combiningFunction.R\n",
      "06/09/2019  08:54 AM         9,273,422 DaBaby_-_Kujo-U5rk2qASIOI.wav\n",
      "06/09/2019  08:54 AM        26,067,022 Denzel_Curry_-_SHAKE_88_feat._Sam_Sneak-ltN58roAM5c.wav\n",
      "06/08/2019  03:40 PM             6,176 Final Data Creation.ipynb\n",
      "06/08/2019  03:40 PM        25,382,990 FineChina.wav\n",
      "06/08/2019  03:40 PM        32,718,926 GetMine.wav\n",
      "06/08/2019  03:40 PM             1,473 get_artist_features.R\n",
      "06/08/2019  03:40 PM        34,263,118 HotBoy.wav\n",
      "06/08/2019  03:40 PM        44,720,206 ILikeIt.wav\n",
      "06/08/2019  03:40 PM        29,495,374 jungle.wav\n",
      "06/08/2019  03:40 PM            23,837 LDA.ipynb\n",
      "06/08/2019  03:40 PM           397,328 lda_feature_matrix.npy\n",
      "06/09/2019  08:54 AM        25,587,790 Lil_Baby_-_100_Round_Feat._Lil_Yachty_Perfect_Timing-i9VbnLXkxeI.wav\n",
      "06/08/2019  03:40 PM             8,265 Logistic Regression all variables.ipynb\n",
      "06/08/2019  03:40 PM             9,702 Logistic Regression Recursive Feature Elimination.ipynb\n",
      "06/08/2019  03:40 PM            10,119 Logistic Regression using selectkbest feature selection.ipynb\n",
      "06/08/2019  03:40 PM             6,936 Logistic Regression Using Variables from LDA.ipynb\n",
      "06/08/2019  03:40 PM             7,641 Logistic Regression.ipynb\n",
      "06/08/2019  04:57 PM        43,656,070 merged_data.pkl\n",
      "06/08/2019  03:40 PM        43,545,694 MLKDream.wav\n",
      "06/08/2019  03:40 PM         1,996,613 R&B10yrDF.pkl\n",
      "06/09/2019  09:34 AM           126,801 RandomForrests.ipynb\n",
      "06/08/2019  03:40 PM                17 README.md\n",
      "06/08/2019  03:40 PM        38,457,422 Rockstar.wav\n",
      "06/08/2019  03:40 PM        38,940,750 SongKilledBeforeYoungThug.wav\n",
      "06/08/2019  03:40 PM        11,825,902 song_feature_data\n",
      "06/09/2019  08:54 AM    <DIR>          SoundAnalysis\n",
      "06/08/2019  03:40 PM           151,909 soundAnalysisPractice.ipynb\n",
      "06/08/2019  03:40 PM             3,193 spotifyAPI.R\n",
      "06/08/2019  03:40 PM            17,417 spotifyData.ipynb\n",
      "06/08/2019  03:40 PM        40,763,470 Starboy.wav\n",
      "06/08/2019  03:40 PM        45,523,022 StTropez.wav\n",
      "06/08/2019  03:40 PM        27,926,606 Sunflower.wav\n",
      "06/08/2019  03:40 PM            40,086 SVMOnCleanedData.ipynb\n",
      "06/08/2019  03:40 PM            20,999 SVM_Classification.ipynb\n",
      "06/08/2019  03:40 PM             7,591 uni_artists.pkl\n",
      "              42 File(s)    584,785,607 bytes\n",
      "               4 Dir(s)  439,632,613,376 bytes free\n"
     ]
    }
   ],
   "source": [
    "! dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.metrics import roc_curve, auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* signal – the audio signal from which to compute features. Should be an N*1 array\n",
    "* samplerate – the samplerate of the signal we are working with.\n",
    "* winlen – the length of the analysis window in seconds. Default is 0.025s (25 milliseconds)\n",
    "* winstep – the step between successive windows in seconds. Default is 0.01s (10 milliseconds)\n",
    "* numcep – the number of cepstrum to return, default 13\n",
    "* nfilt – the number of filters in the filterbank, default 26.\n",
    "* nfft – the FFT size. Default is 512.\n",
    "* lowfreq – lowest band edge of mel filters. In Hz, default is 0.\n",
    "* highfreq – highest band edge of mel filters. In Hz, default is samplerate/2\n",
    "* preemph – apply preemphasis filter with preemph as coefficient. 0 is no filter. Default is 0.97.\n",
    "* ceplifter – apply a lifter to final cepstral coefficients. 0 is no lifter. Default is 22.\n",
    "* appendEnergy – if this is true, the zeroth cepstral coefficient is replaced with the log of the total frame energy.\n",
    "* winfunc – the analysis window to apply to each frame. By default no window is applied. You can use numpy window functions here e.g. winfunc=numpy.hamming"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HotBoy -  1\n",
    "Sunflower -1\n",
    "ILikeIt - 1\n",
    "Rockstar - 1\n",
    "Starboy - 1\n",
    "\n",
    "\n",
    "MLKDream - 0\n",
    "jungle - 0 \n",
    "GetMine - 0\n",
    "AtTheClub - 0\n",
    "FineChina - 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from python_speech_features import mfcc\n",
    "from python_speech_features import delta\n",
    "from python_speech_features import logfbank\n",
    "import scipy.io.wavfile as wav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n"
     ]
    }
   ],
   "source": [
    "(rate,sig) = wav.read(\"HotBoy.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "#d_mfcc_feat = delta(mfcc_feat, 2)\n",
    "#fbank_feat = logfbank(sig,rate)\n",
    "#hot_boy = pd.DataFrame(np.mean(fbank_feat, axis=0)).transpose()\n",
    "one = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "one['is_hit'] = 1\n",
    "\n",
    "(rate,sig) = wav.read(\"Sunflower.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "two = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "two['is_hit'] = 1\n",
    "\n",
    "(rate,sig) = wav.read(\"ILikeIt.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "three = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "three['is_hit'] = 1\n",
    "\n",
    "(rate,sig) = wav.read(\"Rockstar.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "four = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "four['is_hit'] = 1\n",
    "\n",
    "(rate,sig) = wav.read(\"Starboy.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "five = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "five['is_hit'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n",
      "WARNING:root:frame length (1103) is greater than FFT size (512), frame will be truncated. Increase NFFT to avoid.\n"
     ]
    }
   ],
   "source": [
    "(rate,sig) = wav.read(\"21_Savage_-_Partments_Slaughter_King_2015_+_DOWNLOAD-VbCY_3mWXec.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "six = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "six['is_hit'] = 0\n",
    "\n",
    "(rate,sig) = wav.read(\"DaBaby_-_Kujo-U5rk2qASIOI.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "seven = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "seven['is_hit'] = 0\n",
    "\n",
    "(rate,sig) = wav.read(\"Denzel_Curry_-_SHAKE_88_feat._Sam_Sneak-ltN58roAM5c.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "eight = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "eight['is_hit'] = 0\n",
    "\n",
    "(rate,sig) = wav.read(\"Lil_Baby_-_100_Round_Feat._Lil_Yachty_Perfect_Timing-i9VbnLXkxeI.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "nine = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "nine['is_hit'] = 0\n",
    "\n",
    "(rate,sig) = wav.read(\"SongKilledBeforeYoungThug.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "ten = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "ten['is_hit'] = 0\n",
    "\n",
    "(rate,sig) = wav.read(\"StTropez.wav\")\n",
    "mfcc_feat = mfcc(sig,rate)\n",
    "eleven = pd.DataFrame(np.mean(mfcc_feat,axis=0)).transpose()\n",
    "eleven['is_hit'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine all rows together\n",
    "final = one.append(two, ignore_index=True)\n",
    "final = final.append(three, ignore_index=True)\n",
    "final = final.append(four, ignore_index=True)\n",
    "final = final.append(five, ignore_index=True)\n",
    "final = final.append(six, ignore_index=True)\n",
    "final = final.append(seven, ignore_index=True)\n",
    "final = final.append(eight, ignore_index=True)\n",
    "final = final.append(nine, ignore_index=True)\n",
    "final = final.append(ten, ignore_index=True)\n",
    "final = final.append(eleven, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>is_hit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18.492276</td>\n",
       "      <td>10.332741</td>\n",
       "      <td>-14.095161</td>\n",
       "      <td>-14.755182</td>\n",
       "      <td>22.812128</td>\n",
       "      <td>-41.080560</td>\n",
       "      <td>24.291515</td>\n",
       "      <td>-19.384292</td>\n",
       "      <td>-1.988264</td>\n",
       "      <td>6.711872</td>\n",
       "      <td>-7.014448</td>\n",
       "      <td>1.888141</td>\n",
       "      <td>-0.489726</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18.829886</td>\n",
       "      <td>15.227135</td>\n",
       "      <td>-6.163772</td>\n",
       "      <td>-12.996859</td>\n",
       "      <td>25.377363</td>\n",
       "      <td>-36.513184</td>\n",
       "      <td>22.272397</td>\n",
       "      <td>-20.239632</td>\n",
       "      <td>-8.689257</td>\n",
       "      <td>-6.772696</td>\n",
       "      <td>-0.119384</td>\n",
       "      <td>-2.187521</td>\n",
       "      <td>-0.281808</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.544789</td>\n",
       "      <td>11.458744</td>\n",
       "      <td>-13.363123</td>\n",
       "      <td>-20.034201</td>\n",
       "      <td>24.453032</td>\n",
       "      <td>-41.312218</td>\n",
       "      <td>26.694799</td>\n",
       "      <td>-18.305884</td>\n",
       "      <td>-2.328175</td>\n",
       "      <td>0.656279</td>\n",
       "      <td>-6.955135</td>\n",
       "      <td>1.901923</td>\n",
       "      <td>2.360378</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17.001445</td>\n",
       "      <td>13.360960</td>\n",
       "      <td>-0.446789</td>\n",
       "      <td>-10.022686</td>\n",
       "      <td>22.314148</td>\n",
       "      <td>-38.592228</td>\n",
       "      <td>28.435354</td>\n",
       "      <td>-14.097012</td>\n",
       "      <td>-2.668326</td>\n",
       "      <td>1.125349</td>\n",
       "      <td>-4.401842</td>\n",
       "      <td>1.890709</td>\n",
       "      <td>4.143143</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18.549076</td>\n",
       "      <td>10.037066</td>\n",
       "      <td>-6.512212</td>\n",
       "      <td>-9.687447</td>\n",
       "      <td>24.681812</td>\n",
       "      <td>-38.444779</td>\n",
       "      <td>35.410108</td>\n",
       "      <td>-13.901642</td>\n",
       "      <td>-5.405690</td>\n",
       "      <td>4.824123</td>\n",
       "      <td>-9.279814</td>\n",
       "      <td>8.575719</td>\n",
       "      <td>3.932357</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20.649120</td>\n",
       "      <td>5.523562</td>\n",
       "      <td>-22.374754</td>\n",
       "      <td>-14.201121</td>\n",
       "      <td>28.760701</td>\n",
       "      <td>-37.834523</td>\n",
       "      <td>33.750815</td>\n",
       "      <td>-11.742357</td>\n",
       "      <td>-2.036246</td>\n",
       "      <td>10.488903</td>\n",
       "      <td>-13.026305</td>\n",
       "      <td>9.979980</td>\n",
       "      <td>-0.876847</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>19.140319</td>\n",
       "      <td>2.303054</td>\n",
       "      <td>-18.916943</td>\n",
       "      <td>-14.085364</td>\n",
       "      <td>33.862196</td>\n",
       "      <td>-40.959910</td>\n",
       "      <td>25.704343</td>\n",
       "      <td>-16.039340</td>\n",
       "      <td>4.724307</td>\n",
       "      <td>5.919328</td>\n",
       "      <td>-14.427974</td>\n",
       "      <td>5.655541</td>\n",
       "      <td>0.565913</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>20.119259</td>\n",
       "      <td>5.211895</td>\n",
       "      <td>-13.663719</td>\n",
       "      <td>-11.479067</td>\n",
       "      <td>28.874323</td>\n",
       "      <td>-46.793734</td>\n",
       "      <td>30.082915</td>\n",
       "      <td>-16.727747</td>\n",
       "      <td>-2.708399</td>\n",
       "      <td>7.588171</td>\n",
       "      <td>-15.564215</td>\n",
       "      <td>9.336860</td>\n",
       "      <td>-1.868600</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>17.817966</td>\n",
       "      <td>6.819132</td>\n",
       "      <td>-10.918760</td>\n",
       "      <td>-1.707474</td>\n",
       "      <td>25.758318</td>\n",
       "      <td>-39.222919</td>\n",
       "      <td>26.986939</td>\n",
       "      <td>-19.358693</td>\n",
       "      <td>-11.498346</td>\n",
       "      <td>9.323270</td>\n",
       "      <td>-15.518924</td>\n",
       "      <td>6.347665</td>\n",
       "      <td>-3.028601</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>19.323565</td>\n",
       "      <td>12.238632</td>\n",
       "      <td>-14.098987</td>\n",
       "      <td>-15.628463</td>\n",
       "      <td>29.422828</td>\n",
       "      <td>-34.472566</td>\n",
       "      <td>30.607839</td>\n",
       "      <td>-13.722605</td>\n",
       "      <td>-3.010370</td>\n",
       "      <td>-3.081251</td>\n",
       "      <td>-12.629850</td>\n",
       "      <td>-6.447067</td>\n",
       "      <td>0.826418</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>17.887588</td>\n",
       "      <td>12.102747</td>\n",
       "      <td>-3.330724</td>\n",
       "      <td>-10.203302</td>\n",
       "      <td>28.883931</td>\n",
       "      <td>-34.084093</td>\n",
       "      <td>30.629613</td>\n",
       "      <td>-14.301521</td>\n",
       "      <td>-1.699471</td>\n",
       "      <td>4.500315</td>\n",
       "      <td>-4.934256</td>\n",
       "      <td>3.873923</td>\n",
       "      <td>3.375040</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0          1          2          3          4          5  \\\n",
       "0   18.492276  10.332741 -14.095161 -14.755182  22.812128 -41.080560   \n",
       "1   18.829886  15.227135  -6.163772 -12.996859  25.377363 -36.513184   \n",
       "2   19.544789  11.458744 -13.363123 -20.034201  24.453032 -41.312218   \n",
       "3   17.001445  13.360960  -0.446789 -10.022686  22.314148 -38.592228   \n",
       "4   18.549076  10.037066  -6.512212  -9.687447  24.681812 -38.444779   \n",
       "5   20.649120   5.523562 -22.374754 -14.201121  28.760701 -37.834523   \n",
       "6   19.140319   2.303054 -18.916943 -14.085364  33.862196 -40.959910   \n",
       "7   20.119259   5.211895 -13.663719 -11.479067  28.874323 -46.793734   \n",
       "8   17.817966   6.819132 -10.918760  -1.707474  25.758318 -39.222919   \n",
       "9   19.323565  12.238632 -14.098987 -15.628463  29.422828 -34.472566   \n",
       "10  17.887588  12.102747  -3.330724 -10.203302  28.883931 -34.084093   \n",
       "\n",
       "            6          7          8          9         10        11        12  \\\n",
       "0   24.291515 -19.384292  -1.988264   6.711872  -7.014448  1.888141 -0.489726   \n",
       "1   22.272397 -20.239632  -8.689257  -6.772696  -0.119384 -2.187521 -0.281808   \n",
       "2   26.694799 -18.305884  -2.328175   0.656279  -6.955135  1.901923  2.360378   \n",
       "3   28.435354 -14.097012  -2.668326   1.125349  -4.401842  1.890709  4.143143   \n",
       "4   35.410108 -13.901642  -5.405690   4.824123  -9.279814  8.575719  3.932357   \n",
       "5   33.750815 -11.742357  -2.036246  10.488903 -13.026305  9.979980 -0.876847   \n",
       "6   25.704343 -16.039340   4.724307   5.919328 -14.427974  5.655541  0.565913   \n",
       "7   30.082915 -16.727747  -2.708399   7.588171 -15.564215  9.336860 -1.868600   \n",
       "8   26.986939 -19.358693 -11.498346   9.323270 -15.518924  6.347665 -3.028601   \n",
       "9   30.607839 -13.722605  -3.010370  -3.081251 -12.629850 -6.447067  0.826418   \n",
       "10  30.629613 -14.301521  -1.699471   4.500315  -4.934256  3.873923  3.375040   \n",
       "\n",
       "    is_hit  \n",
       "0        1  \n",
       "1        1  \n",
       "2        1  \n",
       "3        1  \n",
       "4        1  \n",
       "5        0  \n",
       "6        0  \n",
       "7        0  \n",
       "8        0  \n",
       "9        0  \n",
       "10       0  "
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>is_hit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>18.850481</td>\n",
       "      <td>9.510515</td>\n",
       "      <td>-11.262268</td>\n",
       "      <td>-12.254652</td>\n",
       "      <td>26.836434</td>\n",
       "      <td>-39.028247</td>\n",
       "      <td>28.624240</td>\n",
       "      <td>-16.165520</td>\n",
       "      <td>-3.391658</td>\n",
       "      <td>3.753060</td>\n",
       "      <td>-9.442922</td>\n",
       "      <td>3.710534</td>\n",
       "      <td>0.787061</td>\n",
       "      <td>0.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.061440</td>\n",
       "      <td>3.999773</td>\n",
       "      <td>6.609795</td>\n",
       "      <td>4.620308</td>\n",
       "      <td>3.440336</td>\n",
       "      <td>3.563432</td>\n",
       "      <td>3.955971</td>\n",
       "      <td>2.836882</td>\n",
       "      <td>4.155054</td>\n",
       "      <td>5.293902</td>\n",
       "      <td>5.167379</td>\n",
       "      <td>5.032530</td>\n",
       "      <td>2.398597</td>\n",
       "      <td>0.522233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>17.001445</td>\n",
       "      <td>2.303054</td>\n",
       "      <td>-22.374754</td>\n",
       "      <td>-20.034201</td>\n",
       "      <td>22.314148</td>\n",
       "      <td>-46.793734</td>\n",
       "      <td>22.272397</td>\n",
       "      <td>-20.239632</td>\n",
       "      <td>-11.498346</td>\n",
       "      <td>-6.772696</td>\n",
       "      <td>-15.564215</td>\n",
       "      <td>-6.447067</td>\n",
       "      <td>-3.028601</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>18.189932</td>\n",
       "      <td>6.171347</td>\n",
       "      <td>-14.097074</td>\n",
       "      <td>-14.478152</td>\n",
       "      <td>24.567422</td>\n",
       "      <td>-41.020235</td>\n",
       "      <td>26.199571</td>\n",
       "      <td>-18.832288</td>\n",
       "      <td>-4.208030</td>\n",
       "      <td>0.890814</td>\n",
       "      <td>-13.727139</td>\n",
       "      <td>1.889425</td>\n",
       "      <td>-0.683287</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>18.829886</td>\n",
       "      <td>10.332741</td>\n",
       "      <td>-13.363123</td>\n",
       "      <td>-12.996859</td>\n",
       "      <td>25.758318</td>\n",
       "      <td>-38.592228</td>\n",
       "      <td>28.435354</td>\n",
       "      <td>-16.039340</td>\n",
       "      <td>-2.668326</td>\n",
       "      <td>4.824123</td>\n",
       "      <td>-9.279814</td>\n",
       "      <td>3.873923</td>\n",
       "      <td>0.565913</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>19.434177</td>\n",
       "      <td>12.170689</td>\n",
       "      <td>-6.337992</td>\n",
       "      <td>-10.112994</td>\n",
       "      <td>28.879127</td>\n",
       "      <td>-37.173854</td>\n",
       "      <td>30.618726</td>\n",
       "      <td>-13.999327</td>\n",
       "      <td>-2.012255</td>\n",
       "      <td>7.150021</td>\n",
       "      <td>-5.944695</td>\n",
       "      <td>7.461692</td>\n",
       "      <td>2.867709</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>20.649120</td>\n",
       "      <td>15.227135</td>\n",
       "      <td>-0.446789</td>\n",
       "      <td>-1.707474</td>\n",
       "      <td>33.862196</td>\n",
       "      <td>-34.084093</td>\n",
       "      <td>35.410108</td>\n",
       "      <td>-11.742357</td>\n",
       "      <td>4.724307</td>\n",
       "      <td>10.488903</td>\n",
       "      <td>-0.119384</td>\n",
       "      <td>9.979980</td>\n",
       "      <td>4.143143</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0          1          2          3          4          5  \\\n",
       "count  11.000000  11.000000  11.000000  11.000000  11.000000  11.000000   \n",
       "mean   18.850481   9.510515 -11.262268 -12.254652  26.836434 -39.028247   \n",
       "std     1.061440   3.999773   6.609795   4.620308   3.440336   3.563432   \n",
       "min    17.001445   2.303054 -22.374754 -20.034201  22.314148 -46.793734   \n",
       "25%    18.189932   6.171347 -14.097074 -14.478152  24.567422 -41.020235   \n",
       "50%    18.829886  10.332741 -13.363123 -12.996859  25.758318 -38.592228   \n",
       "75%    19.434177  12.170689  -6.337992 -10.112994  28.879127 -37.173854   \n",
       "max    20.649120  15.227135  -0.446789  -1.707474  33.862196 -34.084093   \n",
       "\n",
       "               6          7          8          9         10         11  \\\n",
       "count  11.000000  11.000000  11.000000  11.000000  11.000000  11.000000   \n",
       "mean   28.624240 -16.165520  -3.391658   3.753060  -9.442922   3.710534   \n",
       "std     3.955971   2.836882   4.155054   5.293902   5.167379   5.032530   \n",
       "min    22.272397 -20.239632 -11.498346  -6.772696 -15.564215  -6.447067   \n",
       "25%    26.199571 -18.832288  -4.208030   0.890814 -13.727139   1.889425   \n",
       "50%    28.435354 -16.039340  -2.668326   4.824123  -9.279814   3.873923   \n",
       "75%    30.618726 -13.999327  -2.012255   7.150021  -5.944695   7.461692   \n",
       "max    35.410108 -11.742357   4.724307  10.488903  -0.119384   9.979980   \n",
       "\n",
       "              12     is_hit  \n",
       "count  11.000000  11.000000  \n",
       "mean    0.787061   0.454545  \n",
       "std     2.398597   0.522233  \n",
       "min    -3.028601   0.000000  \n",
       "25%    -0.683287   0.000000  \n",
       "50%     0.565913   0.000000  \n",
       "75%     2.867709   1.000000  \n",
       "max     4.143143   1.000000  "
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.describe()\n",
    "# higher variance -> better at explaining?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting the test data into the predictor and response variables \n",
    "X = final.iloc[:, :-1].values  \n",
    "y = final.iloc[:, -1].values  \n",
    "\n",
    "# Predicting y values from test data using the model and coefficents found in part A\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=1)\n",
    "\n",
    "# random forest model creation\n",
    "rfc = RandomForestClassifier(n_estimators=25)\n",
    "rfc.fit(X_train,y_train)\n",
    "\n",
    "# predictions\n",
    "y_predict = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Confusion Matrix ===\n",
      "Predicted  0\n",
      "Actual      \n",
      "0          1\n",
      "1          3\n",
      "\n",
      "\n",
      "=== Classification Report ===\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.25      1.00      0.40         1\n",
      "           1       0.00      0.00      0.00         3\n",
      "\n",
      "   micro avg       0.25      0.25      0.25         4\n",
      "   macro avg       0.12      0.50      0.20         4\n",
      "weighted avg       0.06      0.25      0.10         4\n",
      "\n",
      "=== All AUC Scores ===\n",
      "[1. 1. 0. 1. 1.]\n",
      "\n",
      "\n",
      "=== Mean AUC Score ===\n",
      "Mean AUC Score - Random Forest:  0.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Jonathan\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "rfc_cv_score = cross_val_score(rfc, X, y, cv=5, scoring=\"roc_auc\")\n",
    "\n",
    "print(\"=== Confusion Matrix ===\")\n",
    "# print(confusion_matrix(y_test, y_predict))\n",
    "x = pd.crosstab(y_test, y_predict, rownames=['Actual'], colnames=['Predicted'])\n",
    "print(x)\n",
    "print('\\n')\n",
    "print(\"=== Classification Report ===\")\n",
    "print(classification_report(y_test, y_predict))\n",
    "print(\"=== All AUC Scores ===\")\n",
    "print(rfc_cv_score)\n",
    "print('\\n')\n",
    "print(\"=== Mean AUC Score ===\")\n",
    "print(\"Mean AUC Score - Random Forest: \", rfc_cv_score.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:                    7 Importance: 0.2\n",
      "Variable:                    6 Importance: 0.16\n",
      "Variable:                   11 Importance: 0.16\n",
      "Variable:                    1 Importance: 0.08\n",
      "Variable:                    9 Importance: 0.08\n",
      "Variable:                    3 Importance: 0.05\n",
      "Variable:                    4 Importance: 0.04\n",
      "Variable:                   10 Importance: 0.04\n",
      "Variable:                    8 Importance: 0.02\n",
      "Variable:                    2 Importance: 0.01\n",
      "Variable:                    0 Importance: 0.0\n",
      "Variable:                    5 Importance: 0.0\n",
      "Variable:                   12 Importance: 0.0\n"
     ]
    }
   ],
   "source": [
    "importances = list(rfc.feature_importances_)\n",
    "feature_list = list(final.drop('is_hit', axis=1).columns)\n",
    "\n",
    "# Get numerical feature importances\n",
    "importances = list(rfc.feature_importances_)\n",
    "\n",
    "# List of tuples with variable and importance\n",
    "feature_importances = [(feature, round(importance, 2)) for feature, importance in zip(feature_list, importances)]\n",
    "\n",
    "# Sort the feature importances by most important first\n",
    "feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)\n",
    "\n",
    "# Print out the feature and importances \n",
    "items = [print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importances]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEZCAYAAAC99aPhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XucXWV97/HPlwQCxGS4ZCSQC+ESLyAeJCHY9giogKECwRZKgEJQbNTKQY+tFa2CptqCpx6qFS2pXOQaA1YYazAgGHpaRWcCEQghMokhGUMgBJgJm2vgd/5Yz5SVnT0ze2btlckk3/frtV9Z67n89rP2TPZv1rNuigjMzMwGaqfBHoCZmQ1tTiRmZlaIE4mZmRXiRGJmZoU4kZiZWSFOJGZmVogTiW13JE2U9LykYXW0PVZSRy/110r6amNHaLZ9cSKxQSVpoaQ5NcpnSFonaXh/Y0bE6oh4U0S81phRDoykkHTwYI6hm6RVko4b7HHY9smJxAbbtcA5klRVfg5wY0Rs6k+wgSSe7Zk/D9sanEhssN0G7AW8p7tA0p7AScB1af2Dkh6Q1CVpjaQv59pOSn/5ny9pNXBPrmx4avNhScskbZS0UtLHqgch6QuSnk5/uZ/d02AlnSRpiaTnJP1C0jvr2UhJX5Z0i6Qb0jgekvQWSZ+X9FTarhNy7RdJ+gdJv5bUKel2SXvl6k+RtDSNY5Gkt+fqVkn6nKQHgYqkm4GJwI/TlN/fpHa3pL2+Tkn/IenQXIxrJV0h6SdpvL+SdFCu/lBJd0l6RtKTkr6QyneSdJGkFZI2SJrfPW5Ju6bt35DG3Sppn3o+P9u2OZHYoIqIF4H5wLm54j8DHo2I36T1SqrfA/gg8AlJp1aFOgZ4O/CBGm/zFFliGg18GLhc0hG5+rHAGGAcMAuYK+mt1UFSn6uBjwF7A1cCLZJG1Lm5JwPXA3sCDwALyf4PjgPmpHh55wIfAfYDNgHfSuN4C3Az8GmgGVhAliR2yfU9k+yz2iMizgRWAyenKb+vpzZ3AJOBNwP3AzdWvf+ZwFfSeNuBr6X3HwX8DPhpGtvBwN2pz4XAqWQ/j/2AZ4ErUt0soAmYQPb5fRx4sa8PzYaAiPDLr0F9Af8T6AR2S+v/BfzvXtr/E3B5Wp4EBHBgrr67bHgP/W8DPpWWjyX7kh6Zq58PfCktXwt8NS1/F/i7qljLgWN6eJ8ADk7LXwbuytWdDDwPDEvro1L7PdL6IuDSXPtDgFeAYcCXgPm5up2A3wPHpvVVwEeqxrIKOK6Xz3SP9P5Nue3+Xq7+j8mSO2QJ5oEe4iwD3p9b3xd4FRhOlhR/AbxzsH/n/Grsy3skNugi4j+B9cAMSQcCRwI3dddLOkrSzyWtl9RJ9pfsmKowa3qKL+lESfelaZjnyL4U8/2fjYhKbv1xsr+mq+0P/FWalnkuxZrQQ9tanswtvwg8HW+cEND9l/mbcm3y2/Q4sHMa935pHYCIeD21HddD3y1IGibp0jQF1UWWaGDzz2VdbvmF3NgmACt6CL0/8KPc57MMeA3Yh2xvbCEwT9JaSV+XtHNv47ShwYnEthXXkU3lnAPcGRH5L92bgBZgQkQ0Af8CVB+cr3kb6zTt9EPgH4F9ImIPsqmgfP89JY3MrU8E1tYItwb4WkTskXvtHhE3172V/TOhakyvAk+nse3fXZFOVJhAtlfSrfrzqF4/C5gBHEc23TSpO1wd41oDHNRL3YlVn9GuEfH7iHg1Ir4SEYcAf0g23XhuD3FsCHEisW3FdWRfan8BfL+qbhTwTES8JGka2ZdgvXYBRpDt8WySdCJwQo12X5G0i6T3kH3B3VKjzb8CH097SJI0Mp0IMKof4+mPP5d0iKTdyY6h3Jr2YOYDH5T0/vQX/V8BL5NNG/XkSeDA3Pqo1GcDsDvw9/0Y178DYyV9WtIISaMkHZXq/gX4mqT9ASQ1S5qRlt8r6TBl1/d0kSXGQT1F2xrDicS2CRGxiuyLcCTZ3kfeXwJzJG0ELib7Iq037kayA8DzyQ78nlUj/rpUt5bsgPPHI+LRGrHayBLdt1P7duC8escyANeTHatYB+xKth1ExHLgz4F/JttDOZnsQPorvcT6B+CLacrpr8kS9+NkezGPAPfVO6j0mR6f3ncd8Bjw3lT9TbLP987087oP6E4yY4FbyZLIMuBe4IZ639e2XYrwg63MtjWSFgE3RMT3BnssZn3xHomZmRXiRGJmZoV4asvMzArxHomZmRXiRGJmZoXsEHcGHTNmTEyaNGmwh2FmNqQsXrz46Yho7qvdDpFIJk2aRFtb22APw8xsSJH0eN+tPLVlZmYFOZGYmVkhTiRmZlaIE4mZmRXiRGJmZoWUmkgkTZe0XFK7pItq1H9G0iOSHpR0d/etp1PdLEmPpdesXPkUZc+7bpf0rfQsBjMzGySlJZL0zIErgBPJHhN6pqRDqpo9AEyNiHeS3V7666nvXsAlZLefngZcImnP1Oe7wGyyZ01PBqaXtQ1mZta3Mq8jmQa0R8RKAEnzyJ7I9kh3g4j4ea79fWTPWAD4ANnzrZ9Jfe8Cpqdba4+OiF+m8uuAU4E7StwOs21OZ2cnlUql74Z1GjlyJE1NTQ2LZzuWMhPJODZ/bnQHbzzgppbzeSMh1Oo7Lr06apRvQdJssj0XJk6c2J9xm23TOjs7OWjiRDZ0dTUs5t6jR7Ni9WonExuQMhNJrWMXPT1X+8+BqcAxffStO2ZEzAXmAkydOtW3OLbtRqVSYUNXF4vJHjlY1DpgSlcXlUrFicQGpMxE0gFMyK2PJ3uU6WYkHQf8LXBMRLyc63tsVd9FqXx8XzHNdgRjgf0GexBmlHvWViswWdIBknYBZlL1rGxJ7wKuBE6JiKdyVQuBEyTtmQ6ynwAsjIgngI2S3p3O1joXuL3EbTAzsz6UtkcSEZskXUCWFIYBV0fEUklzgLaIaAH+D/Am4JZ0Fu/qiDglIp6R9HdkyQhgTveBd+ATwLXAbmTHVHyg3cxsEJV699+IWAAsqCq7OLd8XC99rwaurlHeBryjgcM0M7MCfGW7mZkV4kRiZmaFOJGYmVkhTiRmZlaIE4mZmRXiRGJmZoU4kZiZWSFOJGZmVogTiZmZFeJEYmZmhTiRmJlZIU4kZmZWiBOJmZkV4kRiZmaFOJGYmVkhTiRmZlZIqYlE0nRJyyW1S7qoRv3Rku6XtEnSabny90paknu9JOnUVHetpN/l6g4vcxvMzKx3pT0hUdIw4ArgeKADaJXUEhGP5JqtBs4D/jrfNyJ+Dhye4uwFtAN35pp8NiJuLWvsZmZWvzIftTsNaI+IlQCS5gEzgP9OJBGxKtW93kuc04A7IuKF8oZqZmYDVebU1jhgTW69I5X110zg5qqyr0l6UNLlkkYMdIBmZlZcmYlENcqiXwGkfYHDgIW54s8DbwOOBPYCPtdD39mS2iS1rV+/vj9va2Zm/VBmIukAJuTWxwNr+xnjz4AfRcSr3QUR8URkXgauIZtC20JEzI2IqRExtbm5uZ9va2Zm9SozkbQCkyUdIGkXsimqln7GOJOqaa20l4IkAacCDzdgrGZmNkClJZKI2ARcQDYttQyYHxFLJc2RdAqApCMldQCnA1dKWtrdX9Iksj2ae6tC3yjpIeAhYAzw1bK2wczM+lbmWVtExAJgQVXZxbnlVrIpr1p9V1Hj4HxEvK+xozQzsyJ8ZbuZmRXiRGJmZoU4kZiZWSFOJGZmVogTiZmZFeJEYmZmhTiRmJlZIU4kZmZWiBOJmZkVUuqV7WY7ss7OTiqVSkNijRw5kqampobEsuL8s92cE4lZCTo7Ozlo4kQ2dHU1JN7eo0ezYvXqIf+Fsz3wz3ZLTiRmJahUKmzo6mIxMLZgrHXAlK4uKpXKkP6y2V74Z7slJxKzEo0F9hvsQVgp/LN9gw+2m5lZIU4kZmZWiBOJmZkV4kRiZmaFlJpIJE2XtFxSu6SLatQfLel+SZsknVZV95qkJenVkis/QNKvJD0m6QfpefBmZjZISkskkoYBVwAnAocAZ0o6pKrZauA84KYaIV6MiMPT65Rc+WXA5RExGXgWOL/hgzczs7qVuUcyDWiPiJUR8QowD5iRbxARqyLiQeD1egJKEvA+4NZU9H3g1MYN2czM+qvMRDIOWJNb70hl9dpVUpuk+yR1J4u9geciYtMAY5qZWYOVeUGiapRFP/pPjIi1kg4E7pH0EFDrngQ1Y0qaDcwGmDhxYj/e1szM+qPMPZIOYEJufTywtt7OEbE2/bsSWAS8C3ga2ENSdwLsMWZEzI2IqRExtbm5uf+jNzOzupSZSFqByeksq12AmUBLH30AkLSnpBFpeQzwR8AjERHAz4HuM7xmAbc3fORmZla30hJJOo5xAbAQWAbMj4ilkuZIOgVA0pGSOoDTgSslLU3d3w60SfoNWeK4NCIeSXWfAz4jqZ3smMlVZW2DmZn1rdSbNkbEAmBBVdnFueVWsump6n6/AA7rIeZKsjPCzMxsG+Ar283MrBAnEjMzK8SJxMzMCnEiMTOzQpxIzMysECcSMzMrxInEzMwKcSIxM7NCnEjMzKwQJxIzMyvEicTMzApxIjEzs0KcSMzMrBAnEjMzK8SJxMzMCqk7kUjaX9JxaXk3SaPKG5aZmQ0VdSUSSX8B3ApcmYrGA7eVNSgzMxs66t0j+STZc9O7ACLiMeDNfXWSNF3Sckntki6qUX+0pPslbZJ0Wq78cEm/lLRU0oOSzsjVXSvpd5KWpNfhdW6DmZmVoN5H7b4cEa9IAkDScCB66yBpGHAFcDzQAbRKask9ex1gNXAe8NdV3V8Azo2IxyTtByyWtDAinkv1n42IW+scu5mZlajeRHKvpC8Au0k6HvhL4Md99JkGtKdnrCNpHjAD+O9EEhGrUt3r+Y4R8dvc8lpJTwHNwHOYmdk2pd6prYuA9cBDwMeABcAX++gzDliTW+9IZf0iaRqwC7AiV/y1NOV1uaQR/Y1pZmaNU28i2Q24OiJOj4jTgKtTWW9Uo6zX6bAtAkj7AtcDH46I7r2WzwNvA44E9gI+10Pf2ZLaJLWtX7++P29rZmb9UG8iuZvNE8duwM/66NMBTMitjwfW1jswSaOBnwBfjIj7ussj4onIvAxcQzaFtoWImBsRUyNianNzc71va2Zm/VRvItk1Ip7vXknLu/fRpxWYLOkASbsAM4GWet4stf8RcF1E3FJVt2/6V8CpwMN1boOZmZWg3kRSkXRE94qkKcCLvXWIiE3ABcBCYBkwPyKWSpoj6ZQU50hJHcDpwJWSlqbufwYcDZxX4zTfGyU9RHa8Zgzw1Tq3wczMSlDvWVufBm6R1D01tS9wRi/tAYiIBWQH5vNlF+eWW8mmvKr73QDc0EPM99U5ZjMz2wrqSiQR0SrpbcBbyQ6iPxoRr5Y6MjMzGxLq3SOB7CypSanPuyQREdeVMiozMxsy6kokkq4HDgKWAK+l4gCcSMzMdnD17pFMBQ6JiH5dB2JmZtu/es/aehgYW+ZAzMxsaKp3j2QM8IikXwMvdxdGxCmljMrMzIaMehPJl8schJmZDV31nv57b9kDMTOzoaneJyS+W1KrpOclvSLpNUldZQ/OzMy2ffUebP82cCbwGNkNGz+ayszMbAdX9wWJEdEuaVhEvAZcI+kXJY7LzMyGiHoTyQvpjrxLJH0deAIYWd6wzMxsqKh3auuc1PYCoEL2nJE/KWtQZmY2dNSbSE6NiJcioisivhIRnwFOKnNgZmY2NNSbSGbVKDuvgeMwM7MhqtdjJJLOBM4CDpSUf7rhKGBDmQMzM7Ohoa+D7b8gO7A+BvhGrnwj8GBZgzIzs6Gj16mtiHgc+H9AJSLuzb3uT4/S7ZWk6ZKWS2qXdFGN+qMl3S9pk6TTqupmSXosvWblyqdIeijF/FZ6druZmQ2SPo+RpOtGXpDU1J/AkoYBVwAnAocAZ0o6pKrZarJjLTdV9d0LuAQ4CpgGXCJpz1T9XWA2MDm9pvdnXGZm1lj1XkfyEvCQpLvITv8FICIu7KXPNKA9IlYCSJoHzAAeyfVfleper+r7AeCuiHgm1d8FTJe0CBgdEb9M5dcBpwJ31LkdZmbWYPUmkp+kV3+MA9bk1jvI9jAG2ndcenXUKDczs0FS791/v5+ubH9LKloeEa/20a3WsYt6n7DYU9+6Y0qaTTYFxsSJE+t8WzMz66967/57LNkNG68AvgP8VtLRfXTrILsCvtt4YG2d4+qpb0da7jNmRMyNiKkRMbW5ubnOtzUzs/6q94LEbwAnRMQxEXE02TGMy/vo0wpMlnRA2puZCbT00afbQuAESXumg+wnAAsj4glgY7qtvYBzgdvrjGlmZiWoN5HsHBHLu1ci4rfAzr11SKcHX0CWFJYB8yNiqaQ5kk4BkHSkpA7gdOBKSUtT32eAvyNLRq3AnO4D78AngO8B7cAKfKDdzGxQ1XuwvU3SVcD1af1sYHFfnSJiAbCgquzi3HIrm09V5dtdDVxdo7wNeEed4zYzs5LVm0g+AXwSuJDsgPd/kB0rMTOzHVy9Z229LOnbwN3A62Rnbb1S6sjMzGxIqCuRSPog8C9kxyQEHCDpYxHh4xNmZju4eqe2vgG8NyLaASQdRHaBohOJmdkOrt6ztp7qTiLJSuCpEsZjZmZDTL17JEslLQDmk11JfjrQKulPACLi30oan5mZbePqTSS7Ak8Cx6T19cBewMlkicWJxMxsB1XvWVsfLnsgZmY2NNV71tYBwP8CJuX7RMQp5QzLzMyGinqntm4DrgJ+THYdiZmZGdCPB1tFxLdKHYmZmQ1J9SaSb0q6BLgTeLm7MCLuL2VUZmY2ZNSbSA4DzgHexxtTW5HWzcxsB1ZvIvkQcKDvr2VmZtXqTSS/AfbAV7M3XGdnJ5VKpSGxRo4cSVNTU0NimZXJv/fbl3oTyT7Ao5Ja2fwYiU//LaCzs5ODJk5kQ1dXQ+LtPXo0K1av9n8q26b59377U28iuaTUUeygKpUKG7q6WAyMLRhrHTClq4tKpeL/ULZN8+/99qfeK9vvHUhwSdOBbwLDgO9FxKVV9SOA64ApwAbgjIhYJels4LO5pu8EjoiIJZIWAfsCL6a6EyJiSE+5jQX2G+xBmG1l/r3ffvSaSCRtJDs7a4sqICJidC99hwFXAMcDHWQ3eWyJiEdyzc4Hno2IgyXNBC4jSyY3AjemOIcBt0fEkly/s9Mjd83MbJD1mkgiYlSB2NOA9ohYCSBpHjADyCeSGcCX0/KtwLclKSLyyetM4OYC4zAzsxLV+zySgRgHrMmtd6Symm0iYhPQCexd1eYMtkwk10haIulLklTrzSXNltQmqW39+vUD3QYzM+tDmYmk1hd89TRZr20kHQW8EBEP5+rPjojDgPek1zm13jwi5kbE1IiY2tzc3L+Rm5lZ3cpMJB3AhNz6eGBtT20kDQeagGdy9TOp2huJiN+nfzcCN5FNoZmZ2SApM5G0ApMlHSBpF7Kk0FLVpgWYlZZPA+7pPj4iaSeyJzHO624sabikMWl5Z+Ak4GHMzGzQ1HsdSb9FxCZJFwALyU7/vToilkqaA7RFRAvZremvl9ROticyMxfiaKCj+2B9MgJYmJLIMOBnwL+WtQ1mZta30hIJQEQsABZUlV2cW36JbK+jVt9FwLuryipk15yYmdk2osypLTMz2wE4kZiZWSFOJGZmVogTiZmZFeJEYmZmhTiRmJlZIU4kZmZWiBOJmZkV4kRiZmaFOJGYmVkhTiRmZlaIE4mZmRXiRGJmZoU4kZiZWSFOJGZmVkipzyOx7VtnZyeVSqUhsUaOHElTU1NDYpnZ1uVEYgPS2dnJQRMnsqGrqyHx9h49mhWrVzuZmA1BpSYSSdOBb5I9Fvd7EXFpVf0I4Dqypx5uAM6IiFWSJgHLgOWp6X0R8fHUZwpwLbAb2dMXP9X9nHfbeiqVChu6ulgMjC0Yax0wpauLSqXiRGI2BJWWSCQNA64Ajgc6gFZJLRHxSK7Z+cCzEXGwpJnAZcAZqW5FRBxeI/R3gdnAfWSJZDpwR0mbYX0YC+w32IMws0FV5sH2aUB7RKyMiFeAecCMqjYzgO+n5VuB90tSTwEl7QuMjohfpr2Q64BTGz90MzOrV5mJZBywJrfekcpqtomITUAnsHeqO0DSA5LulfSeXPuOPmICIGm2pDZJbevXry+2JWZm1qMyE0mtPYvqYxk9tXkCmBgR7wI+A9wkaXSdMbPCiLkRMTUipjY3N/dj2GZm1h9lJpIOYEJufTywtqc2koYDTcAzEfFyRGwAiIjFwArgLan9+D5impnZVlRmImkFJks6QNIuwEygpapNCzArLZ8G3BMRIak5HaxH0oHAZGBlRDwBbJT07nQs5Vzg9hK3wczM+lDaWVsRsUnSBcBCstN/r46IpZLmAG0R0QJcBVwvqR14hizZABwNzJG0CXgN+HhEPJPqPsEbp//egc/YMjMbVKVeRxIRC8hO0c2XXZxbfgk4vUa/HwI/7CFmG/COxo7UzMwGyvfaMjOzQpxIzMysECcSMzMrxInEzMwK8d1/bZvl29TbQPl3Z+tyIrFtkm9TbwPl352tz4nEtkm+Tb0NlH93tj4nEtum+Tb1NlD+3dl6fLDdzMwKcSIxM7NCnEjMzKwQJxIzMyvEicTMzApxIjEzs0KcSMzMrBAnEjMzK8SJxMzMCik1kUiaLmm5pHZJF9WoHyHpB6n+V5ImpfLjJS2W9FD69325PotSzCXp9eYyt8HMzHpX2i1SJA0DrgCOBzqAVkktEfFIrtn5wLMRcbCkmcBlwBnA08DJEbFW0jvInvs+Ltfv7PTIXTMzG2Rl7pFMA9ojYmVEvALMA2ZUtZkBfD8t3wq8X5Ii4oGIWJvKlwK7ShpR4ljNzGyAykwk44A1ufUONt+r2KxNRGwCOoG9q9r8KfBARLycK7smTWt9SZJqvbmk2ZLaJLWtX7++yHaYmVkvykwktb7goz9tJB1KNt31sVz92RFxGPCe9Dqn1ptHxNyImBoRU5ubm/s1cDMzq1+ZiaQDmJBbHw+s7amNpOFAE/BMWh8P/Ag4NyJWdHeIiN+nfzcCN5FNoZmZ2SApM5G0ApMlHSBpF2Am0FLVpgWYlZZPA+6JiJC0B/AT4PMR8V/djSUNlzQmLe8MnAQ8XOI2mJlZH0pLJOmYxwVkZ1wtA+ZHxFJJcySdkppdBewtqR34DNB9ivAFwMHAl6pO8x0BLJT0ILAE+D3wr2Vtg5mZ9a3UJyRGxAJgQVXZxbnll4DTa/T7KvDVHsJOaeQYzcysGF/ZbmZmhTiRmJlZIaVObZnZ0NTZ2UmlUmlIrJEjR9LU1NSQWLZtciIxs810dnZy0MSJbOjqaki8vUePZsXq1U4m2zEnEjPbTKVSYUNXF4uBsQVjrQOmdHVRqVScSLZjTiRmVtNYYL/BHoQNCT7YbmZmhTiRmJlZIU4kZmZWiI+RbOd8GqeZlc2JZDvm0zjNbGtwItmO+TROM9sanEh2AD6N08zK5IPtZmZWiBOJmZkV4kRiZmaFOJGYmVkhpSYSSdMlLZfULumiGvUjJP0g1f9K0qRc3edT+XJJH6g3ppmZbV2lJRJJw4ArgBOBQ4AzJR1S1ex84NmIOBi4HLgs9T0EmAkcCkwHviNpWJ0xzcxsKyrz9N9pQHtErASQNA+YATySazMD+HJavhX4tiSl8nkR8TLwO0ntKR51xGyojRs3snHjxobEGjVqFKNGjdqifF0DYvcWo8z4Q3nsQ/1n24jYZccfrJ/tUI6/NX4vG00RUU5g6TRgekR8NK2fAxwVERfk2jyc2nSk9RXAUWTJ5b6IuCGVXwXckbr1GjMXezYwO62+FVje8I3c3Bjg6SEY2/EHL7bjD278oTz2rREfYP+IaO6rUZl7JKpRVp21emrTU3mtqbiamTAi5gJzextgI0lqi4ipQy224w9ebMcf3PhDeexbI35/lHmwvQOYkFsfD6ztqY2k4UAT8EwvfeuJaWZmW1GZiaQVmCzpAEm7kB08b6lq0wLMSsunAfdENtfWAsxMZ3UdAEwGfl1nTDMz24pKm9qKiE2SLgAWAsOAqyNiqaQ5QFtEtABXAdeng+nPkCUGUrv5ZAfRNwGfjIjXAGrFLGsb+qnMabSyp+gcf3BiO/7gxh/KY98a8etW2sF2MzPbMfjKdjMzK8SJxMzMCnEiMTOzQvxgqwGQ9DayK+rHkV3HshZoiYhlgzqwOqXxjwN+FRHP58qnR8RPGxB/GhAR0ZpuYTMdeDQiFhSNXeO9rouIcxsdN8X+n2R3VHg4Iu5sQLyjgGUR0SVpN+Ai4Aiyk0r+PiI6C8a/EPhRRKwpOtYasbvPklwbET+TdBbwh8AyYG5EvNqA9zgI+BDZKf6bgMeAm4t+LlY+H2zvJ0mfA84E5pFd1wLZ9SwzyW7rcmmJ7/3hiLimYIwLgU+SfQEcDnwqIm5PdfdHxBEF419Cdi+04cBdZHcqWAQcByyMiK8ViF19qreA9wL3AETEKQONneL/OiKmpeW/IPucfgScAPy46M9W0lLgf6QzGucCL5DdGuj9qfxPCsbvBCrACuBm4JaIWF8kZi72jWQ/092B54A3Af9GNnZFxKxeutcT/0LgZOBe4I+BJcCzZInlLyNiUZH4VrKI8KsfL+C3wM41yncBHiv5vVc3IMZDwJvS8iSgjSyZADzQoPjDyL5wuoDRqXw34MGCse8HbgCOBY5J/z6Rlo9pwNgfyC23As1peSTwUAPiL8tvS1XdkkaMn2y6+gSyU+vXAz8lu1ZrVMHYD6Z/hwNPAsPSuor+XPO/N2l5d2BRWp7YoN/LJuBS4FFgQ3otS2V7FI3fx3vf0YAYo4F/AK4Hzqqq+06Z46/n5amt/nud7BHoj1eV75vqCpH0YE9VwD5F45P9Z30eICJWSToWuFXS/tS+NU1/bYrsmp8XJK2IiK70Xi9KKvr5TAU+Bfwt8NmIWCLpxYi4t2DcbjtJ2pPsy1iR/pqPiIqkTQ2I/3Bur/I3kqZGRJuktwCFp4bIphNfB+4E7pS0M9ne4ZnAPwJ93jOpFzul6a2RZF/03XehGAHsXGjUbxgOvJZijgKIiNVpO4qaT7bnemxErAOQNJafL3O+AAAEdklEQVQsyd4CHF8kuKSe9uRFtudf1DVkU30/BD4i6U/JEsrLwLsbEL8QJ5L++zRwt6THgO656InAwcAWN48cgH2AD5Dt1ucJ+EUD4q+TdHhELAGIiOclnQRcDRzWgPivSNo9Il4ApnQXSmqiYKJNX5KXS7ol/fskjf0dbgIWk33WIWlsRKyT9CYak2Q/CnxT0hfJbrb3S0lryH6PPtqA+JuNMbLjFi1ASzomU8RVZH/NDyNL5LdIWkn2JTavYGyA7wGtku4DjuaNR0o0kyWsoiZFxGX5gpRQLpP0kQbEbyWblqv1e7JHA+IfFBF/mpZvk/S3wD2SCk3nNoqPkQyApJ3IDsKOI/vF6QBa01/iRWNfBVwTEf9Zo+6miDirYPzxZHsNW9zBWtIfRcR/FYw/Iv2VVF0+Btg3Ih4qEr8q5geBP4qILzQqZg/vszuwT0T8rkHxRgEHkiXBjoh4skFx3xIRv21ErB7i7wcQEWsl7UF23Gt1RPy6QfEPBd5OdnLDo42ImYt9J/Az4Pvdn7ekfYDzgOMj4riC8R8GPhQRj9WoWxMRE2p060/8ZcCh6Y+p7rJZwN+QTVXvXyR+UU4kZrbdS1OWF5GdbfnmVPwk2R7bpRFRPQPQ3/inkR1H2+JxFZJOjYjbCsb/OnBnRPysqnw68M8RMblI/KKcSMxsh9aIsyG35/h1jcGJxMx2ZJJWR8RExx84H2w3s+1e2WdDDvX4RTmRmNmOoOyzIYd6/EKcSMxsR/DvZGc3LamukLTI8YvxMRIzMyvEd/81M7NCnEjMzKwQJxKzfpC0SNIHqso+Lek7/YixIF0Z3lub53sovzZd/Ga2zXAiMeufm8keGZA3M5X3SpmdIuKPI+K5UkZnNgicSMz651bgJEkjACRNIrsb9BJJd0u6X9JDkmZ010talvZY7gcmSFqV7j2GpNskLZa0VNLs/BtJ+kaKd3e6eSFV9VMk3Zv6L5S0b6lbbtYDJxKzfoiIDcCvyZ76CNneyA+AF8lu2ncE2cO2viGp+06wbwWui4h3RUT14wc+EhFTyG6Rf6GkvVP5SLJnlhxBdlfZS/Kd0q3V/xk4LfW/GhjwQ8PMivB1JGb91z29dXv69yNkF4b9vaSjyW6XP443rjh+PCLu6yHWhZI+lJYnAJPJHrr0OlmCguxhXv9W1e+twDuAu1K+Gkb2kC+zrc6JxKz/bgP+b3qY0W4Rcb+k88geHDUlIl6VtArYNbWv1AqSHip2HPAHEfFCurBs11ptgeoLvgQsjYg/KLIhZo3gqS2zfkpPmFxENp3UfZC9CXgqJZH3AvU8H6IJeDYlkbex+ZPudgK6z846C6h+Ps1yoFnSH0A21ZWe52G21XmPxGxgbiabbuo+g+tG4MeS2oAlZE8T7MtPgY+nG/ItB/LTXxXgUEmLgU7gjHzHiHglnQb8rfT0yeHAPwFLB75JZgPjW6SYmVkhntoyM7NCnEjMzKwQJxIzMyvEicTMzApxIjEzs0KcSMzMrBAnEjMzK8SJxMzMCvn/Cgzt9bGc1hMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# list of x locations for plotting\n",
    "x_values = list(range(len(importances)))\n",
    "\n",
    "# Make a bar chart\n",
    "plt.bar(x_values, importances, orientation = 'vertical', color = 'r', edgecolor = 'k', linewidth = 1.2)\n",
    "\n",
    "# Tick labels for x axis\n",
    "plt.xticks(x_values, feature_list, rotation='vertical')\n",
    "\n",
    "# Axis labels and title\n",
    "plt.ylabel('Importance'); plt.xlabel('Variable'); plt.title('Variable Importances');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEZCAYAAAB4hzlwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XeYFFXWx/HvIWdQQFTCYADMcRRRRBBUMCBmxIS7iq+YdVFZEJFg1lUUA2ZURIyLLgiSURAZEAMgikgWQZQgIDDMef+oYmyGCT2hpyb8Ps8zz1RX3bp1ugfq9L1Vda+5OyIiIgBlog5ARESKDiUFERFJp6QgIiLplBRERCSdkoKIiKRTUhARkXRKClLsmFlfM3sjH/vPNbPWBRiSSImhpCBxM7MuZpZiZn+a2S9mNtrMWkYdV3bM7FUzGxC7zt0PdfdJBXycxmbmZlauIOvNqzCWA6OOQ4ofJQWJi5ndDjwB3A/UAxoBzwDnRhmX7KqoJCUpvpQUJEdmVhPoB9zg7u+7+yZ33+7uH7l7j7DMLt/Izay1mS2Peb3YzHqY2TdmtsnMXjKzemFrY6OZjTOzPTLbN2b/dlnE946ZrTKz9WY2xcwODdd3Ay4D7gxbNx/F1mVm+5rZFjPbM6auo83sNzMrH77+h5nNN7M/zGyMmSXF+Zm9ambPhO/vTzP73Mz2NrMnwrq+N7OjM7y/nmY2L9z+iplVitl+rZktNLPfzWykme0bs83N7AYz+xH40cymhJu+Do99iZntYWYfm9masP6PzaxBTB2TzKx/GOdGMxtrZnVitrc0s2lmts7MlplZ13B9RTN71MyWmtmvZvacmVUOt9UJj7MujHuqmemcU8TpDyTxaAFUAj7IZz0XAKcBTYFzgNHAv4E6BP8Wb85jvaOBJsBewGzgTQB3HxIuP+zu1dz9nNid3H0lMD2Ma6cuwLvuvt3MOoXxnQ/UBaYCb+UirouB3gTvb2t4rNnh63eBxzOUvww4AziA4DPqDWBmpwIPhPXtAywBhmfYtxPQHDjE3VuF644M3/fbBJ/vK0ASQStvC/B0hjq6AFcTfI4VgH+Fx29E8Bk/FX4ORwFzwn0eCmM9CjgQqA/0CbfdASwP96lH8FlqXJ0iTklB4lEb+M3dU/NZz1Pu/qu7ryA4wc5w96/cfStBwjk6+90z5+4vu/vGsJ6+wJFh6yYew4BLAczMgM7hOoDrgAfcfX743u8Hjoq3tQB84O6z3P0vgvf3l7sPdfcdwNvs/n6fdvdl7v47MHBnXATJ4mV3nx2+x55ACzNrHLPvA+7+u7tvySwQd1/r7u+5+2Z33xjWf0qGYq+4+w9hHSMITvQ7jz/O3d8KW4hr3X1O+HldC9wWHntj+Bl1DvfbTpDEksL9proGWyvylBQkHmuBOgXQX/1rzPKWTF5Xy22FZlbWzB40s5/MbAOwONxUJ5vdYr1LcILdF2hF8E12argtCXgy7P5YB/wOGMG34Xjk9v0ui1leAuzsIto3fA2Au/9J8DeJjSN2392YWRUze97MloSf0xSglpmVjSm2KmZ5c0x8DYGfMqm2LlAFmBXzGX0Srgd4BFgIjDWzRWZ2d3YxStGgpCDxmA78RdBFkZVNBCeInfbOx/F2qSs8cdXNomwXgovd7YCaQOOdu4W/s/1m6u7rgLEEXTNdgLdivs0uA65z91oxP5XdfVru31JcGsYsNwJWhssrCRIUAGZWlaD1tiL2reRQ9x1AM6C5u9cgSIDw9+eUnWUEXVoZ/UaQ3A6N+Xxquns1gLD1doe770/QXXi7mbWN43gSISUFyZG7ryfoJx5sZp3Cb53lzayDmT0cFpsDnGlme5rZ3sCt+TjkD0AlMzsrvODbG6iYRdnqBP31awkSyf0Ztv8K7J/D8YYBVxJcWxgWs/45oGfMheuaZnZRbt5ILt1gZg3CC9//Juhi2hnf1WZ2lJlVJHiPM9x9cTZ1ZXzf1QlO4OvC+u/NRVxvAu3M7GIzK2dmtc3sKHdPA14A/mNmewGYWX0zOyNcPtvMDgy7mTYAO8IfKcKUFCQu7v44cDvBCXoNwbfHG4EPwyKvA18TdN+M5e8TWl6OtR7oDrxI8G14E8EFy8wMJehaWQHMA77IsP0l4JCwe+PDjDuHRhJcqP7V3b+OieMDggupw8Mul++ADnl6U/EZRvDZLQp/BoRxjAfuAd4DfiH41t45izp26gu8Fr7viwluJ65M8O3+C4Junri4+1LgTILWxu8EXwCODDffRdBF9EX4GY0jaJFA8JmOA/4kaG0+U9DPh0jBM133EYmemS0GrnH3cVHHIqWbWgoiIpJOSUFERNKp+0hERNKppSAiIumUFEREJF2xG1GxTp063rhx46jDEBEpVmbNmvWbu2f1EGi6YpcUGjduTEpKStRhiIgUK2a2JOdS6j4SEZEYSgoiIpJOSUFERNIpKYiISLqEJgUza29mC8JpBHcbS93MksxsvAVTNE6KnR5QREQKX8KSQjgG/mCCUSUPAS41s0MyFHsUGOruRxDMAfxAouIREZGcJfKW1OOBhe6+CMDMhhNMhjIvpswhwG3h8kT+HoY5MVq33n3dxRdD9+6weTOceebu27t2DX5++w0uvHD37ddfD5dcAsuWwRVX7L79jjvgnHNgwQK47rrdt/fuDe3awZw5cGsmUxDcfz+ceCJMmwb//vfu2594Ao46CsaNgwEDdt/+/PPQrBl89BE89tju219/HRo2hLffhmef3X37u+9CnTrw6qvBT0ajRkGVKvDMMzBixO7bJ00Kfj/6KHz88a7bKleG0aOD5f79Yfz4XbfXrg3vvRcs9+wJ06fvur1BA3jjjWD51luDzzBW06YwZEiw3K0b/PDDrtuPOir4/AAuvxyWZxidu0ULeCD8nnLBBbB27a7b27aFe+4Jljt0gC0ZZsI8+2z417+CZf3b2327/u0Fy7n5t7fzPSVQIruP6rPrFIHL2X0aw6/5e9L084DqZlY7Y0Vm1s3MUswsZc2aNQkJVkREEjggXjhD1Rnufk34+grgeHe/KabMvsDTwH4Ec8ZeQDC13/qs6k1OTnY9vCYikjtmNsvdk3Mql8juo+XsOudsA/6ecxYAd18JnA9gZtWAC7JLCCIikliJ7D6aCTQxs/3MrALB9IEjYwuYWR0z2xlDT+DlBMYjIiI5SFhScPdUgjl8xwDzgRHuPtfM+plZx7BYa2CBmf0A1AMGJioeERHJWbGbZEfXFEREci/eawp6ollERNIpKYiISDolBRERSaekICIi6ZQUREQknZKCiIikU1IQEZF0SgoiIpJOSUFERNIpKYiISLpEjpIqIiXI1tStLNuwjCXrlrBx28aowymyOh3UKeoQ8kVJQUQA2LB1A0vWLWHJ+iV//w6Xl65fyi9//hJ1iMWC31u8xpPLSElBpBRwd1ZvWr3rCT/mxL90/VLW/bVul30qlK1Ao5qNSKqZRPsD25NUM4mkWkkk1UyiVqVamFlE70YSSUlBpATZsn0LM1bMYPqy6fz0x0/pJ/yl65fyV+pfu5StUbFG+on+5EYnpyeAnSf+etXqUcZ02bG0UVIQKcbW/7WeacumMWXJFKYsncLMFTPZnrYdgHpV65FUK4kj6x1Jx6Ydg5N+eMJPqhV82xfJSElBpBhZs2kNU5dOZcqSKUxdOpU5q+aQ5mmUK1OO5H2Tue2E22iV1IqTGp2kk77kiZKCSBG2bP2yoBUQJoH5v80HoHK5ypzQ4ATuaXUPrZJa0bx+c6pWqBpxtFISKCmIFBHuzo+//7hLEli8bjEANSvW5KRGJ3HVkVfRKqkVx+57LBXKVog2YCmRlBREIrIjbQffrf4u/XrA1CVT+XXTrwDsVXUvTm50cnp30OF7HU7ZMmUjjlhKg4QmBTNrDzwJlAVedPcHM2xvBLwG1ArL3O3uoxIZk0jUFv2xiIc+e4gR80ak3waaVDOJ0w84nZMbnUyrpFY0rd1Ut3xKJBKWFMysLDAYOA1YDsw0s5HuPi+mWG9ghLs/a2aHAKOAxomKSSRK89fM54HPHmDYt8MoV6YcnQ/rTLv923Fyo5NJqpUUdXgiQGJbCscDC919EYCZDQfOBWKTggM1wuWawMoExiMSiTmr5jBw6kDem/celctX5tYTbuWOFnewT/V9og5NZDeJTAr1gWUxr5cDzTOU6QuMNbObgKpAuwTGI1Kopi+bzsCpA/nfj/+jRsUa9Dq5F7eccAt1qtSJOjSRLCUyKWTWIZpxUJBLgVfd/TEzawG8bmaHuXvaLhWZdQO6ATRq1CghwYoUBHdn4uKJDJw6kAk/T6B25doMPHUgNxx3AzUr1Yw6PJEcJTIpLAcaxrxuwO7dQ/8E2gO4+3QzqwTUAVbHFnL3IcAQgOTk5OI92pSUSO7OqB9HMXDqQKYvn84+1fbhsdMf47pjr9PzA1KsJDIpzASamNl+wAqgM9AlQ5mlQFvgVTM7GKgErElgTCIFKs3TeH/++wycOpA5q+aQVDOJZ896lq5HdaVSuUpRhyeSawlLCu6eamY3AmMIbjd92d3nmlk/IMXdRwJ3AC+Y2W0EXUtd3V0tASnyUtNSeevbt7j/s/v5/rfvaVq7Ka+c+wqXHX4Z5cuWjzo8kTxL6HMK4TMHozKs6xOzPA84KZExiBSkralbee3r13jo84dY9Mcijqh3BMMvGM6Fh1yoh8ukRNATzSJx2Lx9My/MeoFHpj3Cio0rOL7+8TxxxhOc3fRsPWQmJYqSgkg2NmzdwDMzn+Hx6Y+zZvMaTkk6hVfOfYV2+7dTMpASSUlBJBNrN69l0IxBDPpyEOv+Wkf7A9vT6+RetGzUMurQRBJKSUEkxqo/V/H49Md5ZuYzbNq+iU4HdaLXyb1I3jc56tBECoWSggiwdP1SHvn8EV786kW27dhG58M607NlTw7b67CoQxMpVEoKUqot/H0hD372IEO/HorjXHXkVdx10l00qd0k6tBEIqGkIKXS3NVzuf+z+xn+3XDKlynPdcdeR4+TetCopoZRkdJNSUFKlVkrZzFw6kA++P4Dqpavyu0n3M7tLW7XiKUiobiSgpm1BJq4+ytmVheo5u4/JzY0kYLz+dLPGTB1AJ8s/ISaFWtyT6t7uKX5LdSuUjvq0ESKlByTgpndCyQDzYBXgPLAG+hJZCni3J3xP49nwJQBTF4ymTpV6nD/qffT/bjuGrFUJAvxtBTOA44GZgO4+0ozq57QqETywd35+IePGTB1AF+u+JJ9q+/Lf874D9cec61GLBXJQTxJYZu7u5k5gJnpf5UUSTvSdvDe/PcYOHUg3/z6DY1rNea5s56j61FdqViuYtThiRQL8SSFEWb2PFDLzK4F/gG8kNiwROK3fcd2hn07jAc+e4AFaxfQrHYzXuv0GpcedqlGLBXJpRyTgrs/amanARsIriv0cfdPEx6ZSA627djGy1+9zEOfP8TidYs5st6RjLhwBOcffL5GLBXJo3guNO8HTN2ZCMysspk1dvfFiQ5OJCvjF43nxtE38v1v39O8fnOe6vAUZzU5S4PUieRTPN1H7wAnxrzeEa47LiERiWRjxYYV3DH2Dt6e+zb777E/H136kZKBSAGKJymUc/dtO1+4+zYzq5DAmER2s33HdgbNGETfyX1JTUvlvtb3cedJd2rKS5ECFk9SWGNmHcPpMzGzc4HfEhuWyN8mLZ7EDaNuYN6aeZzV5CwGdRjE/nvsH3VYIiVSPEnh/4A3zexpwIBlwJUJjUoE+GXjL/T4tAdvfvsmjWs1ZmTnkZzT7JyowxIp0eK5++gn4AQzqwaYu29MfFhSmqWmpTL4y8H0mdSHv1L/4p5W93B3y7upUr5K1KGJlHjx3H1UEbgAaAyU23lBz937xbFve+BJoCzwors/mGH7f4A24csqwF7uXisX8UsJ89nSz+j+v+58u/pbzjjgDJ7q8JSGsRYpRPF0H/0XWA/MArbGW7GZlQUGA6cBy4GZZjbS3eftLOPut8WUv4lgOA0phX7981fuGncXr339Gg1rNOT9i9+n00GddFeRSCGLJyk0cPf2eaj7eGChuy8CMLPhwLnAvCzKXwrcm4fjSDGWmpbKcynP0XtCbzZv30zPlj3pdXIvjVEkEpF4ksI0Mzvc3b/NZd31CS5K77QcaJ5ZQTNLAvYDJmSxvRvQDaBRI02CUlJMXzad7qO6M2fVHNrt346nOzxNszrNog5LpFSLJym0BLqa2c8E3UcGuLsfkcN+mbX7PYuynYF33X1HZhvdfQgwBCA5OTmrOqSYWLNpDXePu5uX57xM/er1GXHhCC485EJ1FYkUAfEkhQ55rHs50DDmdQNgZRZlOwM35PE4UkzsSNvBC7Nf4N/j/83GbRvpcWIP+pzSh2oVqkUdmoiE4rkldQmAme0F5Obx0ZlAk3DspBUEJ/4uGQuZWTNgD2B6LuqWYmbmipl0H9WdlJUptG7cmsFnDuaQuodEHZaIZFAmpwJm1tHMfgR+BiYDi4HROe3n7qnAjcAYYD4wwt3nmlk/M+sYU/RSYLi7q1uoBFq7eS3XfXQdzV9szooNKxh2/jAmXDlBCUGkiIqn+6g/cAIwzt2PNrM2BCfyHLn7KGBUhnV9MrzuG1+oUtws/H0hbYe2ZcWGFdx6wq30bd2XGhVrRB2WiGQjnqSw3d3XmlkZMyvj7hPN7KGERybF2vw182k7tC3bdmxj+j+nc1x9DaorUhzEkxTWhUNcTCEYA2k1kJrYsKQ4+3rV15z2+mmUsTJM7jqZQ/c6NOqQRCROOV5TIHjgbDNwG/AJ8BNwdiKDkuIrZWUKbV5rQ4WyFZhy9RQlBJFiJp6k0Mfd09w91d1fc/dBwF2JDkyKn2nLptF2aFtqVqrJlKun0LR206hDEpFciicpnJbJurw+uyAl1KTFkzj99dOpV7UeU7pO0XwHIsVUltcUzOx6oDtwgJl9E7OpOvB5ogOT4mPMwjF0ersT+++xP+OuGMc+1feJOiQRyaPsLjQPI3ge4QHg7pj1G93994RGJcXGyAUjueidizi4zsF8esWn1K1aN+qQRCQfskwK7r7ezDYCh+98qlkk1jtz36HL+104eu+jGXP5GPaovEfUIYlIPmV7TcHd04CvzUxDk8ou3vjmDTq/15nm9Zsz7spxSggiJUQ8zynsA8w1sy+BTTtXunvHrHeRkuzF2S/S7aNutNmvDf/t/F8NaCdSgsSTFO5LeBRSbDz95dPcNPomOhzYgfcufo/K5StHHZKIFKAcb0l198nA9wR3HVUH5ofrpJR55PNHuGn0TZzb7Fw+uOQDJQSREiieUVIvBr4ELgIuBmaY2YWJDkyKDnen/+T+3DnuTi459BLeuegdKparGHVYIpIA8XQf9QKOc/fVAGZWFxgHvJvIwKRocHd6TejFA589wFVHXsVLHV+ibJmyUYclIgkST1IoszMhhNYS35PQUsy5O7eNuY0nZzzJdcdexzNnPUMZ059epCSLJyl8YmZjgLfC15eQYY4EKXnSPI3u/+vO87Oe55bmt/CfM/6jOZRFSoF4puPsYWbnAy0BA4a4+wcJj0wisyNtB9d8dA2vznmVni17MvDUgUoIIqVEPC0FgGnADiCNYO5lKaG279jOFR9cwdtz36Zf6370btVbCUGkFInn7qNrCO4+Og+4EPjCzP6R6MCk8G1N3crF717M23Pf5uF2D3PPKfcoIYiUMvFcNewBHO3uXd39KuBY4pxPwczam9kCM1toZndnUeZiM5tnZnPNbFj8oUtB2rJ9C+e9fR4ffv8hT3V4ih4n9Yg6JBGJQDzdR8uBjTGvNwLLctrJzMoCgwnmY1gOzDSzke4+L6ZME6AncJK7/2Fme+UmeCkYm7ZtouPwjkz8eSIvnPMC1xxzTdQhiUhE4kkKKwgeWPsv4ATTc35pZrcDuPvjWex3PLDQ3RcBmNnwcN95MWWuBQa7+x9hXat3q0USasPWDZw17CymLZvG0POGcvkRl0cdkohEKJ6k8FP4s9N/w9/Vc9ivPru2KJYDzTOUaQpgZp8DZYG+7v5JxorMrBvQDaBRIw3YWlD+2PIH7d9sz+xfZjP8guFcdOhFUYckIhGL55bUvA6Il9kVSs/k+E2A1kADYKqZHebu6zLEMAQYApCcnJyxDsmDNZvWcPobpzNvzTzev/h9zml2TtQhiUgRkGNSMLNkgqEukmLLu/sROey6HGgY87oBsDKTMl+4+3bgZzNbQJAkdNtrAv2y8Rfavd6ORX8sYmTnkZxx4BlRhyQiRUQ83UdvEtyB9C3Bcwrxmgk0MbP9CK5LdAa6ZCjzIXAp8KqZ1SHoTlqUi2NILi1bv4y2Q9uycuNKRl82mtaNW0cdkogUIfEkhTXuPjK3Fbt7qpndCIwhuF7wsrvPNbN+QEpY5xjgdDObR/BwXA93X5vbY0l8fv7jZ04deiq/b/mdsVeM5cSGJ0YdkogUMeaefRe9mbUl+DY/Hti6c727v5/Y0DKXnJzsKSkpURy6WPth7Q+0HdqWTds2MfaKsSTvmxx1SCJSiMxslrvn+B8/npbC1cBBQHn+7j5yIJKkILk3d/Vc2g5tS5qnManrJI6ol9PlIBEpreJJCke6++EJj0QSYs6qOZz2+mmUL1OeiVdN5OC6B0cdkogUYfEMc/GFmR2S8EikwH254kvavNaGKuWrMOXqKUoIIpKjeFoKLYGrzOxngmsKBngct6RKhD5b+hlnvnkmdavWZcKVE0iqlRR1SCJSDMSTFNonPAopUOMXjafj8I40rNGQ8VeOp36N+lGHJCLFRJZJwcz2DBc3ZlVGip5RP47i/LfPp2ntpnx6xafUq1Yv6pBEpBjJrqUwi+Auo6yGq9g/IRFJnn0w/wMuefcSDq93OGMvH0vtKrWjDklEipksk4K771eYgUj+DP9uOJe/fznH1T+O0ZeNplalWlGHJCLFUDx3H0kR9+qcV7ns/cs4qdFJjL18rBKCiOSZkkIx91zKc1z936tpu19bRl82muoVcxrRXEQka0oKxdgTXzzB9f+7nrObns3IS0dSpXyVqEMSkWIurqRgZi3N7OpwuW448qlE6IGpD3DbmNu44OALeO/i96hUrlLUIYlICZBjUjCze4G7COZShmAMpDcSGZRkzd3pM7EP/57wby47/DKGXzicCmUrRB2WiJQQ8bQUzgM6ApsA3H0lOU/FKQng7tz56Z30n9Kffx79T17r9BrlysTz/KGISHziOaNsc3c3Mwcws6oJjkkykeZp3Dz6ZgbPHMwNx93AoA6DKGO6JCQiBSues8oIM3seqGVm1wLjgBcSG5bE2pG2g24fdWPwzMH8q8W/eKrDU0oIIpIQObYU3P1RMzsN2AA0A/q4+6cJj0zS9Rzfk5e+eol7Wt3Dfa3vwyyzh8xFRPIvx6RgZrcB7ygRRGPJuiU88cUTdD2qK/3a9Is6HBEp4eLpg6gBjDGzqWZ2g5lphLVC1H9Kf8yMfq2VEEQk8XJMCu5+n7sfCtwA7AtMNrNx8VRuZu3NbIGZLTSzuzPZ3tXM1pjZnPDnmly/gxLsx7U/8uqcV7k++Xoa1mwYdTgiUgrk5n7G1cAqYC2wV06FzawsMBg4DVgOzDSzke4+L0PRt939xlzEUWr0ndyXiuUq0rNlz5wLi4gUgHgeXrvezCYB44E6wLVxzrp2PLDQ3Re5+zZgOHBufoItTb5b/R1vffsWNx9/s+ZEEJFCE09LIQm41d3n5LLu+sCymNfLgeaZlLvAzFoBPwC3ufuyjAXMrBvQDaBRo0a5DKN46jOxD9UrVqfHST2iDkVESpEsWwpmViNcfBhYamZ7xv7EUXdWk/PE+ghoHLY8xgGvZVaRuw9x92R3T65bt24chy7eUlam8MH3H3BHizvYs3I8H7WISMHIrqUwDDibzGdgi2fmteVA7NXRBsDK2ALuvjbm5QvAQznUWSr0ntCb2pVrc+sJt0YdioiUMtnNvHZ2+DuvI6LOBJqEI6quADoDXWILmNk+7v5L+LIjMD+Pxyoxpi6ZypifxvDIaY9Qo2KNnHcQESlA8VxoHh/PuozcPRW4ERhDcLIf4e5zzayfmXUMi91sZnPN7GvgZqBrboIvadyd3hN7s3e1vel+XPeowxGRUijLloKZVQKqAHXMbA/+7j6qQfC8Qo7cfRQwKsO6PjHLPfl7SO5S79NFnzJlyRSe7vC0JswRkUhkd03hOuBWggQwi7+TwgaC5w+kALk7vSf0JqlmEtcco2f4RCQa2V1TeBJ40sxucvenCjGmUmnkgpHMXDmTlzq+RMVyFaMOR0RKqXhGSX3KzA4DDgEqxawfmsjASpM0T+OeiffQtHZTrjzyyqjDEZFSLJ5RUu8FWhMkhVFAB+AzQEmhgIyYO4JvV3/LWxe8pZnURCRS8YySeiHQFljl7lcDRwLq3yggqWmp3DvpXg7f63AuPvTiqMMRkVIunq+lW9w9zcxSw6ecV5Pzg2sSp6FfD+WHtT/w4SUfajY1EYlcPEkhxcxqETxxPAv4E/gyoVGVEltTt3Lf5Ps4bt/j6NisY847iIgkWDwXmnc+RfWcmX0C1HD3bxIbVunw4uwXWbp+KS+c84Km2BSRIiG7h9eOyW6bu89OTEilw+btmxkwdQCtklpx2v6nRR2OiAiQfUvhsWy2OXBqAcdSqjwz8xlW/bmKEReOUCtBRIqM7B5ea1OYgZQmG7Zu4MHPHuSMA87g5KSTow5HRCRdPM8pZPo0lR5ey7snvniCtVvWMuDUAVGHIiKyi3juPjouZrkSwTMLs9HDa3ny+5bfeWz6Y3Q6qBPJ+yZHHY6IyC7iufvoptjXZlYTeD1hEZVwj3z+CBu3bqR/m/5RhyIispu8PC21GWhS0IGUBqv+XMWgLwdx6eGXctheh0UdjojIbuK5pvARf8+tXIZgDKQRiQyqpHrwswfZmrqVvqf0jToUEZFMxXNN4dGY5VRgibsvT1A8Jday9ct4NuVZuh7VlSa11dASkaIpnmsKkwHCcY/Khct7uvvvCY6tROk/JbiG0OeUPjmUFBGJTjzdR92A/sAWII1gBjZHg+LFbeHvC3n5q5e5Pvl6GtVsFHU4IiJZiudCcw/gUHdv7O77u/t+7h5XQjCz9maaAfBOAAARIUlEQVS2wMwWmtnd2ZS70MzczErkPZr3Tb6PCmUr0KtVr6hDERHJVjxJ4SeCO45yxczKEszl3IHg4vSlZnZIJuWqAzcDM3J7jOJg7uq5vPnNm9x0/E3sXW3vqMMREclWPBeaewLTzGwGsHXnSne/OYf9jgcWuvsiADMbDpwLzMtQrj/wMPCveIMuTu6ddC/VKlTjzpPujDoUEZEcxdNSeB6YAHxBMJ/Czp+c1AeWxbxeHq5LZ2ZHAw3d/eO4oi1mZv8ym/fmv8ftLW6ndpXaUYcjIpKjeFoKqe5+ex7qzmzoT0/faFYG+A/QNceKgovd3QAaNSo+F2p7T+jNnpX35LYTbos6FBGRuMTTUphoZt3MbB8z23PnTxz7LQcaxrxuAKyMeV0dOAyYZGaLgROAkZldbHb3Ie6e7O7JdevWjePQ0ft86eeMXjiaO0+8k5qVakYdjohIXOJpKXQJf/eMWRfPLakzgSZmth+wAugcUxfuvh6os/O1mU0C/uXuKXHEVKS5O70n9qZe1XrcePyNUYcjIhK3eB5e2y8vFbt7qpndCIwBygIvu/tcM+sHpLj7yLzUWxyM/3k8kxZPYlD7QVStUDXqcERE4mbunn2BIjafQnJysqekFN3GhLvT4qUWrNy4kh9v+pGK5SpGHZKICGY2y91zfBZM8ykUsI9/+JgZK2bwwjkvKCGISLGj+RQKUJqn0Xtibw7c80CuOvKqqMMREcm1eFoKGWk+hSy8O+9dvvn1G9447w3Kly0fdTgiIrmm+RQKSGpaKn0m9uHQuofS+bDOUYcjIpInmk+hgLzxzRssWLuA9y9+n7JlykYdjohInmSZFMzsQKDezvkUYtafbGYV3f2nhEdXTGzbsY37Jt/HsfscS6eDOkUdjohInmX3RPMTwMZM1m8Jt0nopdkvsXjdYgacOgCzzEb3EBEpHrJLCo3d/ZuMK8MnjhsnLKJiZsv2LQyYOoCWjVpyxgFnRB2OiEi+ZHdNoVI22yoXdCDF1bMpz7Jy40qGnT9MrQQRKfayaynMNLNrM640s38S39DZJd7GrRt54LMHOG3/0zil8SlRhyMikm/ZtRRuBT4ws8v4OwkkAxWA8xIdWHEwaMYgftv8GwNOHRB1KCIiBSLLpODuvwInmlkbgiGuAf7n7hMKJbIi7o8tf/DItEfo2Kwjx9c/PupwREQKRDzDXEwEJhZCLMXKo9MeZcPWDfRv0z/qUERECkw8k+xIBqs3rebJGU9yyWGXcES9I6IOR0SkwCgp5MGDnz3IltQt9D2lb9ShiIgUKCWFXFq+YTnPzHyGq468imZ1mkUdjohIgVJSyKWBUwaS5mn0OaVP1KGIiBQ4JYVcWPTHIl786kWuPeZaGtdqHHU4IiIFTkkhF+6bfB/lypSjV6teUYciIpIQSgpxmr9mPm988wY3Hncj+1bfN+pwREQSIqFJwczam9kCM1toZndnsv3/zOxbM5tjZp+Z2SGJjCc/7p10L1XKV+GulndFHYqISMIkLCmYWVlgMNCBYLa2SzM56Q9z98Pd/SjgYeDxRMWTH1/98hXvzHuH2064jTpV6kQdjohIwiSypXA8sNDdF7n7NmA4cG5sAXffEPOyKn9P+1mk9JnUh1qVanF7i9ujDkVEJKHimY4zr+oDy2JeLweaZyxkZjcAtxMMtHdqZhWZWTegG0CjRo0KPNDsfLH8Cz7+4WPuP/V+alWqVajHFhEpbIlsKWQ2ucBuLQF3H+zuBwB3Ab0zq8jdh7h7srsn161bt4DDzF6vCb3Yq+pe3Nz85kI9rohIFBKZFJYDDWNeNwBWZlN+OFCkJjie8PMEJvw8gZ4te1K1QtWowxERSbhEJoWZQBMz28/MKgCdgZGxBcysSczLs4AfExhPrrg7vSf0pkGNBvxf8v9FHY6ISKFI2DUFd081sxuBMUBZ4GV3n2tm/YAUdx8J3Ghm7YDtwB/AVYmKJ7dG/TiK6cun8/zZz1OpXHYzk4qIlBzmXiRv+MlScnKyp6SkJPQYaZ5G8pBk1m9dz/c3fE/5suUTejwRkUQzs1nunpxTuUTefVRsvT//fb5a9RVDOw1VQhCRUkXDXGSwI20HfSb24eA6B9Pl8C5RhyMiUqjUUshg2LfDmP/bfN656B3KlikbdTgiIoVKLYUY23dsp+/kvhy999Gcf/D5UYcjIlLo1FKI8fJXL7Poj0X8r8v/KGPKlyJS+ujMF/or9S/6T+lPiwYt6HBgh6jDERGJhFoKoedSnmPFxhW8ft7rmGU2QoeISMmnlgLw57Y/uX/q/bTdry1t9msTdTgiIpFRUgCemvEUazavYcCpA6IORUQkUqU+Kaz7ax0PT3uYs5uezQkNTog6HBGRSJX6pPDYtMdY99c6+rfpH3UoIiKRK9VJYc2mNTwx4wkuOuQijtr7qKjDERGJXKlOCg99/hCbt2+mX5t+UYciIlIklNqksHLjSgbPHMwVR1zBQXUOijocEZEiodQmhYFTBpKalkqfU/pEHYqISJFRKpPC4nWLeWH2C1xz9DXsv8f+UYcjIlJklMqk0G9yP8pYGXq36h11KCIiRUqpSwoLflvAa1+/RvfjulO/Rv2owxERKVJKXVK4d9K9VC5Xmbtb3h11KCIiRU5Ck4KZtTezBWa20Mx2Owub2e1mNs/MvjGz8WaWlMh4vvn1G96e+za3nnAre1XdK5GHEhEplhKWFMysLDAY6AAcAlxqZodkKPYVkOzuRwDvAg8nKh6AeybeQ82KNbmjxR2JPIyISLGVyJbC8cBCd1/k7tuA4cC5sQXcfaK7bw5ffgE0SFQwM5bPYOSCkfQ4sQd7VN4jUYcRESnWEpkU6gPLYl4vD9dl5Z/A6Mw2mFk3M0sxs5Q1a9bkKZgvV3zJPtX24ZYTbsnT/iIipUEik0JmM9V4pgXNLgeSgUcy2+7uQ9w92d2T69atm6dgbmp+EwtvXki1CtXytL+ISGmQyJnXlgMNY143AFZmLGRm7YBewCnuvjWB8VClfJVEVi8iUuwlsqUwE2hiZvuZWQWgMzAytoCZHQ08D3R099UJjEVEROKQsKTg7qnAjcAYYD4wwt3nmlk/M+sYFnsEqAa8Y2ZzzGxkFtWJiEghSGT3Ee4+ChiVYV2fmOV2iTy+iIjkTql7ollERLKmpCAiIumUFEREJJ2SgoiIpDP3TJ8nK7LMbA2wJI+71wF+K8BwCrN+xV74dSe6fsVe+HUnuv5Ex54fSe6e49O/xS4p5IeZpbh7cnGsX7EXft2Jrl+xF37dia4/0bEXBnUfiYhIOiUFERFJV9qSwpBiXL9iL/y6E12/Yi/8uhNdf6JjT7hSdU1BRESyV9paCiIikg0lBRERSaekICIi6RI6SmqUzOxm4AN3X5Zj4dzX3RyY7+4bzKwycDdwDDAPuN/d1xf0MYsTMzsAOI9gkqVU4EfgrYL4XMzseMDdfaaZHQK0B74PR+QtUGY21N2vLOh6C1rMfCUr3X2cmXUBTiQYsn6Iu28vgGMcRDCd7gx3/zNmfXt3/6QA6j43rN8JJuMa6e7z81NvFsdqSTB//HfuPrag6y8JSuyFZjNbD2wCfgLeAt5x97xN8Lx73XOBI9091cyGAJuBd4G24frzC+I4UTCzq939lXzsfzNwDjAZOBOYA/xBkCS6u/ukfNR9L9CB4MvMp0BzYBLQDhjj7gPzUXfGuTwMaANMAHD3jrvtVESY2ZsEn0kVYB3BHCXvE/x7NHe/Kp/13wzcQJBkjgJucff/httmu/sx+aj7LuBSYDjBbI0QzNLYGRju7g/mM/Yv3f34cPlagvfxAXA68FF+6y+R3L1E/gBfEXSPnQ68BKwBPgGuAqrns+75McuzM2ybk+D3NTrB9S/N5/7fAmXD5SrApHC5EfBVQdQd1rsBqBGurwx8k8+6ZwNvAK2BU8Lfv4TLpxTA51oTeBD4Hlgb/swP19XKZ93fhL/LAb/GfP6W388l5nOvFi43BlIIEgMF8Df9ASifyfoKwI8FEPtXMcszgbrhclXg23zWXQN4AHgd6JJh2zP5jT2qnxLbfUTQxZAGjAXGmll5gm+ZlwKPAjmOAZKN72K+UX9tZsnunmJmTYGCaKpn9c3LCL6p5bf+b7Kpv15+6yc4Oe0AKgLVAdx9afg3yI9Ud98BbDazn9x9Q1j3FjNLy2fdycAtBPOF93D3OWa2xd0n57PenUYQtDpau/sqADPbm+BLyjvAafmou0zYhVSVIGHWBH4n+Pzz+5lDkGT+BHD3xWbWGnjXzJII/s3kRxqwL7uPZ7ZPuC2/ypjZHgRfEM3D3gJ332Rmqfms+xWCrtH3gH+Y2QUEyWErcEI+645MSU4Ku/xj9aBfdSQwMrwOkB/XAE+aWW+Cwa+mm9kyYFm4Lb9mEnS/ZPYfrlYB1F8POIOgWyeWAdPyWfeLwEwz+wJoBTwEYGZ1CU5U+bHNzKq4+2bg2J0rzawm+TyBhF8g/mNm74S/f6Vg/380dveHMhxzFfCQmf0jn3W/RNACKUuQ1N4xs0UEJ6bh+awbYJWZHeXucwDc/U8zOxt4GTg8n3XfCow3sx8J/v9A0Ko8kGA63/yqCcwi+LftZra3u68ys2rkP6Ed4O4XhMsfmlkvYELMdMPFUkm+ptDU3X9I8DGqA/sTnDyWu/uvBVTvd8B57v5jJtuWuXvDfNb/EvCKu3+WybZh7t4ln/UfChxMcDHv+/zUlaHeiuG3sIzr6wD7uPu3BXiss4CT3P3fBVTfWGAc8NrOfydmVg/oCpzm+Zya1sz2BXD3lWZWi+A6y1J3/zJfgQd1NyBopa3KZNtJ7v55PusvQ3Dxtz7BiXo5MDNsFSaEmVUB6rn7z/moYz5waPiFYue6q4A7CbrbkvIfaeErsUmhODOzCwn6Oxdksq2Tu38YQViSD2EXxt0Ed9nsFa7+laD1+qC7Z2y1SRFnZg8DY919XIb17YGn3L1JNJHlj5JCMZPfu4Ok6NHftOQpzn9TJYVixsyWunujqOOQgqO/aclTnP+mJflCc7FVCHcHSSHT37TkKal/UyWFoimRdwdJNPQ3LXlK5N9USaFo+pjg7oU5GTeY2aTCD0cKgP6mJU+J/JvqmoKIiKTTKKkiIpJOSUFERNIpKUipZWaTzOyMDOtuNbNnclHHqPAJ4uzK/JnF+lfDBxVFigwlBSnN3iIYojlW53B9tixQxt3PdPd1CYlOJAJKClKavQucbWYVAcysMcGInXPMbLyZzTazb83s3J3bzWx+2JKYDTQ0s8Xh2EuY2YdmNsvM5ppZt9gDmdljYX3jw8EBybD9WDObHO4/xsz2Seg7F8mCkoKUWu6+FviSYPY2CFoJbwNbCAYkPIZgop3HzGzniJrNgKHufrS7Zxzu+R/ufizBMNw3m1ntcH1Vgnk3jiEY/fbe2J3CIcWfAi4M938ZyPOEQSL5oecUpLTb2YX03/D3PwgePrrfzFoRDMldn7+fUF3i7l9kUdfNZnZeuNwQaEIwmU4aQbKBYCKf9zPs1ww4DPg0zD1lCSb4ESl0SgpS2n0IPB5ObFTZ3WebWVeCSZiOdfftZrYYqBSW35RZJeHEM+2AFu6+OXx4qVJmZQnmId5ld2Cuu7fIzxsRKQjqPpJSLZxRbBJBl83OC8w1gdVhQmgDxDMufk3gjzAhHMSuM2+VAXbeZdQFyDiPxQKgrpm1gKA7KZyTQqTQqaUgEiSD9/n7TqQ3gY/MLAWYQzCrWU4+Af4vHCRtARDbxbQJONTMZgHrgUtid3T3beGtqYPCWeTKAU8Ac/P+lkTyRsNciIhIOnUfiYhIOiUFERFJp6QgIiLplBRERCSdkoKIiKRTUhARkXRKCiIikk5JQURE0v0/VB8Bw+58fdoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List of features sorted from most to least important\n",
    "sorted_importances = [importance[1] for importance in feature_importances]\n",
    "sorted_features = [importance[0] for importance in feature_importances]\n",
    "\n",
    "# Cumulative importances\n",
    "cumulative_importances = np.cumsum(sorted_importances)\n",
    "\n",
    "# Make a line graph\n",
    "plt.plot(x_values, cumulative_importances, 'g-')\n",
    "\n",
    "# Draw line at 95% of importance retained\n",
    "plt.hlines(y = 0.9, xmin=0, xmax=len(sorted_importances), color = 'r', linestyles = 'dashed')\n",
    "\n",
    "# Format x ticks and labels\n",
    "plt.xticks(x_values, sorted_features, rotation = 'vertical')\n",
    "\n",
    "# Axis labels and title\n",
    "plt.xlabel('Variable'); plt.ylabel('Cumulative Importance'); plt.title('Cumulative Importances');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Confusion Matrix ===\n",
      "Predicted  0  1\n",
      "Actual         \n",
      "0          1  0\n",
      "1          3  1\n",
      "\n",
      "\n",
      "=== Classification Report ===\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.25      1.00      0.40         1\n",
      "           1       1.00      0.25      0.40         4\n",
      "\n",
      "   micro avg       0.40      0.40      0.40         5\n",
      "   macro avg       0.62      0.62      0.40         5\n",
      "weighted avg       0.85      0.40      0.40         5\n",
      "\n",
      "=== All AUC Scores ===\n",
      "[1. 1. 1. 1. 1.]\n",
      "\n",
      "\n",
      "=== Mean AUC Score ===\n",
      "Mean AUC Score - Random Forest:  1.0\n"
     ]
    }
   ],
   "source": [
    "# Extract the names of the most important features\n",
    "important_feature_names = [feature[0] for feature in feature_importances[0:10]]\n",
    "\n",
    "X = pd.DataFrame(X)\n",
    "\n",
    "# Create training and testing sets with only the important features\n",
    "important_X = X[important_feature_names] # this removes unimportant features\n",
    "y = y # y remains the same\n",
    "\n",
    "# implementing train-test-split on important features\n",
    "X_train, X_test, y_train, y_test = train_test_split(important_X, y, test_size=0.40, random_state=1)\n",
    "\n",
    "# random forest model creation\n",
    "rfc = RandomForestClassifier(n_estimators=25)\n",
    "rfc.fit(X_train,y_train)\n",
    "\n",
    "# predictions\n",
    "y_predict = rfc.predict(X_test)\n",
    "\n",
    "rfc_cv_score = cross_val_score(rfc, X, y, cv=5, scoring=\"roc_auc\")\n",
    "\n",
    "print(\"=== Confusion Matrix ===\")\n",
    "# print(confusion_matrix(y_test, y_predict))\n",
    "x = pd.crosstab(y_test, y_predict, rownames=['Actual'], colnames=['Predicted'])\n",
    "print(x)\n",
    "print('\\n')\n",
    "print(\"=== Classification Report ===\")\n",
    "print(classification_report(y_test, y_predict))\n",
    "print(\"=== All AUC Scores ===\")\n",
    "print(rfc_cv_score)\n",
    "print('\\n')\n",
    "print(\"=== Mean AUC Score ===\")\n",
    "print(\"Mean AUC Score - Random Forest: \", rfc_cv_score.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': [True, False],\n",
      " 'max_depth': [5, 10, 15, 20, None],\n",
      " 'max_features': ['auto', 'sqrt'],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [10, 20, 25, 30]}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "from pprint import pprint\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# Number of trees in random forest\n",
    "n_estimators= [10,20,25,30]\n",
    "\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [5,10,15,20]\n",
    "max_depth.append(None)\n",
    "\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "\n",
    "pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 25 candidates, totalling 75 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Jonathan\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:652: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=3.\n",
      "  % (min_groups, self.n_splits)), Warning)\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:    3.4s\n",
      "[Parallel(n_jobs=-1)]: Done  75 out of  75 | elapsed:    3.8s finished\n",
      "C:\\Users\\Jonathan\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score='raise-deprecating',\n",
       "          estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "          fit_params=None, iid='warn', n_iter=25, n_jobs=-1,\n",
       "          param_distributions={'n_estimators': [10, 20, 25, 30], 'max_features': ['auto', 'sqrt'], 'max_depth': [5, 10, 15, 20, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]},\n",
       "          pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "          return_train_score='warn', scoring=None, verbose=2)"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the random grid to search for best hyperparameters\n",
    "# First create the base model to tune\n",
    "rfc = RandomForestClassifier()\n",
    "\n",
    "# Random search of parameters, using 3 fold cross validation: i.e. \"20 percent is used for testing\"\n",
    "# search across 25 different combinations, and use all available cores\n",
    "\n",
    "rfc_random = RandomizedSearchCV(estimator = rfc, param_distributions = random_grid,\n",
    "                                n_iter = 25, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "\n",
    "# Fit the random search model\n",
    "rfc_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 10,\n",
       " 'min_samples_split': 5,\n",
       " 'min_samples_leaf': 2,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': None,\n",
       " 'bootstrap': True}"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc_random.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implementing train-test-split on important features\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=1)\n",
    "\n",
    "# random forest model creation\n",
    "rfc = RandomForestClassifier(n_estimators=30,min_samples_split=2,min_samples_leaf=1,max_features='sqrt',\n",
    "                             max_depth = 5, bootstrap = True,random_state=1)\n",
    "rfc.fit(X_train,y_train)\n",
    "\n",
    "# predictions\n",
    "y_predict = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUC with cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Confusion Matrix ===\n",
      "Predicted  0  1\n",
      "Actual         \n",
      "0          1  0\n",
      "1          1  2\n",
      "\n",
      "\n",
      "=== Classification Report ===\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.50      1.00      0.67         1\n",
      "           1       1.00      0.67      0.80         3\n",
      "\n",
      "   micro avg       0.75      0.75      0.75         4\n",
      "   macro avg       0.75      0.83      0.73         4\n",
      "weighted avg       0.88      0.75      0.77         4\n",
      "\n",
      "=== All AUC Scores ===\n",
      "[1. 1. 0. 1. 1.]\n",
      "\n",
      "\n",
      "=== Mean AUC Score ===\n",
      "Mean AUC Score - Random Forest:  0.8\n"
     ]
    }
   ],
   "source": [
    "rfc_cv_score = cross_val_score(rfc, X, y, cv=5, scoring=\"roc_auc\")\n",
    "\n",
    "print(\"=== Confusion Matrix ===\")\n",
    "# print(confusion_matrix(y_test, y_predict))\n",
    "x = pd.crosstab(y_test, y_predict, rownames=['Actual'], colnames=['Predicted'])\n",
    "print(x)\n",
    "print('\\n')\n",
    "print(\"=== Classification Report ===\")\n",
    "print(classification_report(y_test, y_predict))\n",
    "print(\"=== All AUC Scores ===\")\n",
    "print(rfc_cv_score)\n",
    "print('\\n')\n",
    "print(\"=== Mean AUC Score ===\")\n",
    "print(\"Mean AUC Score - Random Forest: \", rfc_cv_score.mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}