access_to_water.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5973443d",
   "metadata": {},
   "source": [
    "# Setup enviorment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5ef56ce9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-17 11:59:36.992131: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-08-17 11:59:37.030822: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2023-08-17 11:59:37.031910: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-17 11:59:37.862941: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from Preprocessing.time_series_preprocessing import normalize_train_features, normalize_test_features\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "131c42e2",
   "metadata": {},
   "source": [
    "### 1. Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "062f8659",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Read Dataset:\"\"\"\n",
    "embeddings_path = 'Embeddings/variational_autoencoder/variational_autoencoder_ResNet50V2__224_1024_3Bands_no_black_images_full_dataset.csv'\n",
    "labels_path = '../Tabular_data/dengue_tabular_v2.csv'\n",
    "\n",
    "\"\"\" Dataset Preprocessing:\"\"\"\n",
    "train_percentage = 80\n",
    "T = 3\n",
    "autoregressive = False\n",
    "feature_range = (-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c17ea5f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality Code</th>\n",
       "      <th>Date</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>76400</td>\n",
       "      <td>2017-07-30</td>\n",
       "      <td>1.655762</td>\n",
       "      <td>-0.284353</td>\n",
       "      <td>-0.884130</td>\n",
       "      <td>2.468364</td>\n",
       "      <td>-0.369896</td>\n",
       "      <td>-0.799041</td>\n",
       "      <td>-0.822622</td>\n",
       "      <td>-0.016440</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.173675</td>\n",
       "      <td>-0.517362</td>\n",
       "      <td>1.526553</td>\n",
       "      <td>-0.359275</td>\n",
       "      <td>0.024719</td>\n",
       "      <td>0.314871</td>\n",
       "      <td>0.616411</td>\n",
       "      <td>0.979528</td>\n",
       "      <td>0.450631</td>\n",
       "      <td>0.889211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>76400</td>\n",
       "      <td>2017-11-19</td>\n",
       "      <td>-0.798141</td>\n",
       "      <td>0.156090</td>\n",
       "      <td>0.759308</td>\n",
       "      <td>0.415680</td>\n",
       "      <td>-0.157119</td>\n",
       "      <td>0.175089</td>\n",
       "      <td>0.876964</td>\n",
       "      <td>2.257652</td>\n",
       "      <td>...</td>\n",
       "      <td>0.397869</td>\n",
       "      <td>0.833390</td>\n",
       "      <td>0.051487</td>\n",
       "      <td>-0.307350</td>\n",
       "      <td>-0.545954</td>\n",
       "      <td>-2.025124</td>\n",
       "      <td>0.942197</td>\n",
       "      <td>0.474716</td>\n",
       "      <td>-0.227024</td>\n",
       "      <td>-0.391869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>76400</td>\n",
       "      <td>2016-11-20</td>\n",
       "      <td>0.036293</td>\n",
       "      <td>-0.695327</td>\n",
       "      <td>-2.338713</td>\n",
       "      <td>-1.637983</td>\n",
       "      <td>-1.029155</td>\n",
       "      <td>-0.467536</td>\n",
       "      <td>-0.599246</td>\n",
       "      <td>2.644862</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.035237</td>\n",
       "      <td>-0.604652</td>\n",
       "      <td>-0.465066</td>\n",
       "      <td>-0.145733</td>\n",
       "      <td>0.163719</td>\n",
       "      <td>-1.455186</td>\n",
       "      <td>1.381717</td>\n",
       "      <td>0.411164</td>\n",
       "      <td>-0.229822</td>\n",
       "      <td>-0.927672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>76400</td>\n",
       "      <td>2017-07-16</td>\n",
       "      <td>-0.367984</td>\n",
       "      <td>-0.388351</td>\n",
       "      <td>1.078847</td>\n",
       "      <td>0.025843</td>\n",
       "      <td>1.036031</td>\n",
       "      <td>-0.149209</td>\n",
       "      <td>1.250700</td>\n",
       "      <td>1.699442</td>\n",
       "      <td>...</td>\n",
       "      <td>0.152797</td>\n",
       "      <td>0.651429</td>\n",
       "      <td>0.505704</td>\n",
       "      <td>0.368969</td>\n",
       "      <td>0.169281</td>\n",
       "      <td>1.002393</td>\n",
       "      <td>-1.335710</td>\n",
       "      <td>-0.694701</td>\n",
       "      <td>-0.957350</td>\n",
       "      <td>1.159607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>76400</td>\n",
       "      <td>2017-03-12</td>\n",
       "      <td>-0.733450</td>\n",
       "      <td>-1.864988</td>\n",
       "      <td>-0.011385</td>\n",
       "      <td>-2.051258</td>\n",
       "      <td>-0.319450</td>\n",
       "      <td>-1.088751</td>\n",
       "      <td>-0.932645</td>\n",
       "      <td>0.292711</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.887089</td>\n",
       "      <td>0.462687</td>\n",
       "      <td>-1.128932</td>\n",
       "      <td>1.641547</td>\n",
       "      <td>-0.680204</td>\n",
       "      <td>-0.341273</td>\n",
       "      <td>0.380461</td>\n",
       "      <td>1.057776</td>\n",
       "      <td>-1.272891</td>\n",
       "      <td>0.926245</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1026 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Municipality Code        Date         0         1         2         3  \\\n",
       "0              76400  2017-07-30  1.655762 -0.284353 -0.884130  2.468364   \n",
       "1              76400  2017-11-19 -0.798141  0.156090  0.759308  0.415680   \n",
       "2              76400  2016-11-20  0.036293 -0.695327 -2.338713 -1.637983   \n",
       "3              76400  2017-07-16 -0.367984 -0.388351  1.078847  0.025843   \n",
       "4              76400  2017-03-12 -0.733450 -1.864988 -0.011385 -2.051258   \n",
       "\n",
       "          4         5         6         7  ...      1014      1015      1016  \\\n",
       "0 -0.369896 -0.799041 -0.822622 -0.016440  ... -1.173675 -0.517362  1.526553   \n",
       "1 -0.157119  0.175089  0.876964  2.257652  ...  0.397869  0.833390  0.051487   \n",
       "2 -1.029155 -0.467536 -0.599246  2.644862  ... -0.035237 -0.604652 -0.465066   \n",
       "3  1.036031 -0.149209  1.250700  1.699442  ...  0.152797  0.651429  0.505704   \n",
       "4 -0.319450 -1.088751 -0.932645  0.292711  ... -0.887089  0.462687 -1.128932   \n",
       "\n",
       "       1017      1018      1019      1020      1021      1022      1023  \n",
       "0 -0.359275  0.024719  0.314871  0.616411  0.979528  0.450631  0.889211  \n",
       "1 -0.307350 -0.545954 -2.025124  0.942197  0.474716 -0.227024 -0.391869  \n",
       "2 -0.145733  0.163719 -1.455186  1.381717  0.411164 -0.229822 -0.927672  \n",
       "3  0.368969  0.169281  1.002393 -1.335710 -0.694701 -0.957350  1.159607  \n",
       "4  1.641547 -0.680204 -0.341273  0.380461  1.057776 -1.272891  0.926245  \n",
       "\n",
       "[5 rows x 1026 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = pd.read_csv(embeddings_path)\n",
    "embeddings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3ddc5f36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality code</th>\n",
       "      <th>Municipality</th>\n",
       "      <th>Population2007</th>\n",
       "      <th>Population2008</th>\n",
       "      <th>Population2009</th>\n",
       "      <th>Population2010</th>\n",
       "      <th>Population2011</th>\n",
       "      <th>Population2012</th>\n",
       "      <th>Population2013</th>\n",
       "      <th>Population2014</th>\n",
       "      <th>...</th>\n",
       "      <th>2022/w41</th>\n",
       "      <th>2022/w42</th>\n",
       "      <th>2022/w43</th>\n",
       "      <th>2022/w44</th>\n",
       "      <th>2022/w45</th>\n",
       "      <th>2022/w46</th>\n",
       "      <th>2022/w47</th>\n",
       "      <th>2022/w48</th>\n",
       "      <th>2022/w49</th>\n",
       "      <th>poverty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5001</td>\n",
       "      <td>Medellín</td>\n",
       "      <td>2286126</td>\n",
       "      <td>2309689</td>\n",
       "      <td>2331389</td>\n",
       "      <td>2353410</td>\n",
       "      <td>2379920</td>\n",
       "      <td>2410046</td>\n",
       "      <td>2428850</td>\n",
       "      <td>2436071</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.749178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5002</td>\n",
       "      <td>Abejorral</td>\n",
       "      <td>20643</td>\n",
       "      <td>20944</td>\n",
       "      <td>21197</td>\n",
       "      <td>21355</td>\n",
       "      <td>20973</td>\n",
       "      <td>20067</td>\n",
       "      <td>19732</td>\n",
       "      <td>19796</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.079347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5004</td>\n",
       "      <td>Abriaquí</td>\n",
       "      <td>2261</td>\n",
       "      <td>2290</td>\n",
       "      <td>2321</td>\n",
       "      <td>2353</td>\n",
       "      <td>2386</td>\n",
       "      <td>2424</td>\n",
       "      <td>2471</td>\n",
       "      <td>2488</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.824533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5021</td>\n",
       "      <td>Alejandría</td>\n",
       "      <td>3935</td>\n",
       "      <td>4020</td>\n",
       "      <td>4096</td>\n",
       "      <td>4171</td>\n",
       "      <td>4244</td>\n",
       "      <td>4337</td>\n",
       "      <td>4419</td>\n",
       "      <td>4467</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.318095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5030</td>\n",
       "      <td>Amagá</td>\n",
       "      <td>27996</td>\n",
       "      <td>28331</td>\n",
       "      <td>28624</td>\n",
       "      <td>28839</td>\n",
       "      <td>28776</td>\n",
       "      <td>28371</td>\n",
       "      <td>28259</td>\n",
       "      <td>28432</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.145102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1172 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Municipality code Municipality  Population2007  Population2008  \\\n",
       "0               5001     Medellín         2286126         2309689   \n",
       "1               5002    Abejorral           20643           20944   \n",
       "2               5004     Abriaquí            2261            2290   \n",
       "3               5021   Alejandría            3935            4020   \n",
       "4               5030        Amagá           27996           28331   \n",
       "\n",
       "   Population2009  Population2010  Population2011  Population2012  \\\n",
       "0         2331389         2353410         2379920         2410046   \n",
       "1           21197           21355           20973           20067   \n",
       "2            2321            2353            2386            2424   \n",
       "3            4096            4171            4244            4337   \n",
       "4           28624           28839           28776           28371   \n",
       "\n",
       "   Population2013  Population2014  ...  2022/w41  2022/w42  2022/w43  \\\n",
       "0         2428850         2436071  ...       4.0       8.0       6.0   \n",
       "1           19732           19796  ...       0.0       0.0       0.0   \n",
       "2            2471            2488  ...       0.0       0.0       0.0   \n",
       "3            4419            4467  ...       0.0       0.0       0.0   \n",
       "4           28259           28432  ...       1.0       0.0       0.0   \n",
       "\n",
       "   2022/w44  2022/w45  2022/w46  2022/w47  2022/w48  2022/w49    poverty  \n",
       "0       7.0       5.0      10.0       7.0       1.0       0.0  13.749178  \n",
       "1       0.0       0.0       0.0       0.0       0.0       0.0  21.079347  \n",
       "2       0.0       0.0       0.0       0.0       0.0       0.0  10.824533  \n",
       "3       0.0       0.0       0.0       0.0       0.0       0.0  16.318095  \n",
       "4       0.0       0.0       0.0       0.0       0.0       0.0  16.145102  \n",
       "\n",
       "[5 rows x 1172 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = pd.read_csv(labels_path)\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "46816836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality code</th>\n",
       "      <th>Municipality</th>\n",
       "      <th>Population2007</th>\n",
       "      <th>Population2008</th>\n",
       "      <th>Population2009</th>\n",
       "      <th>Population2010</th>\n",
       "      <th>Population2011</th>\n",
       "      <th>Population2012</th>\n",
       "      <th>Population2013</th>\n",
       "      <th>Population2014</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>117992</td>\n",
       "      <td>...</td>\n",
       "      <td>1.131579</td>\n",
       "      <td>0.101524</td>\n",
       "      <td>0.725622</td>\n",
       "      <td>0.245010</td>\n",
       "      <td>-0.015950</td>\n",
       "      <td>-1.647340</td>\n",
       "      <td>-0.335706</td>\n",
       "      <td>-0.759658</td>\n",
       "      <td>0.200150</td>\n",
       "      <td>-0.016401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>117992</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.685572</td>\n",
       "      <td>0.768649</td>\n",
       "      <td>-1.060369</td>\n",
       "      <td>0.646797</td>\n",
       "      <td>-1.105846</td>\n",
       "      <td>-0.451644</td>\n",
       "      <td>-0.751608</td>\n",
       "      <td>-1.393362</td>\n",
       "      <td>0.333627</td>\n",
       "      <td>-0.147103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>117992</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.770062</td>\n",
       "      <td>0.191217</td>\n",
       "      <td>-0.341647</td>\n",
       "      <td>-0.208239</td>\n",
       "      <td>-0.449105</td>\n",
       "      <td>-2.136420</td>\n",
       "      <td>-0.823950</td>\n",
       "      <td>0.054495</td>\n",
       "      <td>-0.365410</td>\n",
       "      <td>0.575053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>117992</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.516224</td>\n",
       "      <td>0.609407</td>\n",
       "      <td>-0.469430</td>\n",
       "      <td>0.564922</td>\n",
       "      <td>0.185102</td>\n",
       "      <td>-0.210481</td>\n",
       "      <td>-0.997317</td>\n",
       "      <td>-0.333545</td>\n",
       "      <td>-0.737093</td>\n",
       "      <td>1.031747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>117992</td>\n",
       "      <td>...</td>\n",
       "      <td>0.803845</td>\n",
       "      <td>-0.446803</td>\n",
       "      <td>0.520420</td>\n",
       "      <td>0.350235</td>\n",
       "      <td>1.068591</td>\n",
       "      <td>0.644604</td>\n",
       "      <td>0.805475</td>\n",
       "      <td>-0.007781</td>\n",
       "      <td>-1.434304</td>\n",
       "      <td>-0.481188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2198 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Municipality code Municipality  Population2007  Population2008  \\\n",
       "0               5045     Apartadó          113220          111883   \n",
       "1               5045     Apartadó          113220          111883   \n",
       "2               5045     Apartadó          113220          111883   \n",
       "3               5045     Apartadó          113220          111883   \n",
       "4               5045     Apartadó          113220          111883   \n",
       "\n",
       "   Population2009  Population2010  Population2011  Population2012  \\\n",
       "0          112089          112914          114241          115856   \n",
       "1          112089          112914          114241          115856   \n",
       "2          112089          112914          114241          115856   \n",
       "3          112089          112914          114241          115856   \n",
       "4          112089          112914          114241          115856   \n",
       "\n",
       "   Population2013  Population2014  ...      1014      1015      1016  \\\n",
       "0          117088          117992  ...  1.131579  0.101524  0.725622   \n",
       "1          117088          117992  ... -0.685572  0.768649 -1.060369   \n",
       "2          117088          117992  ... -0.770062  0.191217 -0.341647   \n",
       "3          117088          117992  ... -0.516224  0.609407 -0.469430   \n",
       "4          117088          117992  ...  0.803845 -0.446803  0.520420   \n",
       "\n",
       "       1017      1018      1019      1020      1021      1022      1023  \n",
       "0  0.245010 -0.015950 -1.647340 -0.335706 -0.759658  0.200150 -0.016401  \n",
       "1  0.646797 -1.105846 -0.451644 -0.751608 -1.393362  0.333627 -0.147103  \n",
       "2 -0.208239 -0.449105 -2.136420 -0.823950  0.054495 -0.365410  0.575053  \n",
       "3  0.564922  0.185102 -0.210481 -0.997317 -0.333545 -0.737093  1.031747  \n",
       "4  0.350235  1.068591  0.644604  0.805475 -0.007781 -1.434304 -0.481188  \n",
       "\n",
       "[5 rows x 2198 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.merge(labels, embeddings, right_on='Municipality Code', left_on='Municipality code')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "113656d7",
   "metadata": {},
   "source": [
    "### Filter Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ef33e00e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality Code</th>\n",
       "      <th>Municipality</th>\n",
       "      <th>Date</th>\n",
       "      <th>Population2007</th>\n",
       "      <th>Population2008</th>\n",
       "      <th>Population2009</th>\n",
       "      <th>Population2010</th>\n",
       "      <th>Population2011</th>\n",
       "      <th>Population2012</th>\n",
       "      <th>Population2013</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-07-30</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>...</td>\n",
       "      <td>1.131579</td>\n",
       "      <td>0.101524</td>\n",
       "      <td>0.725622</td>\n",
       "      <td>0.245010</td>\n",
       "      <td>-0.015950</td>\n",
       "      <td>-1.647340</td>\n",
       "      <td>-0.335706</td>\n",
       "      <td>-0.759658</td>\n",
       "      <td>0.200150</td>\n",
       "      <td>-0.016401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2016-11-20</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.685572</td>\n",
       "      <td>0.768649</td>\n",
       "      <td>-1.060369</td>\n",
       "      <td>0.646797</td>\n",
       "      <td>-1.105846</td>\n",
       "      <td>-0.451644</td>\n",
       "      <td>-0.751608</td>\n",
       "      <td>-1.393362</td>\n",
       "      <td>0.333627</td>\n",
       "      <td>-0.147103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-07-16</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.770062</td>\n",
       "      <td>0.191217</td>\n",
       "      <td>-0.341647</td>\n",
       "      <td>-0.208239</td>\n",
       "      <td>-0.449105</td>\n",
       "      <td>-2.136420</td>\n",
       "      <td>-0.823950</td>\n",
       "      <td>0.054495</td>\n",
       "      <td>-0.365410</td>\n",
       "      <td>0.575053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-02-19</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.516224</td>\n",
       "      <td>0.609407</td>\n",
       "      <td>-0.469430</td>\n",
       "      <td>0.564922</td>\n",
       "      <td>0.185102</td>\n",
       "      <td>-0.210481</td>\n",
       "      <td>-0.997317</td>\n",
       "      <td>-0.333545</td>\n",
       "      <td>-0.737093</td>\n",
       "      <td>1.031747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5045</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-03-12</td>\n",
       "      <td>113220</td>\n",
       "      <td>111883</td>\n",
       "      <td>112089</td>\n",
       "      <td>112914</td>\n",
       "      <td>114241</td>\n",
       "      <td>115856</td>\n",
       "      <td>117088</td>\n",
       "      <td>...</td>\n",
       "      <td>0.803845</td>\n",
       "      <td>-0.446803</td>\n",
       "      <td>0.520420</td>\n",
       "      <td>0.350235</td>\n",
       "      <td>1.068591</td>\n",
       "      <td>0.644604</td>\n",
       "      <td>0.805475</td>\n",
       "      <td>-0.007781</td>\n",
       "      <td>-1.434304</td>\n",
       "      <td>-0.481188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8349</th>\n",
       "      <td>95025</td>\n",
       "      <td>El Retorno</td>\n",
       "      <td>2016-08-14</td>\n",
       "      <td>3300</td>\n",
       "      <td>3387</td>\n",
       "      <td>3517</td>\n",
       "      <td>3703</td>\n",
       "      <td>4491</td>\n",
       "      <td>7035</td>\n",
       "      <td>9114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.559630</td>\n",
       "      <td>0.569307</td>\n",
       "      <td>-1.965172</td>\n",
       "      <td>1.363532</td>\n",
       "      <td>-0.574345</td>\n",
       "      <td>0.217865</td>\n",
       "      <td>0.759732</td>\n",
       "      <td>-0.129125</td>\n",
       "      <td>-1.374886</td>\n",
       "      <td>0.073161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8350</th>\n",
       "      <td>95025</td>\n",
       "      <td>El Retorno</td>\n",
       "      <td>2017-02-26</td>\n",
       "      <td>3300</td>\n",
       "      <td>3387</td>\n",
       "      <td>3517</td>\n",
       "      <td>3703</td>\n",
       "      <td>4491</td>\n",
       "      <td>7035</td>\n",
       "      <td>9114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.195234</td>\n",
       "      <td>0.189281</td>\n",
       "      <td>-0.049411</td>\n",
       "      <td>-0.244062</td>\n",
       "      <td>-0.445049</td>\n",
       "      <td>1.644774</td>\n",
       "      <td>-0.504784</td>\n",
       "      <td>-0.192506</td>\n",
       "      <td>0.320951</td>\n",
       "      <td>1.074259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8351</th>\n",
       "      <td>95025</td>\n",
       "      <td>El Retorno</td>\n",
       "      <td>2018-12-02</td>\n",
       "      <td>3300</td>\n",
       "      <td>3387</td>\n",
       "      <td>3517</td>\n",
       "      <td>3703</td>\n",
       "      <td>4491</td>\n",
       "      <td>7035</td>\n",
       "      <td>9114</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.224778</td>\n",
       "      <td>-1.624294</td>\n",
       "      <td>-0.505348</td>\n",
       "      <td>-0.418190</td>\n",
       "      <td>1.108645</td>\n",
       "      <td>0.864343</td>\n",
       "      <td>-0.187170</td>\n",
       "      <td>0.228194</td>\n",
       "      <td>-0.051805</td>\n",
       "      <td>0.150505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8352</th>\n",
       "      <td>95025</td>\n",
       "      <td>El Retorno</td>\n",
       "      <td>2018-05-20</td>\n",
       "      <td>3300</td>\n",
       "      <td>3387</td>\n",
       "      <td>3517</td>\n",
       "      <td>3703</td>\n",
       "      <td>4491</td>\n",
       "      <td>7035</td>\n",
       "      <td>9114</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.825643</td>\n",
       "      <td>0.647629</td>\n",
       "      <td>0.207195</td>\n",
       "      <td>-0.031073</td>\n",
       "      <td>0.515373</td>\n",
       "      <td>0.229971</td>\n",
       "      <td>-1.150133</td>\n",
       "      <td>-1.101902</td>\n",
       "      <td>-0.603411</td>\n",
       "      <td>-0.722236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8353</th>\n",
       "      <td>95025</td>\n",
       "      <td>El Retorno</td>\n",
       "      <td>2018-08-12</td>\n",
       "      <td>3300</td>\n",
       "      <td>3387</td>\n",
       "      <td>3517</td>\n",
       "      <td>3703</td>\n",
       "      <td>4491</td>\n",
       "      <td>7035</td>\n",
       "      <td>9114</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.678957</td>\n",
       "      <td>1.215729</td>\n",
       "      <td>-1.573198</td>\n",
       "      <td>0.892289</td>\n",
       "      <td>1.266582</td>\n",
       "      <td>-0.816953</td>\n",
       "      <td>-1.161835</td>\n",
       "      <td>1.381739</td>\n",
       "      <td>-0.995350</td>\n",
       "      <td>2.236238</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8354 rows × 1065 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Municipality Code Municipality        Date  Population2007  \\\n",
       "0                  5045     Apartadó  2017-07-30          113220   \n",
       "1                  5045     Apartadó  2016-11-20          113220   \n",
       "2                  5045     Apartadó  2017-07-16          113220   \n",
       "3                  5045     Apartadó  2017-02-19          113220   \n",
       "4                  5045     Apartadó  2017-03-12          113220   \n",
       "...                 ...          ...         ...             ...   \n",
       "8349              95025   El Retorno  2016-08-14            3300   \n",
       "8350              95025   El Retorno  2017-02-26            3300   \n",
       "8351              95025   El Retorno  2018-12-02            3300   \n",
       "8352              95025   El Retorno  2018-05-20            3300   \n",
       "8353              95025   El Retorno  2018-08-12            3300   \n",
       "\n",
       "      Population2008  Population2009  Population2010  Population2011  \\\n",
       "0             111883          112089          112914          114241   \n",
       "1             111883          112089          112914          114241   \n",
       "2             111883          112089          112914          114241   \n",
       "3             111883          112089          112914          114241   \n",
       "4             111883          112089          112914          114241   \n",
       "...              ...             ...             ...             ...   \n",
       "8349            3387            3517            3703            4491   \n",
       "8350            3387            3517            3703            4491   \n",
       "8351            3387            3517            3703            4491   \n",
       "8352            3387            3517            3703            4491   \n",
       "8353            3387            3517            3703            4491   \n",
       "\n",
       "      Population2012  Population2013  ...      1014      1015      1016  \\\n",
       "0             115856          117088  ...  1.131579  0.101524  0.725622   \n",
       "1             115856          117088  ... -0.685572  0.768649 -1.060369   \n",
       "2             115856          117088  ... -0.770062  0.191217 -0.341647   \n",
       "3             115856          117088  ... -0.516224  0.609407 -0.469430   \n",
       "4             115856          117088  ...  0.803845 -0.446803  0.520420   \n",
       "...              ...             ...  ...       ...       ...       ...   \n",
       "8349            7035            9114  ...  0.559630  0.569307 -1.965172   \n",
       "8350            7035            9114  ...  0.195234  0.189281 -0.049411   \n",
       "8351            7035            9114  ... -0.224778 -1.624294 -0.505348   \n",
       "8352            7035            9114  ... -0.825643  0.647629  0.207195   \n",
       "8353            7035            9114  ... -1.678957  1.215729 -1.573198   \n",
       "\n",
       "          1017      1018      1019      1020      1021      1022      1023  \n",
       "0     0.245010 -0.015950 -1.647340 -0.335706 -0.759658  0.200150 -0.016401  \n",
       "1     0.646797 -1.105846 -0.451644 -0.751608 -1.393362  0.333627 -0.147103  \n",
       "2    -0.208239 -0.449105 -2.136420 -0.823950  0.054495 -0.365410  0.575053  \n",
       "3     0.564922  0.185102 -0.210481 -0.997317 -0.333545 -0.737093  1.031747  \n",
       "4     0.350235  1.068591  0.644604  0.805475 -0.007781 -1.434304 -0.481188  \n",
       "...        ...       ...       ...       ...       ...       ...       ...  \n",
       "8349  1.363532 -0.574345  0.217865  0.759732 -0.129125 -1.374886  0.073161  \n",
       "8350 -0.244062 -0.445049  1.644774 -0.504784 -0.192506  0.320951  1.074259  \n",
       "8351 -0.418190  1.108645  0.864343 -0.187170  0.228194 -0.051805  0.150505  \n",
       "8352 -0.031073  0.515373  0.229971 -1.150133 -1.101902 -0.603411 -0.722236  \n",
       "8353  0.892289  1.266582 -0.816953 -1.161835  1.381739 -0.995350  2.236238  \n",
       "\n",
       "[8354 rows x 1065 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = ['Municipality Code', 'Municipality', 'Date', 'Population2007', 'Population2008',\n",
    "           'Population2009', 'Population2010', 'Population2011', 'Population2012',\n",
    "           'Population2013', 'Population2014', 'Population2015', 'Population2016', \n",
    "           'Population2017', 'Population2018', 'Population2019', 'Age0-4(%)', 'Age5-14(%)',\n",
    "           'Age15-29(%)', 'Age>30(%)', 'AfrocolombianPopulation(%)',\n",
    "           'IndianPopulation(%)', 'PeoplewithDisabilities(%)',\n",
    "           'Peoplewhocannotreadorwrite(%)', 'Secondary/HigherEducation(%)',\n",
    "           'Employedpopulation(%)', 'Unemployedpopulation(%)',\n",
    "           'Peopledoinghousework(%)', 'Retiredpeople(%)', 'Men(%)', 'Women(%)',\n",
    "           'Householdswithoutwateraccess(%)', 'Householdswithoutinternetaccess(%)',\n",
    "           'Buildingstratification1(%)', 'Buildingstratification2(%)',\n",
    "           'Buildingstratification3(%)', 'Buildingstratification4(%)',\n",
    "           'Buildingstratification5(%)', 'Buildingstratification6(%)', \n",
    "           'NumberofhospitalsperKm2', 'NumberofhousesperKm2', 'poverty'] + list(df.columns[1175:])\n",
    "\n",
    "df = df[columns]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a24fcd71",
   "metadata": {},
   "source": [
    "### Train Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87ae941e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Splitting using Municipality Code, using 80% for train, and 20% for test\n",
      "The resulting train dataset has 6599 rows, and 1065 columns\n",
      "The resulting test dataset has 1755 rows, and 1065 columns\n"
     ]
    }
   ],
   "source": [
    "def split_on_column(df, train_percentage, column, random_state=42):\n",
    "    # Get the unique 'Municipality Code' values\n",
    "    unique_municipality_codes = pd.unique(df[column])\n",
    "    \n",
    "    print(f'Splitting using {column}, using {train_percentage}% for train, and {100-train_percentage}% for test')\n",
    "\n",
    "    # Split the unique codes into train and test sets\n",
    "    train_codes, test_codes = train_test_split(unique_municipality_codes, train_size=train_percentage/100, random_state=random_state)\n",
    "\n",
    "    # Create the train and test sets based on the selected 'Municipality Code' values\n",
    "    train_data = df[df[column].isin(train_codes)]\n",
    "    print(f'The resulting train dataset has {train_data.shape[0]} rows, and {train_data.shape[1]} columns')\n",
    "    test_data = df[df[column].isin(test_codes)]\n",
    "    print(f'The resulting test dataset has {test_data.shape[0]} rows, and {test_data.shape[1]} columns')\n",
    "        \n",
    "    return train_data, test_data\n",
    "\n",
    "train_df, test_df = split_on_column(df, train_percentage, column='Municipality Code')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c56a2f5b",
   "metadata": {},
   "source": [
    "### Normalize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "96c714b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality Code</th>\n",
       "      <th>Municipality</th>\n",
       "      <th>Date</th>\n",
       "      <th>Population2007</th>\n",
       "      <th>Population2008</th>\n",
       "      <th>Population2009</th>\n",
       "      <th>Population2010</th>\n",
       "      <th>Population2011</th>\n",
       "      <th>Population2012</th>\n",
       "      <th>Population2013</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>-1.553536</td>\n",
       "      <td>Barbosa</td>\n",
       "      <td>2017-07-30</td>\n",
       "      <td>-0.581057</td>\n",
       "      <td>-0.580269</td>\n",
       "      <td>-0.580882</td>\n",
       "      <td>-0.58174</td>\n",
       "      <td>-0.582346</td>\n",
       "      <td>-0.583026</td>\n",
       "      <td>-0.583875</td>\n",
       "      <td>...</td>\n",
       "      <td>0.300822</td>\n",
       "      <td>-0.894012</td>\n",
       "      <td>0.235015</td>\n",
       "      <td>0.195948</td>\n",
       "      <td>2.138017</td>\n",
       "      <td>0.650532</td>\n",
       "      <td>-0.294962</td>\n",
       "      <td>0.465480</td>\n",
       "      <td>-0.967125</td>\n",
       "      <td>0.966581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>-1.553536</td>\n",
       "      <td>Barbosa</td>\n",
       "      <td>2017-11-19</td>\n",
       "      <td>-0.581057</td>\n",
       "      <td>-0.580269</td>\n",
       "      <td>-0.580882</td>\n",
       "      <td>-0.58174</td>\n",
       "      <td>-0.582346</td>\n",
       "      <td>-0.583026</td>\n",
       "      <td>-0.583875</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.493850</td>\n",
       "      <td>-0.514964</td>\n",
       "      <td>1.149728</td>\n",
       "      <td>-0.668418</td>\n",
       "      <td>1.091695</td>\n",
       "      <td>0.643280</td>\n",
       "      <td>1.518980</td>\n",
       "      <td>-0.390376</td>\n",
       "      <td>0.457881</td>\n",
       "      <td>-0.281490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>-1.553536</td>\n",
       "      <td>Barbosa</td>\n",
       "      <td>2016-11-20</td>\n",
       "      <td>-0.581057</td>\n",
       "      <td>-0.580269</td>\n",
       "      <td>-0.580882</td>\n",
       "      <td>-0.58174</td>\n",
       "      <td>-0.582346</td>\n",
       "      <td>-0.583026</td>\n",
       "      <td>-0.583875</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058121</td>\n",
       "      <td>-0.680784</td>\n",
       "      <td>1.254683</td>\n",
       "      <td>-0.286621</td>\n",
       "      <td>-0.231792</td>\n",
       "      <td>0.174809</td>\n",
       "      <td>0.462033</td>\n",
       "      <td>-0.075391</td>\n",
       "      <td>-0.258419</td>\n",
       "      <td>-0.519269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>-1.553536</td>\n",
       "      <td>Barbosa</td>\n",
       "      <td>2017-07-16</td>\n",
       "      <td>-0.581057</td>\n",
       "      <td>-0.580269</td>\n",
       "      <td>-0.580882</td>\n",
       "      <td>-0.58174</td>\n",
       "      <td>-0.582346</td>\n",
       "      <td>-0.583026</td>\n",
       "      <td>-0.583875</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.590955</td>\n",
       "      <td>1.177296</td>\n",
       "      <td>-1.039481</td>\n",
       "      <td>0.331760</td>\n",
       "      <td>0.422822</td>\n",
       "      <td>-2.260753</td>\n",
       "      <td>-0.475488</td>\n",
       "      <td>2.314020</td>\n",
       "      <td>0.395654</td>\n",
       "      <td>-1.085908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>-1.553536</td>\n",
       "      <td>Barbosa</td>\n",
       "      <td>2017-03-12</td>\n",
       "      <td>-0.581057</td>\n",
       "      <td>-0.580269</td>\n",
       "      <td>-0.580882</td>\n",
       "      <td>-0.58174</td>\n",
       "      <td>-0.582346</td>\n",
       "      <td>-0.583026</td>\n",
       "      <td>-0.583875</td>\n",
       "      <td>...</td>\n",
       "      <td>1.341019</td>\n",
       "      <td>-0.618146</td>\n",
       "      <td>1.200579</td>\n",
       "      <td>-0.944087</td>\n",
       "      <td>0.141817</td>\n",
       "      <td>-2.436057</td>\n",
       "      <td>-0.059626</td>\n",
       "      <td>-1.482334</td>\n",
       "      <td>-2.151409</td>\n",
       "      <td>-0.800865</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1065 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Municipality Code Municipality        Date  Population2007  \\\n",
       "103          -1.553536      Barbosa  2017-07-30       -0.581057   \n",
       "104          -1.553536      Barbosa  2017-11-19       -0.581057   \n",
       "105          -1.553536      Barbosa  2016-11-20       -0.581057   \n",
       "106          -1.553536      Barbosa  2017-07-16       -0.581057   \n",
       "107          -1.553536      Barbosa  2017-03-12       -0.581057   \n",
       "\n",
       "     Population2008  Population2009  Population2010  Population2011  \\\n",
       "103       -0.580269       -0.580882        -0.58174       -0.582346   \n",
       "104       -0.580269       -0.580882        -0.58174       -0.582346   \n",
       "105       -0.580269       -0.580882        -0.58174       -0.582346   \n",
       "106       -0.580269       -0.580882        -0.58174       -0.582346   \n",
       "107       -0.580269       -0.580882        -0.58174       -0.582346   \n",
       "\n",
       "     Population2012  Population2013  ...      1014      1015      1016  \\\n",
       "103       -0.583026       -0.583875  ...  0.300822 -0.894012  0.235015   \n",
       "104       -0.583026       -0.583875  ... -0.493850 -0.514964  1.149728   \n",
       "105       -0.583026       -0.583875  ...  0.058121 -0.680784  1.254683   \n",
       "106       -0.583026       -0.583875  ... -1.590955  1.177296 -1.039481   \n",
       "107       -0.583026       -0.583875  ...  1.341019 -0.618146  1.200579   \n",
       "\n",
       "         1017      1018      1019      1020      1021      1022      1023  \n",
       "103  0.195948  2.138017  0.650532 -0.294962  0.465480 -0.967125  0.966581  \n",
       "104 -0.668418  1.091695  0.643280  1.518980 -0.390376  0.457881 -0.281490  \n",
       "105 -0.286621 -0.231792  0.174809  0.462033 -0.075391 -0.258419 -0.519269  \n",
       "106  0.331760  0.422822 -2.260753 -0.475488  2.314020  0.395654 -1.085908  \n",
       "107 -0.944087  0.141817 -2.436057 -0.059626 -1.482334 -2.151409 -0.800865  \n",
       "\n",
       "[5 rows x 1065 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scale train:\n",
    "train_df, scalers = normalize_train_features(train_df, feature_range=feature_range)\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c4a89186",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Municipality Code</th>\n",
       "      <th>Municipality</th>\n",
       "      <th>Date</th>\n",
       "      <th>Population2007</th>\n",
       "      <th>Population2008</th>\n",
       "      <th>Population2009</th>\n",
       "      <th>Population2010</th>\n",
       "      <th>Population2011</th>\n",
       "      <th>Population2012</th>\n",
       "      <th>Population2013</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.55473</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-07-30</td>\n",
       "      <td>-0.079103</td>\n",
       "      <td>-0.096703</td>\n",
       "      <td>-0.104224</td>\n",
       "      <td>-0.10741</td>\n",
       "      <td>-0.107136</td>\n",
       "      <td>-0.104822</td>\n",
       "      <td>-0.105106</td>\n",
       "      <td>...</td>\n",
       "      <td>1.145377</td>\n",
       "      <td>0.081114</td>\n",
       "      <td>0.733529</td>\n",
       "      <td>0.242734</td>\n",
       "      <td>0.019391</td>\n",
       "      <td>-1.651133</td>\n",
       "      <td>-0.354651</td>\n",
       "      <td>-0.744818</td>\n",
       "      <td>0.235106</td>\n",
       "      <td>-0.024368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.55473</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2016-11-20</td>\n",
       "      <td>-0.079103</td>\n",
       "      <td>-0.096703</td>\n",
       "      <td>-0.104224</td>\n",
       "      <td>-0.10741</td>\n",
       "      <td>-0.107136</td>\n",
       "      <td>-0.104822</td>\n",
       "      <td>-0.105106</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.671244</td>\n",
       "      <td>0.750567</td>\n",
       "      <td>-1.080059</td>\n",
       "      <td>0.643643</td>\n",
       "      <td>-1.065556</td>\n",
       "      <td>-0.448594</td>\n",
       "      <td>-0.772156</td>\n",
       "      <td>-1.376648</td>\n",
       "      <td>0.369946</td>\n",
       "      <td>-0.154023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.55473</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-07-16</td>\n",
       "      <td>-0.079103</td>\n",
       "      <td>-0.096703</td>\n",
       "      <td>-0.104224</td>\n",
       "      <td>-0.10741</td>\n",
       "      <td>-0.107136</td>\n",
       "      <td>-0.104822</td>\n",
       "      <td>-0.105106</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.755709</td>\n",
       "      <td>0.171120</td>\n",
       "      <td>-0.350231</td>\n",
       "      <td>-0.209526</td>\n",
       "      <td>-0.411796</td>\n",
       "      <td>-2.143012</td>\n",
       "      <td>-0.844777</td>\n",
       "      <td>0.066927</td>\n",
       "      <td>-0.336232</td>\n",
       "      <td>0.562349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.55473</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-02-19</td>\n",
       "      <td>-0.079103</td>\n",
       "      <td>-0.096703</td>\n",
       "      <td>-0.104224</td>\n",
       "      <td>-0.10741</td>\n",
       "      <td>-0.107136</td>\n",
       "      <td>-0.104822</td>\n",
       "      <td>-0.105106</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.501945</td>\n",
       "      <td>0.590769</td>\n",
       "      <td>-0.479989</td>\n",
       "      <td>0.561947</td>\n",
       "      <td>0.219530</td>\n",
       "      <td>-0.206051</td>\n",
       "      <td>-1.018812</td>\n",
       "      <td>-0.319966</td>\n",
       "      <td>-0.711712</td>\n",
       "      <td>1.015385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.55473</td>\n",
       "      <td>Apartadó</td>\n",
       "      <td>2017-03-12</td>\n",
       "      <td>-0.079103</td>\n",
       "      <td>-0.096703</td>\n",
       "      <td>-0.104224</td>\n",
       "      <td>-0.10741</td>\n",
       "      <td>-0.107136</td>\n",
       "      <td>-0.104822</td>\n",
       "      <td>-0.105106</td>\n",
       "      <td>...</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>-0.469127</td>\n",
       "      <td>0.525157</td>\n",
       "      <td>0.347729</td>\n",
       "      <td>1.099007</td>\n",
       "      <td>0.653927</td>\n",
       "      <td>0.790931</td>\n",
       "      <td>0.004834</td>\n",
       "      <td>-1.416045</td>\n",
       "      <td>-0.485433</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1065 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Municipality Code Municipality        Date  Population2007  Population2008  \\\n",
       "0           -1.55473     Apartadó  2017-07-30       -0.079103       -0.096703   \n",
       "1           -1.55473     Apartadó  2016-11-20       -0.079103       -0.096703   \n",
       "2           -1.55473     Apartadó  2017-07-16       -0.079103       -0.096703   \n",
       "3           -1.55473     Apartadó  2017-02-19       -0.079103       -0.096703   \n",
       "4           -1.55473     Apartadó  2017-03-12       -0.079103       -0.096703   \n",
       "\n",
       "   Population2009  Population2010  Population2011  Population2012  \\\n",
       "0       -0.104224        -0.10741       -0.107136       -0.104822   \n",
       "1       -0.104224        -0.10741       -0.107136       -0.104822   \n",
       "2       -0.104224        -0.10741       -0.107136       -0.104822   \n",
       "3       -0.104224        -0.10741       -0.107136       -0.104822   \n",
       "4       -0.104224        -0.10741       -0.107136       -0.104822   \n",
       "\n",
       "   Population2013  ...      1014      1015      1016      1017      1018  \\\n",
       "0       -0.105106  ...  1.145377  0.081114  0.733529  0.242734  0.019391   \n",
       "1       -0.105106  ... -0.671244  0.750567 -1.080059  0.643643 -1.065556   \n",
       "2       -0.105106  ... -0.755709  0.171120 -0.350231 -0.209526 -0.411796   \n",
       "3       -0.105106  ... -0.501945  0.590769 -0.479989  0.561947  0.219530   \n",
       "4       -0.105106  ...  0.817739 -0.469127  0.525157  0.347729  1.099007   \n",
       "\n",
       "       1019      1020      1021      1022      1023  \n",
       "0 -1.651133 -0.354651 -0.744818  0.235106 -0.024368  \n",
       "1 -0.448594 -0.772156 -1.376648  0.369946 -0.154023  \n",
       "2 -2.143012 -0.844777  0.066927 -0.336232  0.562349  \n",
       "3 -0.206051 -1.018812 -0.319966 -0.711712  1.015385  \n",
       "4  0.653927  0.790931  0.004834 -1.416045 -0.485433  \n",
       "\n",
       "[5 rows x 1065 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scale test:\n",
    "test_df = normalize_test_features(test_df, scalers=scalers)\n",
    "\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "732fa47f",
   "metadata": {},
   "source": [
    "# 2. Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "157dc895",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.layers import Input, Dense, MultiHeadAttention, LayerNormalization\n",
    "import xgboost as xgb\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error, mean_squared_error, r2_score\n",
    "\n",
    "def create_nn_model(input_shape):\n",
    "    input_dim = Input(shape=input_shape)\n",
    "    x = Dense(500, activation='relu')(input_dim)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dense(250, activation='relu')(x)\n",
    "    output = Dense(1)(x)\n",
    "    model = keras.Model(inputs=input_dim, outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fb43437b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate_model(model, train_X, train_y, test_X, test_y, scaler):\n",
    "    if isinstance(model, keras.Sequential):\n",
    "        # Train Keras model\n",
    "        model.fit(train_X, train_y, epochs=50, batch_size=32, verbose=0)\n",
    "        y_pred = model.predict(test_X).flatten()\n",
    "    else:\n",
    "        # For other models, follow the previous approach\n",
    "        model.fit(train_X, train_y)\n",
    "        y_pred = model.predict(test_X)\n",
    "        \n",
    "    y_pred = scaler.inverse_transform(y_pred.reshape(-1, 1))\n",
    "    test_y = scaler.inverse_transform(test_y.reshape(-1, 1))\n",
    "\n",
    "    # Calculate evaluation metrics for the current model\n",
    "    mae = mean_absolute_error(test_y, y_pred)\n",
    "    #mape = mean_absolute_percentage_error(test_y, y_pred)\n",
    "    #smape = 200 * np.mean(np.abs(y_pred - test_y) / (np.abs(y_pred) + np.abs(test_y)))\n",
    "    rmse = np.sqrt(mean_squared_error(test_y, y_pred))\n",
    "    r2 = r2_score(test_y, y_pred)  # Calculate R-squared (R2) score\n",
    "\n",
    "    return rmse, mae, r2 #smape, rmse\n",
    "\n",
    "def evaluate_models(train_X, train_y, test_X, test_y, num_iterations, scaler):\n",
    "    # Create a dictionary to store evaluation results for each model\n",
    "    model_results = {}\n",
    "\n",
    "    # Perform iterations\n",
    "    for i in range(num_iterations):\n",
    "        print(f\"Iteration {i + 1}:\")\n",
    "\n",
    "        # Shuffle the training data for each iteration to have different splits\n",
    "        indices = np.arange(train_X.shape[0])\n",
    "        np.random.shuffle(indices)\n",
    "        train_X_shuffled = train_X[indices]\n",
    "        train_y_shuffled = train_y[indices]\n",
    "\n",
    "        # Create the list of regression models\n",
    "        models = [\n",
    "            xgb.XGBRegressor(),\n",
    "            SVR(),\n",
    "            #create_nn_model(input_shape=train_X.shape[1])\n",
    "        ]\n",
    "\n",
    "        # Compile the Keras model if it exists in the models list\n",
    "        for j, model in enumerate(models):\n",
    "            if isinstance(model, keras.Model):\n",
    "                model.compile(optimizer='adam', loss='mean_squared_error')\n",
    "                models[j] = model\n",
    "\n",
    "        # Train and evaluate models\n",
    "        for model in models:\n",
    "            rmse, mae, r2 = train_and_evaluate_model(model, train_X_shuffled, train_y_shuffled, test_X, test_y, scaler)\n",
    "\n",
    "            # Append the scores to the lists in the dictionary\n",
    "            model_name = model.__class__.__name__\n",
    "            if model_name not in model_results:\n",
    "                model_results[model_name] = {\n",
    "                    \"rmse\": [],\n",
    "                    \"mae\": [],\n",
    "                    \"r2\": []\n",
    "                }\n",
    "\n",
    "            model_results[model_name][\"rmse\"].append(rmse)\n",
    "            model_results[model_name][\"mae\"].append(mae)\n",
    "            model_results[model_name][\"r2\"].append(r2)\n",
    "            #model_results[model_name][\"rmse\"].append(rmse)\n",
    "\n",
    "            print(f\"{model_name} - RMSE: {rmse:.2f}, MAE: {mae:.2f}, R2: {r2:.2f}\")\n",
    "\n",
    "    # Compute the mean and standard deviation for each model and metric\n",
    "    model_summary = {}\n",
    "    for model_name, metrics in model_results.items():\n",
    "        model_summary[model_name] = {\n",
    "            \"rmse_mean\": np.mean(metrics[\"rmse\"]),\n",
    "            \"rmse_std\": np.std(metrics[\"rmse\"]),\n",
    "            \"mae_mean\": np.mean(metrics[\"mae\"]),\n",
    "            \"mae_std\": np.std(metrics[\"mae\"]),\n",
    "            \"r2_mean\": np.mean(metrics[\"r2\"]),\n",
    "            \"r2_std\": np.std(metrics[\"r2\"]),\n",
    "        }\n",
    "\n",
    "    return model_summary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "981ed4fe",
   "metadata": {},
   "source": [
    "### Features and Labels Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "60fab2ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['Age0-4(%)', 'Age5-14(%)', 'Age15-29(%)', 'Age>30(%)', \n",
    "            'AfrocolombianPopulation(%)', 'IndianPopulation(%)', \n",
    "            'PeoplewithDisabilities(%)', \n",
    "            'Peoplewhocannotreadorwrite(%)', 'Secondary/HigherEducation(%)',\n",
    "            'Employedpopulation(%)', 'Unemployedpopulation(%)',\n",
    "            'Peopledoinghousework(%)', 'Retiredpeople(%)', 'Men(%)', 'Women(%)',\n",
    "            'Householdswithoutwateraccess(%)',\n",
    "            'Buildingstratification1(%)', 'Buildingstratification2(%)',\n",
    "            'Buildingstratification3(%)', 'Buildingstratification4(%)',\n",
    "            'Buildingstratification5(%)', 'Buildingstratification6(%)', \n",
    "            'NumberofhospitalsperKm2', 'NumberofhousesperKm2', 'poverty']\n",
    "\n",
    "labels = 'Householdswithoutwateraccess(%)'\n",
    "train_cols = features\n",
    "train_cols.remove(labels)\n",
    "scaler = scalers['scaler_' + labels]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85824526",
   "metadata": {},
   "source": [
    "## Exmperiment 1 (Embeddings - Satellite Images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1995bf84",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train features and labels set\n",
    "train_X = train_df.iloc[:, 42:].to_numpy()\n",
    "train_y = train_df[labels].to_numpy()\n",
    "\n",
    "# Test features and labels set\n",
    "test_X = test_df.iloc[:, 42:].to_numpy()\n",
    "test_y = test_df[labels].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2f50647f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1:\n",
      "XGBRegressor - RMSE: 12.58, MAE: 10.49, R2: -1.76\n",
      "SVR - RMSE: 9.19, MAE: 8.03, R2: -0.47\n",
      "Iteration 2:\n",
      "XGBRegressor - RMSE: 12.58, MAE: 10.49, R2: -1.76\n",
      "SVR - RMSE: 9.19, MAE: 8.03, R2: -0.47\n",
      "Iteration 3:\n",
      "XGBRegressor - RMSE: 12.58, MAE: 10.49, R2: -1.76\n",
      "SVR - RMSE: 9.19, MAE: 8.03, R2: -0.47\n",
      "Iteration 4:\n",
      "XGBRegressor - RMSE: 12.58, MAE: 10.49, R2: -1.76\n",
      "SVR - RMSE: 9.19, MAE: 8.03, R2: -0.47\n",
      "Iteration 5:\n",
      "XGBRegressor - RMSE: 12.58, MAE: 10.49, R2: -1.76\n",
      "SVR - RMSE: 9.19, MAE: 8.03, R2: -0.47\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'XGBRegressor': {'rmse_mean': 12.578005737494939,\n",
       "  'rmse_std': 0.0,\n",
       "  'mae_mean': 10.48954906566734,\n",
       "  'mae_std': 0.0,\n",
       "  'r2_mean': -1.7550798238961902,\n",
       "  'r2_std': 0.0},\n",
       " 'SVR': {'rmse_mean': 9.188119559114377,\n",
       "  'rmse_std': 2.784936217186306e-05,\n",
       "  'mae_mean': 8.031998922564473,\n",
       "  'mae_std': 1.9965489196122822e-05,\n",
       "  'r2_mean': -0.47015757125857716,\n",
       "  'r2_std': 8.912147689226117e-06}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate the models\n",
    "evaluate_models(train_X, train_y, test_X, test_y, num_iterations=5, scaler=scaler)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c0e2da9",
   "metadata": {},
   "source": [
    "## Exmperiment 2 (Metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "13aa18b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train features and labels set\n",
    "train_X = train_df[train_cols].to_numpy()\n",
    "train_y = train_df[labels].to_numpy()\n",
    "\n",
    "# Test features and labels set\n",
    "test_X = test_df[train_cols].to_numpy()\n",
    "test_y = test_df[labels].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a11b653f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.22, MAE: 8.98, R2: -1.19\n",
      "Iteration 2:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.22, MAE: 8.98, R2: -1.19\n",
      "Iteration 3:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.22, MAE: 8.98, R2: -1.19\n",
      "Iteration 4:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.22, MAE: 8.98, R2: -1.19\n",
      "Iteration 5:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.22, MAE: 8.98, R2: -1.19\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'XGBRegressor': {'rmse_mean': 13.581672558871862,\n",
       "  'rmse_std': 1.7763568394002505e-15,\n",
       "  'mae_mean': 7.5764754015061255,\n",
       "  'mae_std': 0.0,\n",
       "  'r2_mean': -2.212307568932095,\n",
       "  'r2_std': 0.0},\n",
       " 'SVR': {'rmse_mean': 11.220431312913604,\n",
       "  'rmse_std': 6.255197910455127e-15,\n",
       "  'mae_mean': 8.975565380780049,\n",
       "  'mae_std': 3.462756959325817e-15,\n",
       "  'r2_mean': -1.1924500732803427,\n",
       "  'r2_std': 2.4161065164093008e-15}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate the models\n",
    "evaluate_models(train_X, train_y, test_X, test_y, num_iterations=5, scaler=scaler)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3817820",
   "metadata": {},
   "source": [
    "### Experiment 3 (Both)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d32428ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cols = train_cols + list(train_df.columns[42:])\n",
    "\n",
    "# Train features and labels set\n",
    "train_X = train_df[train_cols].to_numpy()\n",
    "train_y = train_df[labels].to_numpy()\n",
    "\n",
    "# Test features and labels set\n",
    "test_X = test_df[train_cols].to_numpy()\n",
    "test_y = test_df[labels].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5dfa1a54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.26, MAE: 7.39, R2: -1.21\n",
      "Iteration 2:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.26, MAE: 7.39, R2: -1.21\n",
      "Iteration 3:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.26, MAE: 7.39, R2: -1.21\n",
      "Iteration 4:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.26, MAE: 7.39, R2: -1.21\n",
      "Iteration 5:\n",
      "XGBRegressor - RMSE: 13.58, MAE: 7.58, R2: -2.21\n",
      "SVR - RMSE: 11.26, MAE: 7.39, R2: -1.21\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'XGBRegressor': {'rmse_mean': 13.581672558871862,\n",
       "  'rmse_std': 1.7763568394002505e-15,\n",
       "  'mae_mean': 7.5764754015061255,\n",
       "  'mae_std': 0.0,\n",
       "  'r2_mean': -2.212307568932095,\n",
       "  'r2_std': 0.0},\n",
       " 'SVR': {'rmse_mean': 11.263941839421312,\n",
       "  'rmse_std': 0.00014605550574727357,\n",
       "  'mae_mean': 7.394865476764363,\n",
       "  'mae_std': 8.816855023743062e-05,\n",
       "  'r2_mean': -1.2094867837036256,\n",
       "  'r2_std': 5.729941402388365e-05}}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate the models\n",
    "evaluate_models(train_X, train_y, test_X, test_y, num_iterations=5, scaler=scaler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "313c18b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0d86dce1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min value: 0.88\n",
      "Min value: 68.17\n"
     ]
    }
   ],
   "source": [
    "print(f'Min value: {df[labels].min()}')\n",
    "print(f'Min value: {df[labels].max()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "244770ce",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow28_p38_gpu_v1]",
   "language": "python",
   "name": "conda-env-tensorflow28_p38_gpu_v1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}