notebooks/lstm_model_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Top results from all different model categories\n",
    "Here all top models from each model category is compared to eachother"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Date in which the model run took place\n",
    "date = '20170616'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/david/anaconda/lib/python3.6/site-packages/matplotlib/__init__.py:1401: UserWarning:  This call to matplotlib.use() has no effect\n",
      "because the backend has already been chosen;\n",
      "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "  warnings.warn(_use_error_msg)\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'mean_absolute_percentage_error'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-50-b44794211a3f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mStandardScaler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmean_squared_error\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean_absolute_error\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean_absolute_percentage_error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[0;31m# Import custom module functions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'mean_absolute_percentage_error'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import datetime as dt\n",
    "import time as t\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import pytz\n",
    "\n",
    "# Setup for Latex Export: https://matplotlib.org/users/pgf.html. Need to import before pyplot\n",
    "def figsize(scale):\n",
    "    fig_width_pt = 469.755                          # Get this from LaTeX using \\the\\textwidth\n",
    "    inches_per_pt = 1.0/72.27                       # Convert pt to inch\n",
    "    golden_mean = (np.sqrt(5.0)-1.0)/2.0            # Aesthetic ratio (you could change this)\n",
    "    fig_width = fig_width_pt*inches_per_pt*scale    # width in inches\n",
    "    fig_height = fig_width*golden_mean              # height in inches\n",
    "    fig_size = [fig_width,fig_height]\n",
    "    return fig_size\n",
    "\n",
    "import matplotlib as mpl\n",
    "mpl.use('pgf')\n",
    "pgf_with_rc_fonts = {\n",
    "    \"text.usetex\": True,\n",
    "    \"font.family\": \"serif\",\n",
    "    \"axes.labelsize\": 10,               # LaTeX default is 10pt font.\n",
    "    \"font.size\": 10,\n",
    "    \"legend.fontsize\": 8,               # Make the legend/label fonts a little smaller\n",
    "    \"xtick.labelsize\": 8,\n",
    "    \"ytick.labelsize\": 8,\n",
    "    \"figure.figsize\": figsize(0.9),     # default fig size of 0.9 textwidth\n",
    "    #\"font.serif\": [],                   # use latex default serif font\n",
    "    #\"font.sans-serif\": [\"DejaVu Sans\"], # use a specific sans-serif font\n",
    "}\n",
    "mpl.rcParams.update(pgf_with_rc_fonts)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tabulate import tabulate\n",
    "\n",
    "from keras.models import load_model\n",
    "from keras import backend as K\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "# Import custom module functions\n",
    "module_path = os.path.abspath(os.path.join('../'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "from lstm_load_forecasting import data, lstm\n",
    "\n",
    "from IPython.display import HTML\n",
    "from IPython.display import display\n",
    "%matplotlib \n",
    "#notebook\n",
    "#mpl.rcParams['figure.figsize'] = (9,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path were the results are stored\n",
    "path = os.path.join(os.path.abspath(''), '../data/fulldataset.csv')\n",
    "\n",
    "res_path = os.path.abspath('../results/')\n",
    "model_dir = os.path.abspath('../models/')\n",
    "\n",
    "# Load validation results\n",
    "valid_results = {}\n",
    "valid_results['m1'] = pd.read_csv(os.path.join(res_path, 'notebook_01/', str('01_results_' + date + '.csv')), delimiter=';')\n",
    "valid_results['m2'] = pd.read_csv(os.path.join(res_path, 'notebook_02/', str('02_results_' + date + '.csv')), delimiter=';')\n",
    "valid_results['m3'] = pd.read_csv(os.path.join(res_path, 'notebook_03/', str('03_results_' + date + '.csv')), delimiter=';')\n",
    "valid_results['m4'] = pd.read_csv(os.path.join(res_path, 'notebook_04/', str('04_results_' + date + '.csv')), delimiter=';')\n",
    "valid_results['m5'] = pd.read_csv(os.path.join(res_path, 'notebook_05/', str('05_results_' + date + '.csv')), delimiter=';')\n",
    "valid_results['m6'] = pd.read_csv(os.path.join(res_path, 'notebook_06/', str('06_results_' + date + '.csv')), delimiter=';')\n",
    "\n",
    "# Load test results\n",
    "test_results_01 = pd.read_csv(os.path.join(res_path, 'notebook_01/', str('01_test_results' + date + '.csv')), delimiter=';')\n",
    "test_results_02 = pd.read_csv(os.path.join(res_path, 'notebook_02/', str('02_test_results' + date + '.csv')), delimiter=';')\n",
    "test_results_03 = pd.read_csv(os.path.join(res_path, 'notebook_03/', str('03_test_results' + date + '.csv')), delimiter=';')\n",
    "test_results_04 = pd.read_csv(os.path.join(res_path, 'notebook_04/', str('04_test_results' + date + '.csv')), delimiter=';')\n",
    "test_results_05 = pd.read_csv(os.path.join(res_path, 'notebook_05/', str('05_test_results' + date + '.csv')), delimiter=';')\n",
    "test_results_06 = pd.read_csv(os.path.join(res_path, 'notebook_06/', str('06_test_results' + date + '.csv')), delimiter=';')\n",
    "\n",
    "columns=['Model name', 'Mean absolute error', 'Mean squared error']\n",
    "best_models = {}\n",
    "\n",
    "# Splitdate for train and test data. As the TBATS and ARIMA benchmark needs 2 full cycle of all seasonality, needs to be after jan 01. \n",
    "loc_tz = pytz.timezone('Europe/Zurich')\n",
    "split_date = loc_tz.localize(dt.datetime(2017,2,1,0,0,0,0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1: ENTSOE Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01_386_l-150_d-0.1</td>\n",
       "      <td>0.292969</td>\n",
       "      <td>0.137098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01_339_l-125_d-0.2</td>\n",
       "      <td>0.295669</td>\n",
       "      <td>0.141007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01_385_l-150</td>\n",
       "      <td>0.296273</td>\n",
       "      <td>0.139769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01_338_l-125_d-0.1</td>\n",
       "      <td>0.299512</td>\n",
       "      <td>0.140843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01_291_l-100_d-0.2</td>\n",
       "      <td>0.300057</td>\n",
       "      <td>0.141976</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Model name  Mean absolute error  Mean squared error\n",
       "0  01_386_l-150_d-0.1             0.292969            0.137098\n",
       "1  01_339_l-125_d-0.2             0.295669            0.141007\n",
       "2        01_385_l-150             0.296273            0.139769\n",
       "3  01_338_l-125_d-0.1             0.299512            0.140843\n",
       "4  01_291_l-100_d-0.2             0.300057            0.141976"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_01 = test_results_01.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m1'] = [test_results_01.loc[0]['Model name'], ['entsoe']]\n",
    "test_results_01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2: Calendar only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1_02_2_l-5_d-0.1</td>\n",
       "      <td>0.304772</td>\n",
       "      <td>0.150897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_02_50_l-10_d-0.1</td>\n",
       "      <td>0.319560</td>\n",
       "      <td>0.164797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_02_135_l-20_l-50_d-0.2</td>\n",
       "      <td>0.341185</td>\n",
       "      <td>0.180759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_02_51_l-100_d-0.2</td>\n",
       "      <td>0.351201</td>\n",
       "      <td>0.189321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3_02_49_l-100</td>\n",
       "      <td>0.366878</td>\n",
       "      <td>0.209458</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model name  Mean absolute error  Mean squared error\n",
       "0          1_02_2_l-5_d-0.1             0.304772            0.150897\n",
       "1        1_02_50_l-10_d-0.1             0.319560            0.164797\n",
       "2  1_02_135_l-20_l-50_d-0.2             0.341185            0.180759\n",
       "3       3_02_51_l-100_d-0.2             0.351201            0.189321\n",
       "4             3_02_49_l-100             0.366878            0.209458"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_02 = test_results_02.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m2'] = [test_results_02.loc[0]['Model name'], ['calendar']]\n",
    "test_results_02"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 3: Weather only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3_03_54_l-100_l-10_d-0.2</td>\n",
       "      <td>0.432015</td>\n",
       "      <td>0.282780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_03_5_l-30_l-10_d-0.1</td>\n",
       "      <td>0.442317</td>\n",
       "      <td>0.302648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3_03_5_l-75_l-10_d-0.1</td>\n",
       "      <td>0.457086</td>\n",
       "      <td>0.314461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_03_79_l-100_l-20_l-15</td>\n",
       "      <td>0.467399</td>\n",
       "      <td>0.333536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2_03_4_l-30_l-10</td>\n",
       "      <td>0.473051</td>\n",
       "      <td>0.346634</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model name  Mean absolute error  Mean squared error\n",
       "0  3_03_54_l-100_l-10_d-0.2             0.432015            0.282780\n",
       "1    2_03_5_l-30_l-10_d-0.1             0.442317            0.302648\n",
       "2    3_03_5_l-75_l-10_d-0.1             0.457086            0.314461\n",
       "3   3_03_79_l-100_l-20_l-15             0.467399            0.333536\n",
       "4          2_03_4_l-30_l-10             0.473051            0.346634"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_03 = test_results_03.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m3'] = [test_results_03.loc[0]['Model name'], ['weather']]\n",
    "test_results_03"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 4: ENTSOE + Calendar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04_49_l-10</td>\n",
       "      <td>0.289041</td>\n",
       "      <td>0.131035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>04_62_l-10_l-10_d-0.1</td>\n",
       "      <td>0.291371</td>\n",
       "      <td>0.135468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>04_155_l-30_l-20_d-0.1</td>\n",
       "      <td>0.292002</td>\n",
       "      <td>0.133764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>04_99_l-20_d-0.2</td>\n",
       "      <td>0.295192</td>\n",
       "      <td>0.139207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04_145_l-30</td>\n",
       "      <td>0.302183</td>\n",
       "      <td>0.145987</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model name  Mean absolute error  Mean squared error\n",
       "0              04_49_l-10             0.289041            0.131035\n",
       "1   04_62_l-10_l-10_d-0.1             0.291371            0.135468\n",
       "2  04_155_l-30_l-20_d-0.1             0.292002            0.133764\n",
       "3        04_99_l-20_d-0.2             0.295192            0.139207\n",
       "4             04_145_l-30             0.302183            0.145987"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_04 = test_results_04.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m4'] = [test_results_04.loc[0]['Model name'], ['entsoe', 'calendar']]\n",
    "test_results_04"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 5: Calendar + Weather\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05_374_l-125_l-50_d-0.1</td>\n",
       "      <td>0.268845</td>\n",
       "      <td>0.120883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05_255_l-75_l-10_d-0.2</td>\n",
       "      <td>0.280159</td>\n",
       "      <td>0.124944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05_151_l-30_l-15</td>\n",
       "      <td>0.295939</td>\n",
       "      <td>0.138655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05_217_l-50_l-20</td>\n",
       "      <td>0.296139</td>\n",
       "      <td>0.139711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05_277_l-75_l-50</td>\n",
       "      <td>0.313155</td>\n",
       "      <td>0.153873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Model name  Mean absolute error  Mean squared error\n",
       "0  05_374_l-125_l-50_d-0.1             0.268845            0.120883\n",
       "1   05_255_l-75_l-10_d-0.2             0.280159            0.124944\n",
       "2         05_151_l-30_l-15             0.295939            0.138655\n",
       "3         05_217_l-50_l-20             0.296139            0.139711\n",
       "4         05_277_l-75_l-50             0.313155            0.153873"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_05 = test_results_05.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m5'] = [test_results_05.loc[0]['Model name'], ['calendar', 'weather']]\n",
    "test_results_05"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 6: All available data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Mean absolute error</th>\n",
       "      <th>Mean squared error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>06_147_l-30_d-0.2</td>\n",
       "      <td>0.290608</td>\n",
       "      <td>0.134364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>06_243_l-75_d-0.2</td>\n",
       "      <td>0.306467</td>\n",
       "      <td>0.144846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06_86_l-10_l-50_d-0.1</td>\n",
       "      <td>0.307036</td>\n",
       "      <td>0.147213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06_373_l-125_l-50</td>\n",
       "      <td>0.309126</td>\n",
       "      <td>0.145982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>06_199_l-50_l-15</td>\n",
       "      <td>0.335317</td>\n",
       "      <td>0.166415</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Model name  Mean absolute error  Mean squared error\n",
       "0      06_147_l-30_d-0.2             0.290608            0.134364\n",
       "1      06_243_l-75_d-0.2             0.306467            0.144846\n",
       "2  06_86_l-10_l-50_d-0.1             0.307036            0.147213\n",
       "3      06_373_l-125_l-50             0.309126            0.145982\n",
       "4       06_199_l-50_l-15             0.335317            0.166415"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results_06 = test_results_06.sort_values('Mean absolute error', ascending=True)\n",
    "best_models['m6'] = [test_results_06.loc[0]['Model name'], ['all']]\n",
    "test_results_06"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'06_147_l-30_d-0.2'"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config['model_name'].values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All models compared: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_absolute_percentage_error(y_true, y_pred): \n",
    "    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100\n",
    "\n",
    "results = pd.DataFrame(columns=['Model name', 'Mean absolute error', 'Mean squared error', 'MAPE', 'Diff. MAE', 'Diff. MSE'])\n",
    "\n",
    "for idx, m in enumerate(best_models.keys()):\n",
    "    \n",
    "    # Load original config:\n",
    "    config = valid_results[m].loc[valid_results[m]['model_name'] == best_models[m][0]]\n",
    "    features = best_models[m][1]\n",
    "    features.append('actual')\n",
    "    batch_size = int(config['batch_train'].values[0])\n",
    "    \n",
    "    notebook = 'notebook_0' + str(m[1:2])\n",
    "    mod_name = config['model_name'].values[0]\n",
    "    filename = os.path.join(model_dir, notebook, (mod_name +'.h5'))\n",
    "\n",
    "    # Load data and prepare for standardization\n",
    "    df = data.load_dataset(path=path, modules=features)\n",
    "    df_scaled = df.copy()\n",
    "    df_scaled = df_scaled.dropna()\n",
    "\n",
    "    # Get all float type columns and standardize them\n",
    "    floats = [key for key in dict(df_scaled.dtypes) if dict(df_scaled.dtypes)[key] in ['float64']]\n",
    "    scaler = StandardScaler()\n",
    "    scaled_columns = scaler.fit_transform(df_scaled[floats])\n",
    "    df_scaled[floats] = scaled_columns\n",
    "\n",
    "    # Split in train and test dataset\n",
    "    df_train = df_scaled.loc[(df_scaled.index < split_date )].copy()\n",
    "    df_test = df_scaled.loc[df_scaled.index >= split_date].copy()\n",
    "\n",
    "    # Split in features and label data\n",
    "    y_train = df_train['actual'].copy()\n",
    "    X_train = df_train.drop('actual', 1).copy()\n",
    "    y_test = df_test['actual'].copy()\n",
    "    X_test = df_test.drop('actual', 1).copy()\n",
    "    \n",
    "    # Load model and generate predictions\n",
    "    model = load_model(filename)\n",
    "    model.reset_states()\n",
    "    predictions = lstm.get_predictions(model=model, X=X_test, batch_size=batch_size, timesteps=1, verbose=0)\n",
    "        \n",
    "    # Otherwise, we get a memory leak!\n",
    "    K.clear_session()\n",
    "    import tensorflow as tf\n",
    "    tf.reset_default_graph()\n",
    "    \n",
    "    # Get the scaling params from the standarization and rescale\n",
    "    mu = scaler.mean_[0]\n",
    "    sigma = scaler.scale_[0]\n",
    "    predictions_raw = np.reshape(np.round(predictions * sigma + mu), (predictions.shape[0], ))\n",
    "    actual_raw = np.round(y_test * sigma + mu)\n",
    "    \n",
    "    # Load the entsoe benchmark\n",
    "    size = int(y_test.shape[0] / batch_size)\n",
    "    entsoe_benchmark = data.load_dataset(path=path, modules=['entsoe'])\n",
    "    entsoe_benchmark = entsoe_benchmark.loc[entsoe_benchmark.index >= split_date].copy()\n",
    "    entsoe_benchmark = np.reshape(entsoe_benchmark.values, (entsoe_benchmark.shape[0],))\n",
    "    \n",
    "    # Calculate benchmark and model MAE and MSE\n",
    "    mse_entsoe = mean_squared_error(actual_raw[0:size*batch_size], entsoe_benchmark[0:size*batch_size])\n",
    "    mae_entsoe = mean_absolute_error(actual_raw[0:size*batch_size], entsoe_benchmark[0:size*batch_size])\n",
    "    mape_entsoe = mean_absolute_percentage_error(actual_raw[0:size*batch_size], entsoe_benchmark[0:size*batch_size])\n",
    "    mse = mean_squared_error(actual_raw[0:size*batch_size], predictions_raw)\n",
    "    mae = mean_absolute_error(actual_raw[0:size*batch_size], predictions_raw)\n",
    "    mape = mean_absolute_percentage_error(actual_raw[0:size*batch_size], predictions_raw)\n",
    "    \n",
    "    # Store results\n",
    "    result = [{'Model name': mod_name, \n",
    "               'Mean squared error': mse, 'Mean absolute error': mae,\n",
    "               'MAPE': mape,\n",
    "               'Diff. MAE': mae - mae_entsoe, 'Diff. MSE': mse - mse_entsoe,\n",
    "              }]\n",
    "    results = results.append(result, ignore_index=True)\n",
    "    \n",
    "    graph = False\n",
    "    if graph:\n",
    "        \n",
    "        # Set time vector for graph\n",
    "        time_test = y_test.index\n",
    "        time_vector = time_test.values\n",
    "        time_vector = time_vector[0:size*batch_size]\n",
    "        time_vector = np.reshape(time_vector, (size*batch_size,1))\n",
    "\n",
    "        #%matplotlib qt\n",
    "        #plt.clf()\n",
    "        #plt.ion()\n",
    "    \n",
    "        plt.figure()\n",
    "        plt.plot(time_vector, entsoe_benchmark[:size*batch_size], label='ENTSOE Forecast')\n",
    "        plt.plot(time_vector, actual_raw[:size*batch_size], label='Actual Load')\n",
    "        plt.plot(time_vector, predictions_raw, label='Model predictions')\n",
    "        plt.title('LSTM Model: ${}$'.format(mod_name))\n",
    "        plt.ylabel('Electricity load (in MW)')\n",
    "        plt.xlabel('Date')\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.show\n",
    "\n",
    "        #filename = plot_dir + model_name + 'top_model_predictions'\n",
    "        #plt.savefig(filename + '.pgf')\n",
    "        #plt.savefig(filename + '.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test dataset performance of the best models):\n",
      "ENTSOE Forecast (Benchmark) metrics: \tMAE = 518.276  \tMSE = 427415.152  \tMAPE = 7.686\n",
      "      Model name                  Mean absolute error    Mean squared error    MAPE    Diff. MAE    Diff. MSE\n",
      "----  ------------------------  ---------------------  --------------------  ------  -----------  -----------\n",
      "4.00  05_374_l-125_l-50_d-0.1                  311.59             162371.26    4.67      -206.69   -265043.90\n",
      "3.00  04_49_l-10                               335.00             176006.62    5.05      -183.28   -251408.53\n",
      "5.00  06_147_l-30_d-0.2                        336.80             180471.58    5.12      -181.48   -246943.57\n",
      "0.00  01_386_l-150_d-0.1                       339.54             184154.85    5.11      -178.73   -243260.31\n",
      "1.00  1_02_2_l-5_d-0.1                         353.21             202680.59    5.31      -165.06   -224734.56\n",
      "2.00  3_03_54_l-100_l-10_d-0.2                 500.68             379821.29    7.34       -17.59    -47593.86\n"
     ]
    }
   ],
   "source": [
    "results = results.sort_values('Diff. MAE', ascending=True)\n",
    "print('Test dataset performance of the best models):')\n",
    "print('ENTSOE Forecast (Benchmark) metrics: \\tMAE = {:.3f}  \\tMSE = {:.3f}  \\tMAPE = {:.3f}'.format(np.asscalar(mae_entsoe), np.asscalar(mse_entsoe), mape_entsoe))\n",
    "print(tabulate(results, headers='keys', numalign=\"right\", floatfmt=\".2f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}