1st-place-of-jane-street-adapted-to-crypto.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:43.314523Z",
     "iopub.status.busy": "2022-01-25T19:20:43.314184Z",
     "iopub.status.idle": "2022-01-25T19:20:43.36342Z",
     "shell.execute_reply": "2022-01-25T19:20:43.362623Z",
     "shell.execute_reply.started": "2022-01-25T19:20:43.314436Z"
    },
    "papermill": {
     "duration": 0.075234,
     "end_time": "2021-11-27T08:16:38.567326",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.492092",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_51189/612802622.py:1: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import display, HTML, Javascript\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/t/Datasets/kaggle_crypto/internal'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 48>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m html_contents \u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;124m<!DOCTYPE html>\u001b[39m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;124m<html lang=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124men\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m>\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[38;5;124m</html>\u001b[39m\n\u001b[1;32m     45\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m---> 48\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m../input/g-research-crypto-forecasting/\u001b[39m\u001b[38;5;124m\"\u001b[39m): os\u001b[38;5;241m.\u001b[39mchdir(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/t/Datasets/kaggle_crypto/internal\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     49\u001b[0m HTML(html_contents)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/t/Datasets/kaggle_crypto/internal'"
     ]
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML, Javascript\n",
    "\n",
    "# ----- Notebook Theme -----\n",
    "\n",
    "notebook_theme = 'carrot'\n",
    "color_maps = {'turquoise': ['#1abc9c', '#e8f8f5', '#d1f2eb', '#a3e4d7', '#76d7c4', '#48c9b0', '#1abc9c', '#17a589', '#148f77', '#117864', '#0e6251'], 'green': ['#16a085', '#e8f6f3', '#d0ece7', '#a2d9ce', '#73c6b6', '#45b39d', '#16a085', '#138d75', '#117a65', '#0e6655', '#0b5345'], 'emerald': ['#2ecc71', '#eafaf1', '#d5f5e3', '#abebc6', '#82e0aa', '#58d68d', '#2ecc71', '#28b463', '#239b56', '#1d8348', '#186a3b'], 'nephritis': ['#27ae60', '#e9f7ef', '#d4efdf', '#a9dfbf', '#7dcea0', '#52be80', '#27ae60', '#229954', '#1e8449', '#196f3d', '#145a32'], 'peter': ['#3498db', '#ebf5fb', '#d6eaf8', '#aed6f1', '#85c1e9', '#5dade2', '#3498db', '#2e86c1', '#2874a6', '#21618c', '#1b4f72'], 'belize': ['#2980b9', '#eaf2f8', '#d4e6f1', '#a9cce3', '#7fb3d5', '#5499c7', '#2980b9', '#2471a3', '#1f618d', '#1a5276', '#154360'], 'amethyst': ['#9b59b6', '#f5eef8', '#ebdef0', '#d7bde2', '#c39bd3', '#af7ac5', '#9b59b6', '#884ea0', '#76448a', '#633974', '#512e5f'], 'wisteria': ['#8e44ad', '#f4ecf7', '#e8daef', '#d2b4de', '#bb8fce', '#a569bd', '#8e44ad', '#7d3c98', '#6c3483', '#5b2c6f', '#4a235a'], 'wet': ['#34495e', '#ebedef', '#d6dbdf', '#aeb6bf', '#85929e', '#5d6d7e', '#34495e', '#2e4053', '#283747', '#212f3c', '#1b2631'], 'midnight': ['#2c3e50', '#eaecee', '#d5d8dc', '#abb2b9', '#808b96', '#566573', '#2c3e50', '#273746', '#212f3d', '#1c2833', '#17202a'], 'sunflower': ['#f1c40f', '#fef9e7', '#fcf3cf', '#f9e79f', '#f7dc6f', '#f4d03f', '#f1c40f', '#d4ac0d', '#b7950b', '#9a7d0a', '#7d6608'], 'orange': ['#f39c12', '#fef5e7', '#fdebd0', '#fad7a0', '#f8c471', '#f5b041', '#f39c12', '#d68910', '#b9770e', '#9c640c', '#7e5109'], 'carrot': ['#e67e22', '#fdf2e9', '#fae5d3', '#f5cba7', '#f0b27a', '#eb984e', '#e67e22', '#ca6f1e', '#af601a', '#935116', '#784212'], 'pumpkin': ['#d35400', '#fbeee6', '#f6ddcc', '#edbb99', '#e59866', '#dc7633', '#d35400', '#ba4a00', '#a04000', '#873600', '#6e2c00'], 'alizarin': ['#e74c3c', '#fdedec', '#fadbd8', '#f5b7b1', '#f1948a', '#ec7063', '#e74c3c', '#cb4335', '#b03a2e', '#943126', '#78281f'], 'pomegranate': ['#c0392b', '#f9ebea', '#f2d7d5', '#e6b0aa', '#d98880', '#cd6155', '#c0392b', '#a93226', '#922b21', '#7b241c', '#641e16'], 'clouds': ['#ecf0f1', '#fdfefe', '#fbfcfc', '#f7f9f9', '#f4f6f7', '#f0f3f4', '#ecf0f1', '#d0d3d4', '#b3b6b7', '#979a9a', '#7b7d7d'], 'silver': ['#bdc3c7', '#f8f9f9', '#f2f3f4', '#e5e7e9', '#d7dbdd', '#cacfd2', '#bdc3c7', '#a6acaf', '#909497', '#797d7f', '#626567'], 'concrete': ['#95a5a6', '#f4f6f6', '#eaeded', '#d5dbdb', '#bfc9ca', '#aab7b8', '#95a5a6', '#839192', '#717d7e', '#5f6a6a', '#4d5656'], 'asbestos': ['#7f8c8d', '#f2f4f4', '#e5e8e8', '#ccd1d1', '#b2babb', '#99a3a4', '#7f8c8d', '#707b7c', '#616a6b', '#515a5a', '#424949']}\n",
    "# color_maps = {'red': ['#f44336', '#ffebee', '#ffcdd2', '#ef9a9a', '#e57373', '#ef5350', '#f44336', '#e53935', '#d32f2f', '#c62828', '#b71c1c', '#ff8a80', '#ff5252', '#ff1744', '#d50000'], 'pink': ['#e91e63', '#fce4ec', '#f8bbd0', '#f48fb1', '#f06292', '#ec407a', '#e91e63', '#d81b60', '#c2185b', '#ad1457', '#880e4f', '#ff80ab', '#ff4081', '#f50057', '#c51162'], 'purple': ['#9c27b0', '#f3e5f5', '#e1bee7', '#ce93d8', '#ba68c8', '#ab47bc', '#9c27b0', '#8e24aa', '#7b1fa2', '#6a1b9a', '#4a148c', '#ea80fc', '#e040fb', '#d500f9', '#aa00ff'], 'deep': ['#673ab7', '#ede7f6', '#d1c4e9', '#b39ddb', '#9575cd', '#7e57c2', '#673ab7', '#5e35b1', '#512da8', '#4527a0', '#311b92', '#b388ff', '#7c4dff', '#651fff', '#6200ea', '#ff5722', '#fbe9e7', '#ffccbc', '#ffab91', '#ff8a65', '#ff7043', '#ff5722', '#f4511e', '#e64a19', '#d84315', '#bf360c', '#ff9e80', '#ff6e40', '#ff3d00', '#dd2c00'], 'indigo': ['#3f51b5', '#e8eaf6', '#c5cae9', '#9fa8da', '#7986cb', '#5c6bc0', '#3f51b5', '#3949ab', '#303f9f', '#283593', '#1a237e', '#8c9eff', '#536dfe', '#3d5afe', '#304ffe'], 'blue': ['#2196f3', '#e3f2fd', '#bbdefb', '#90caf9', '#64b5f6', '#42a5f5', '#2196f3', '#1e88e5', '#1976d2', '#1565c0', '#0d47a1', '#82b1ff', '#448aff', '#2979ff', '#2962ff', '#607d8b', '#eceff1', '#cfd8dc', '#b0bec5', '#90a4ae', '#78909c', '#607d8b', '#546e7a', '#455a64', '#37474f', '#263238'], 'light': ['#03a9f4', '#e1f5fe', '#b3e5fc', '#81d4fa', '#4fc3f7', '#29b6f6', '#03a9f4', '#039be5', '#0288d1', '#0277bd', '#01579b', '#80d8ff', '#40c4ff', '#00b0ff', '#0091ea', '#8bc34a', '#f1f8e9', '#dcedc8', '#c5e1a5', '#aed581', '#9ccc65', '#8bc34a', '#7cb342', '#689f38', '#558b2f', '#33691e', '#ccff90', '#b2ff59', '#76ff03', '#64dd17'], 'cyan': ['#00bcd4', '#e0f7fa', '#b2ebf2', '#80deea', '#4dd0e1', '#26c6da', '#00bcd4', '#00acc1', '#0097a7', '#00838f', '#006064', '#84ffff', '#18ffff', '#00e5ff', '#00b8d4'], 'teal': ['#009688', '#e0f2f1', '#b2dfdb', '#80cbc4', '#4db6ac', '#26a69a', '#009688', '#00897b', '#00796b', '#00695c', '#004d40', '#a7ffeb', '#64ffda', '#1de9b6', '#00bfa5'], 'green': ['#4caf50', '#e8f5e9', '#c8e6c9', '#a5d6a7', '#81c784', '#66bb6a', '#4caf50', '#43a047', '#388e3c', '#2e7d32', '#1b5e20', '#b9f6ca', '#69f0ae', '#00e676', '#00c853'], 'lime': ['#cddc39', '#f9fbe7', '#f0f4c3', '#e6ee9c', '#dce775', '#d4e157', '#cddc39', '#c0ca33', '#afb42b', '#9e9d24', '#827717', '#f4ff81', '#eeff41', '#c6ff00', '#aeea00'], 'yellow': ['#ffeb3b', '#fffde7', '#fff9c4', '#fff59d', '#fff176', '#ffee58', '#ffeb3b', '#fdd835', '#fbc02d', '#f9a825', '#f57f17', '#ffff8d', '#ffff00', '#ffea00', '#ffd600'], 'amber': ['#ffc107', '#fff8e1', '#ffecb3', '#ffe082', '#ffd54f', '#ffca28', '#ffc107', '#ffb300', '#ffa000', '#ff8f00', '#ff6f00', '#ffe57f', '#ffd740', '#ffc400', '#ffab00'], 'orange': ['#ff9800', '#fff3e0', '#ffe0b2', '#ffcc80', '#ffb74d', '#ffa726', '#ff9800', '#fb8c00', '#f57c00', '#ef6c00', '#e65100', '#ffd180', '#ffab40', '#ff9100', '#ff6d00'], 'brown': ['#795548', '#efebe9', '#d7ccc8', '#bcaaa4', '#a1887f', '#8d6e63', '#795548', '#6d4c41', '#5d4037', '#4e342e', '#3e2723'], 'grey': ['#9e9e9e', '#fafafa', '#f5f5f5', '#eeeeee', '#e0e0e0', '#bdbdbd', '#9e9e9e', '#757575', '#616161', '#424242', '#212121'], 'white': ['#ffffff'], 'black': ['#000000']}\n",
    "\n",
    "color_maps = {i: color_maps[i] for i in color_maps if i not in ['clouds', 'silver', 'concrete', 'asbestos', 'wet asphalt', 'midnight blue', 'wet']}\n",
    "\n",
    "CMAP = 'Oranges'\n",
    "prompt = '#1DBCCD'\n",
    "main_color = '#E58F65' # color_maps[notebook_theme]\n",
    "strong_main_color = '#EB9514' # = color_maps[notebook_theme] \n",
    "custom_colors = [strong_main_color, main_color]\n",
    "\n",
    "# ----- Notebook Theme -----\n",
    "\n",
    "html_contents =\"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html lang=\"en\">\n",
    "    <head>\n",
    "        <link rel=\"stylesheet\" href=\"https://www.w3schools.com/w3css/4/w3.css\">\n",
    "        <link rel=\"stylesheet\" href=\"https://fonts.googleapis.com/css?family=Raleway\">\n",
    "        <link rel=\"stylesheet\" href=\"https://fonts.googleapis.com/css?family=Oswald\">\n",
    "        <link rel=\"stylesheet\" href=\"https://fonts.googleapis.com/css?family=Open Sans\">\n",
    "        <link rel=\"stylesheet\" href=\"https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css\">\n",
    "        <style>\n",
    "        .title-section{\n",
    "            font-family: \"Oswald\", Arial, sans-serif;\n",
    "            font-weight: bold;\n",
    "            color: \"#6A8CAF\";\n",
    "            letter-spacing: 6px;\n",
    "        }\n",
    "        hr { border: 1px solid #E58F65 !important;\n",
    "             color: #E58F65 !important;\n",
    "             background: #E58F65 !important;\n",
    "           }\n",
    "        body {\n",
    "            font-family: \"Open Sans\", sans-serif;\n",
    "            }        \n",
    "        </style>\n",
    "    </head>    \n",
    "</html>\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "if not os.path.exists(\"../input/g-research-crypto-forecasting/\"): os.chdir('/t/Datasets/kaggle_crypto/internal')\n",
    "HTML(html_contents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "42a2c92b-00e6-4c13-b99e-408266d1b964",
    "_uuid": "9b3904ae-5fd7-432b-a10a-1a2446fbcc4b",
    "papermill": {
     "duration": 0.029036,
     "end_time": "2021-11-27T08:16:38.62657",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.597534",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<br><br>\n",
    "<div>    \n",
    "<!--     <div style = \"float:left; width:55%; overflow:hidden;\">         -->\n",
    "        <center><img src=\"https://i.ibb.co/hHpTy3c/g-research-logo6.png\" style = \"max-height:300px;\"></center> \n",
    "<!--     </div> -->\n",
    "<!--     <div style = \"float:right; width:35%; overflow:hidden;\"> -->\n",
    "<!--         <img src=\"img/meditation/Meditation3.gif\">  -->\n",
    "<!--     </div> -->\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "3208562a-4f7a-45d5-b8e2-67da427cd906",
    "_uuid": "d7276e81-829f-4be5-967d-0a641fb1adbf",
    "papermill": {
     "duration": 0.029207,
     "end_time": "2021-11-27T08:16:38.685101",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.655894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "241ce292-edca-4688-a665-c6321e80b438",
    "_uuid": "79576405-5d2f-465f-bb01-10eee623172c",
    "papermill": {
     "duration": 0.02926,
     "end_time": "2021-11-27T08:16:38.745078",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.715818",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div>    \n",
    "    <div style = \"float:left; width:55%; overflow:hidden;\">        \n",
    "        <br><br><br><br>        \n",
    "        <span style = \"float:right;\">\n",
    "        <h2>1st Place of Jane Street 🏆➜ Adapted to Crypto 👌</h2>\n",
    "        <p>G-Research Crypto Forecasting Competition</p>\n",
    "        <br>\n",
    "        <b></b>\n",
    "        <b>\n",
    "        - 🌎 <a href=\"https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/284903\">Discussion Thread</a>\n",
    "        <br>\n",
    "        - 🇰 <a href=\"https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285726\">The dataset</a>\n",
    "        </b>            \n",
    "        </span>\n",
    "    </div>\n",
    "    <div style=\"float:right; width:35%; max-height:300px; overflow: hidden;\">        \n",
    "        <img src=\"https://i.ibb.co/9YFyhT8/Bitcoin2.gif\" style = \"max-height: 300px;\">         \n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "ca79faf7-ca73-4e9b-8552-be96bdc881f9",
    "_uuid": "4e428101-c977-488b-aa10-fe96d228224f",
    "papermill": {
     "duration": 0.029361,
     "end_time": "2021-11-27T08:16:38.804043",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.774682",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<span id=\"introduction\"></span>\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "8af163ff-915a-4fae-aefe-6447e64952e5",
    "_uuid": "b328cc9e-a536-4347-beed-d033e9f5ac6a",
    "papermill": {
     "duration": 0.029387,
     "end_time": "2021-11-27T08:16:38.863749",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.834362",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    ">### 1st Place of Jane Street 🏆 ➜ Adapted to Crypto 👌\n",
    ">This is a simple starter notebook for Kaggle's Crypto Comp showing purged group timeseries KFold with extra data. Purged Times Series is explained [here][2]. There are many configuration variables below to allow you to experiment. Use either GPU or TPU. You can control which years are loaded, which neural networks are used, and whether to use feature engineering. You can experiment with different data preprocessing, model architecture, loss, optimizers, and learning rate schedules. The extra datasets contain the full history of the assets at the same format of the competition, so you can input that into your model too.\n",
    ">\n",
    ">**NOTE:** this notebook lets you run a different experiment in each fold if you want to run lots of experiments. (Then it is like running multiple holdout validaiton experiments but in that case note that the overall CV score is meaningless because LB will be much different when the multiple experiments are ensembled to predict test). **If you want a proper CV with a reliable overall CV score you need to choose the same configuration for each fold.**\n",
    ">\n",
    ">**See Original Links:**\n",
    ">- https://www.kaggle.com/gogo827jz/jane-street-supervised-autoencoder-mlp\n",
    ">- https://www.kaggle.com/c/jane-street-market-prediction/discussion/224348\n",
    ">\n",
    ">This notebook follows the ideas presented in my \"Initial Thoughts\" [here][1].\n",
    "\n",
    "[1]: https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/284903\n",
    "[2]: https://www.kaggle.com/yamqwe/let-s-talk-validation-grouptimeseriessplit\n",
    "\n",
    "<div class=\"alert alert-block alert-warning\">\n",
    "<b>References:</b>\n",
    "<ul>\n",
    "    <li><a href = \"https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285726\">Dataset Thread</a></li>\n",
    "    <li><a href = \"https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/284903\">Initial Thoughts Thread\n",
    "</a></li>\n",
    "    <li><a href = \"https://www.kaggle.com/yamqwe/let-s-talk-validation-grouptimeseriessplit\">Validation Thread\n",
    "</a></li>\n",
    "</ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030232,
     "end_time": "2021-11-27T08:16:38.923363",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.893131",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "____\n",
    "\n",
    "#### <center>All notebooks in the series 👇</center>\n",
    "\n",
    "| CV + Model | Hyperparam Optimization  | Time Series Models | Feature Engineering |\n",
    "| --- | --- | --- | --- |\n",
    "| [Neural Network Starter](https://www.kaggle.com/yamqwe/purgedgrouptimeseries-cv-with-extra-data-nn) | [MLP + AE](https://www.kaggle.com/yamqwe/bottleneck-encoder-mlp-keras-tuner)        | [LSTM](https://www.kaggle.com/yamqwe/time-series-modeling-lstm) | [Technical Analysis #1](https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-features) |\n",
    "| [LightGBM Starter](https://www.kaggle.com/yamqwe/purgedgrouptimeseries-cv-with-extra-data-lgbm)     | [LightGBM](https://www.kaggle.com/yamqwe/purged-time-series-cv-lightgbm-optuna)     | [Wavenet](https://www.kaggle.com/yamqwe/time-series-modeling-wavenet)  | [Technical Analysis #2](https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-feats-2) |\n",
    "| [Catboost Starter](https://www.kaggle.com/yamqwe/purgedgrouptimeseries-cv-extra-data-catboost)      | [Catboost](https://www.kaggle.com/yamqwe/purged-time-series-cv-catboost-gpu-optuna) | [Multivariate-Transformer [written from scratch]](https://www.kaggle.com/yamqwe/time-series-modeling-multivariate-transformer) | [Time Series Agg](https://www.kaggle.com/yamqwe/features-all-time-series-aggregations-ever) | \n",
    "| [XGBoost Starter](https://www.kaggle.com/yamqwe/xgb-extra-data)                                            | [XGboost](https://www.kaggle.com/yamqwe/purged-time-series-cv-xgboost-gpu-optuna) | [N-BEATS](https://www.kaggle.com/yamqwe/crypto-forecasting-n-beats) |  [Neutralization](https://www.kaggle.com/yamqwe/g-research-avoid-overfit-feature-neutralization/) |\n",
    "| [Supervised AE [Janestreet 1st]](https://www.kaggle.com/yamqwe/1st-place-of-jane-street-adapted-to-crypto) | [Supervised AE [Janestreet 1st]](https://www.kaggle.com/yamqwe/1st-place-of-jane-street-keras-tuner) | [DeepAR](https://www.kaggle.com/yamqwe/probabilistic-forecasting-deepar/) | [Quant's Volatility Features](https://www.kaggle.com/yamqwe/crypto-prediction-volatility-features) |\n",
    "| [Transformer)](https://www.kaggle.com/yamqwe/let-s-test-a-transformer)                                     | [Transformer](https://www.kaggle.com/yamqwe/sh-tcoins-transformer-baseline)  |  | ⏳Target Engineering |\n",
    "| [TabNet Starter](https://www.kaggle.com/yamqwe/tabnet-cv-extra-data)                                       |  |  |⏳Fourier Analysis | \n",
    "| [Reinforcement Learning (PPO) Starter](https://www.kaggle.com/yamqwe/g-research-reinforcement-learning-starter) | | | ⏳Wavelets | \n",
    "\n",
    "____"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "3208562a-4f7a-45d5-b8e2-67da427cd906",
    "_uuid": "d7276e81-829f-4be5-967d-0a641fb1adbf",
    "papermill": {
     "duration": 0.030391,
     "end_time": "2021-11-27T08:16:38.983127",
     "exception": false,
     "start_time": "2021-11-27T08:16:38.952736",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"outline\">Table Of Content 📑</span>\n",
    "<hr >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "d70815a1-3398-43b2-be85-2230fffd89a9",
    "_uuid": "e9b980c5-036c-4e25-992b-115169a95ba4",
    "papermill": {
     "duration": 0.029357,
     "end_time": "2021-11-27T08:16:39.041965",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.012608",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Table Of Content\n",
    "\n",
    "1. [📝 Introduction](#introduction)\n",
    "\n",
    "2. [📑 Table Of Content](#outline) \n",
    "\n",
    "3. [🤿 Diving into the Data](#diving) \n",
    "\n",
    "4. [📚 Imports](#imports) \n",
    "\n",
    "5. [🎚️ Configurations](#config) \n",
    "\n",
    "6. [🗃️ Data Loading](#loading)  \n",
    "\n",
    "7. [🔬 Feature Engineering](#features)  \n",
    "\n",
    "8. [⚙️ Configure The Model](#modelconf)\n",
    "\n",
    "9. [⏱️ LR Scheduler](#sched)\n",
    "\n",
    "10. [🏋️ Training](#training)\n",
    "\n",
    "11. [🇰 Submit to Kaggle](#submit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "310c5bce-3fab-4a82-b70e-d5e8a0e5e594",
    "_uuid": "ce509f90-2ae9-4ea3-94ad-39479b72ca12",
    "papermill": {
     "duration": 0.028816,
     "end_time": "2021-11-27T08:16:39.100503",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.071687",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"outline\">Diving into the Data 🤿</span>\n",
    "<hr >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "ca6da936-954f-427b-af0c-f04517c4d4ed",
    "_uuid": "a91494a0-a071-49e5-bcc0-9c7b7566ad75",
    "papermill": {
     "duration": 0.028937,
     "end_time": "2021-11-27T08:16:39.158735",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.129798",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "#### **<span>Dataset Structure</span>**\n",
    "\n",
    "> **train.csv** - The training set\n",
    "> \n",
    "> 1.  timestamp - A timestamp for the minute covered by the row.\n",
    "> 2.  Asset_ID - An ID code for the cryptoasset.\n",
    "> 3.  Count - The number of trades that took place this minute.\n",
    "> 4.  Open - The USD price at the beginning of the minute.\n",
    "> 5.  High - The highest USD price during the minute.\n",
    "> 6.  Low - The lowest USD price during the minute.\n",
    "> 7.  Close - The USD price at the end of the minute.\n",
    "> 8.  Volume - The number of cryptoasset u units traded during the minute.\n",
    "> 9.  VWAP - The volume-weighted average price for the minute.\n",
    "> 10. Target - 15 minute residualized returns. See the 'Prediction and Evaluation section of this notebook for details of how the target is calculated.\n",
    "> 11. Weight - Weight, defined by the competition hosts [here](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)\n",
    "> 12. Asset_Name - Human readable Asset name.\n",
    "> \n",
    ">\n",
    "> **example_test.csv** - An example of the data that will be delivered by the time series API.\n",
    "> \n",
    "> **example_sample_submission.csv** - An example of the data that will be delivered by the time series API. The data is just copied from train.csv.\n",
    "> \n",
    "> **asset_details.csv** - Provides the real name and of the cryptoasset for each Asset_ID and the weight each cryptoasset receives in the metric.\n",
    "> \n",
    "> **supplemental_train.csv** - After the submission period is over this file's data will be replaced with cryptoasset prices from the submission period. In the Evaluation phase, the train, train supplement, and test set will be contiguous in time, apart from any missing data. The current copy, which is just filled approximately the right amount of data from train.csv is provided as a placeholder.\n",
    ">\n",
    "> - 📌 There are 14 coins in the dataset\n",
    ">\n",
    "> - 📌 There are 4 years  in the [full] dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "75405d52-594f-4791-a45a-16354bc24cfa",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "775496bb-f633-46f4-a642-10e6cd29c1bb",
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:43.365656Z",
     "iopub.status.busy": "2022-01-25T19:20:43.365347Z",
     "iopub.status.idle": "2022-01-25T19:20:43.379933Z",
     "shell.execute_reply": "2022-01-25T19:20:43.379238Z",
     "shell.execute_reply.started": "2022-01-25T19:20:43.36562Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.04554,
     "end_time": "2021-11-27T08:16:39.234056",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.188516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'main_color' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 66>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m css_file \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'''\u001b[39m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;124mdiv #notebook \u001b[39m\u001b[38;5;124m{\u001b[39m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;124mbackground-color: white;\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;124m}\u001b[39m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;124m'''\u001b[39m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mto_rgb\u001b[39m(h): \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m(\u001b[38;5;28mint\u001b[39m(h[i:i\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m2\u001b[39m], \u001b[38;5;241m16\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m4\u001b[39m))\n\u001b[0;32m---> 66\u001b[0m main_color_rgba \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrgba(\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m, 0.1)\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (to_rgb(\u001b[43mmain_color\u001b[49m[\u001b[38;5;241m1\u001b[39m:])[\u001b[38;5;241m0\u001b[39m], to_rgb(main_color[\u001b[38;5;241m1\u001b[39m:])[\u001b[38;5;241m1\u001b[39m], to_rgb(main_color[\u001b[38;5;241m1\u001b[39m:])[\u001b[38;5;241m2\u001b[39m])\n\u001b[1;32m     67\u001b[0m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnotebook.css\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mwrite(css_file \u001b[38;5;241m%\u001b[39m (main_color, main_color, main_color_rgba, main_color,  prompt, strong_main_color, main_color, main_color, main_color, main_color))\n\u001b[1;32m     68\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mIPython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdisplay\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m display, HTML, Javascript\n",
      "\u001b[0;31mNameError\u001b[0m: name 'main_color' is not defined"
     ]
    }
   ],
   "source": [
    "css_file = '''\n",
    "div #notebook {\n",
    "background-color: white;\n",
    "font-family: 'Open Sans', Helvetica, sans-serif;\n",
    "line-height: 20px;\n",
    "}\n",
    "\n",
    "#notebook-container {\n",
    "margin-top: 2em;\n",
    "padding-top: 2em;\n",
    "border-top: 4px solid %s; /* light orange */\n",
    "-webkit-box-shadow: 0px 0px 8px 2px rgba(224, 212, 226, 0.5); /* pink */\n",
    "    box-shadow: 0px 0px 8px 2px rgba(224, 212, 226, 0.5); /* pink */\n",
    "}\n",
    "\n",
    "div .input {\n",
    "margin-bottom: 1em;\n",
    "}\n",
    "\n",
    ".rendered_html h1, .rendered_html h2, .rendered_html h3, .rendered_html h4, .rendered_html h5, .rendered_html h6 {\n",
    "color: %s; /* light orange */\n",
    "font-weight: 600;\n",
    "}\n",
    "\n",
    ".rendered_html code {\n",
    "    background-color: #efefef; /* light gray */\n",
    "}\n",
    "\n",
    ".CodeMirror {\n",
    "color: #8c8c8c; /* dark gray */\n",
    "padding: 0.7em;\n",
    "}\n",
    "\n",
    "div.input_area {\n",
    "border: none;\n",
    "    background-color: %s; /* rgba(229, 143, 101, 0.1); light orange [exactly #E58F65] */\n",
    "    border-top: 2px solid %s; /* light orange */\n",
    "}\n",
    "\n",
    "div.input_prompt {\n",
    "color: %s; /* light blue */\n",
    "}\n",
    "\n",
    "div.output_prompt {\n",
    "color: %s; /* strong orange */\n",
    "}\n",
    "\n",
    "div.cell.selected:before, div.cell.selected.jupyter-soft-selected:before {\n",
    "background: %s; /* light orange */\n",
    "}\n",
    "\n",
    "div.cell.selected, div.cell.selected.jupyter-soft-selected {\n",
    "    border-color: %s; /* light orange */\n",
    "}\n",
    "\n",
    ".edit_mode div.cell.selected:before {\n",
    "background: %s; /* light orange */\n",
    "}\n",
    "\n",
    ".edit_mode div.cell.selected {\n",
    "border-color: %s; /* light orange */\n",
    "\n",
    "}\n",
    "'''\n",
    "def to_rgb(h): return tuple(int(h[i:i+2], 16) for i in (0, 2, 4))\n",
    "main_color_rgba = 'rgba(%s, %s, %s, 0.1)' % (to_rgb(main_color[1:])[0], to_rgb(main_color[1:])[1], to_rgb(main_color[1:])[2])\n",
    "open('notebook.css', 'w').write(css_file % (main_color, main_color, main_color_rgba, main_color,  prompt, strong_main_color, main_color, main_color, main_color, main_color))\n",
    "from IPython.core.display import display, HTML, Javascript\n",
    "def nb(): return HTML(\"<style>\" + open(\"notebook.css\", \"r\").read() + \"</style>\")\n",
    "nb()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "75d3e80e-0c31-48f5-88dd-5b6a3c79cf38",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "968efaca-ede8-4817-b3aa-baedaba3205e",
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:43.381517Z",
     "iopub.status.busy": "2022-01-25T19:20:43.38132Z",
     "iopub.status.idle": "2022-01-25T19:20:43.391648Z",
     "shell.execute_reply": "2022-01-25T19:20:43.390861Z",
     "shell.execute_reply.started": "2022-01-25T19:20:43.381494Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.040618,
     "end_time": "2021-11-27T08:16:39.304489",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.263871",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML, Javascript\n",
    "# def nb(): return HTML(\"<style>\" + open(\"../input/starter-utils/css_oranges.css\", \"r\").read() + \"</style>\")\n",
    "def nb(): return HTML(\"<style>\" + open(\"notebook.css\", \"r\").read() + \"</style>\")\n",
    "nb()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030175,
     "end_time": "2021-11-27T08:16:39.364837",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.334662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<center><h2> <span class=\"title-section w3-xxlarge\" id=\"codebook\">1st Place of Jane Street 🏆 ➜ Adapted to Crypto 👌</span></h2> </center>\n",
    "<br>\n",
    "<hr>\n",
    "This is a simple starter notebook for Kaggle's Crypto Comp showing purged group timeseries KFold with extra data. Purged Times Series is explained [here][2]. There are many configuration variables below to allow you to experiment. Use either GPU or TPU. You can control which years are loaded, which neural networks are used, and whether to use feature engineering. You can experiment with different data preprocessing, model architecture, loss, optimizers, and learning rate schedules. The extra datasets contain the full history of the assets at the same format of the competition, so you can input that into your model too.\n",
    "\n",
    "**NOTE:** this notebook lets you run a different experiment in each fold if you want to run lots of experiments. (Then it is like running multiple holdout validaiton experiments but in that case note that the overall CV score is meaningless because LB will be much different when the multiple experiments are ensembled to predict test). **If you want a proper CV with a reliable overall CV score you need to choose the same configuration for each fold.**\n",
    "\n",
    "**See Original Links:**\n",
    "- https://www.kaggle.com/gogo827jz/jane-street-supervised-autoencoder-mlp\n",
    "- https://www.kaggle.com/c/jane-street-market-prediction/discussion/224348\n",
    "\n",
    "**Encoder + MLP**:\n",
    "The main idea of using an encoder is the denoise the data. After many attempts at using a unsupervised autoencoder, the choice landed on a bottleneck encoder as this will preserve the intra-feature relations.\n",
    "![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAtwAAAIiCAYAAAD7MR8AAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAFiUAABYlAUlSJPAAAE7dSURBVHhe7d0trx1ZdsbxAQ1CzYZEiuT5AAMMIoU0MwkY5oCAASFmkcJMLhhkNNRsQBSZRBpkbGAYRW0psGUFBsQ8YMDNfdz3ub16nV11quqcOrWr1v8nbfmeetm1623Vc6uP3b+6BwAAALAaAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwAAALAiAjcAAACwIgI3AAAAsCICNwDsyNu3b+9/9atfNdunT58elwIA9ITADQA78/Xr1/vXr1/vKmhrzK9evXr8BAC1ELgBYIcUshW2X758+Tilb+/fv9/NWAHg2gjcALBDewrcerv97NkzAjeAsgjcALBDewncCtsvXrzY1dt4ALg2AjcA7NBQ4P7y5cv9u3fvvoVczVPg1V+0fP78+dPymmat5eP3w/W9ay1jnu6+TD/HeaL1vN3c9vC9cwC4FgI3AOzQUOBWcNbXNzRPP7958+b+8+fP3+bpZ02Pf3kxBmItr7DtgO1/EUX9xZCu9TU9b/vDhw9PfUXuJy8PAFUQuAFgh8a+UuKAqwAdKTSPBeK8vDhcK6zbWIAe65/ADaAqAjcA7NCUwH2NQOy31noTbgRuAJiHwA0AO3SrwN16K07gBoB5CNwAsEO3CtyS1yFwA8A8BG4A2KFbB279xUkjcAPAPARuANihWwVu/Qsnmhf/ZRN/r5vADQDTELgBYIfWCNzxL0aa/01ubc+Gtu1wrhb/7e6xgA4AFRC4AWCH4v+cJoZh8T/lpwAd//3s+O9k6392Yw7caurX62gZTYv/JKBovv+tb2/7/fv3T//Ot5q27fUcxD0eLZvHDABHRuAGgB2J4Tg3v3nOTevo7XKe7jfO8Y24fo7/4xyF4xaFdwVoL6fPos8K/Fovhn2Hca3jZQGgCgI3ABQXAzcA4PoI3ABQHIEbANZF4AaA4gjcALAuAjcAFDf0lywBANdB4AaAoob+kiX/gggAXBeBGwAAAFgRgRsAAABYEYEbAAAAWBGBGwAAAFgRgRsAAABYEYEbAAAAWBGBGwAAAFgRgRsAAABYEYEbAAAAWBGBGwAwyv8Hyo8fPz5OAQDMQeAGAIwicAPAZQjcAIBRBG4AuAyBGwAwisANAJchcAMARhG4AeAyBG4AwCgCNwBchsANABhF4AaAyxC4AQCjCNwAcBkCNwBgFIEbAC5D4AYAjCJwA8BlCNwAgFEEbgC4DIEbADCKwA0AlyFwAwBGEbgB4DIEbgDAKAI3AFyGwA0AGEXgBoDLELgBAKMI3ABwGQI3AGAUgRsALkPgBgCMInADwGUI3ACAUQRuALgMgRsAMIrADQCXIXADAEYRuAHgMgRuAMAoAjcAXIbADQAYReAGgMsQuAEAowjcAHAZAjcAYBSBGwAuQ+AGAIwicAPAZQjcAIBRBG4AuAyBGwAwisANAJchcAMARhG4AeAyBG4AwCgCNwBchsANABhF4AaAyxC4AQCjCNwAcBkCNwBgFIEbAC5D4AYAjCJwA8BlCNwAgFEEbgC4DIEbADCKwA0AlyFwAwBGEbgB4DIEbgDAKAI3AFyGwA10yAGHRqPRaMdsf/rTnx4rPiogcAMdahVnGo1Gox2nEbhrIXADHXJBBgAcy+9//3sCd0E80YEOEbgB4JgI3DXxRAc6ROAGgGMicNfEEx3oEIEbAI6JwF0TT3SgQwRuADgmAndNPNGBDhG4AeCYCNw18UQHOkTgBoBjInDXxBMd6BCBGwCOicBdE090oEMEbgA4JgJ3TTzRgQ4RuAHgmAjcNfFEBzpE4AaAYyJw18QTHegQgRsAjonAXRNPdKBDBG4AOCYCd0080YEOEbgB4JgI3DVt/kR3sLhWu7u7e+x5nlZfS9oSGnOrr7mNfZ+v1deStgT73u5vTmPf52v1taQtwb63+5vTlu47+kHgrmlZ5biiXEwubRTi+Vp9LWlLsO/t/uY09n2+Vl9L2hLse7u/OY19x54RuGtaVjmuKBeTSxuFeL5WX0vaEux7u785jX2fr9XXkrYE+97ub05j37FnBO6allWOK8rF5NJGIZ6v1deStgT73u5vTmPf52v1taQtwb63+5vT2HfsGYG7pmWV44pyMbm0UYjna/W1pC3Bvrf7m9PY9/lafS1pS7Dv7f7mNPYde0bgrmlZ5QCwKj9cAQDHQuCuiSc60CECNwAcE4G7Jp7oQIcI3ABwTATumniiAx0icAPAMRG4a+KJDnSIwA0Ax0TgroknOtAhAjcAHBOBuyae6ECHCNwAcEwE7pp4ogMdInADwDERuGviiQ50iMANAMdE4K6JJzrQIQI3gBbXhtiwLwTumrhTgQ7xIAXQ4toQG/aFwF0TdyrQIR6kAFpcG2LDvhC4a+JOBTrEgxRAi2tDbNgXAndNm9+pd3d3Jw2o6vvvvz95mGoaAEiuD2rYFwJ3TZvfqblwqAFVffz48eR+0DQAkFwf1LAvBO6aCNxAZ+Jbbt5uA4jis9IN+0LgronADXQmvuXm7TaAKD4r3bAvBO6aCNxAh/Rmm7fbADKemftH4K6JwA10SG+2ebsNIOOZuX8E7poI3Bto7TONRqMtaaiFa2D/CNw1Ebg3UGEfAayPWlJPfFa6YV8I3DURuDdQYR8BrI9aUk98VrphXwjcNRG4N1BhHwGsj1pST3xWumFfCNw1Ebg3UGEfAayPWlJPfFa6YV8I3DURuDdQYR8BrI9aUk98VrphXwjcNRG4N1BhHwGsj1pST3xWumFfCNw1Ebg3UGEfAayPWlJPfFa6YV8I3DURuDdQYR8BrI9aAuwPgbsmAvcGKuwjgPVRS4D9IXDXRODeQIV9xE++fPly/+zZs/sXL148Tqnnw4cP3675N2/ePE7BtVBLgP0hcNdE4N5AhX2c6uvXr/fv3r27f/ny5S+uAQVUBTQF1vfv399/+vTpcY196SFw52Or8Xz+/Plx7i/pOMdl3d6+ffu4xHwE7vXouALYFwJ3TZtX67u7u5N2dDwkf6Jw9/z5828BUIFO4dQUCF+/fv3tWKntNXD3Qr/Y6BcXHWsdT/0CoGlDHLx1DtAvagmwPwTumqjWG+Ah+fNbTwXuGLQzhUQtpz9xOf1io+Op9urVq8epbXozzi86faOWAPtD4K6Jar2B6g9Jf81Cx2Hoqw2R3rJe8pUG/EzHUW+3dezVxo4rgbt/1WsJsEcE7pqo1huo/pD0d4qnfl1BAZ3AfR06jm46B2r6rw0tBO7+Va8lwB4RuGuiWm+g8kNS4flc0JtKb8f1tQi/Ldef+uyvqPh7yG4xtMfpCpaR+9U8feVF49Rf7MzhU/1pvpbTLw/6TnT8mobG4b8Qmrch58YvWkZ/2VDb0fY0T9vyOnP/IqLDtngf1VfrvzSMBW5N9/pqHl/+XrjHr7fq3q5NPc6irxTFN/P5OFWlYwFgXwjcNVGtN1D5IalA5dDUCnlTKWwpKMa//Kcgqn41zTTPoTgHPgVBTc9h2OFRtL77jUFQAVDLOfRpnsYT+4rBOG9jyvh1fHLI1Li0rtZRn5quYzqV1o/75v7jOGwocPsvX/p79VpPfbb6ieP3dm3KcRZNVz8+1r6GNIY85mp0HADsC4G7Jqr1Bio/JB3Mzh0DL5ObA5r7iSFO4cvLRQ6mcVlRsNP0GIY9LQc5hcMYBLVOfrusABr7Ev/l0Dx9yfjz9ob6HqPtxW0q1PuXAgX6SP3m8KvQq2VbId+/wCggR54etzv1OHsfHbbN4TxvqxodAwD7QuCuiWq9gcoPSQdNtXNvJxWyFMC0rEJhDF0OYvozavXtwBoDnzj0xcDqaQpysY/8VQetozHFaZJDa2sbstb4z1EfuR/9ouDtxnnqN++fg24OwKIxu584X33mvj32c8dZx1NvtzP3qXNQmY4BgH0hcNdEtd5A5Yekg5JaDnMtXn4sVCqwKajFry/EvucG1hjy9Xa2FS5jSNX6/npFNrSN6NrjH6M+cj/it9Bq/iVA/cZxiI/NEI81rudzmLc75ThrnpYZa5VV339gjwjcNVGtN1D5Iek3u2qtryVkDmutUKmgGr9aoK9HuO9LAmvs102BMNO+ODSq6ef8vfSxULzW+Meoj9yPeTsKuRqLPsdxiMc3xH3E9XwO83anHGdNm7N/1ej4oJaK/7O4oyFw10S13kD1h6RDqv48x2Ethy5/9zj+ZTrRsmrXCKzqNwbCVugWvd2Ob2vjeIa2cYvxt6iP3I8pAHs/NC61OA7xG+f8i4V5rHE9n8Oh7Y4dZ33WNocM9VmFjg9q8X0SG/aFwF0Td+oGqhfIOW+5HdZyqHQwzMHP/cbApwCnaTmctQKrpuWvh2gb7tcUEDN93zhvp7UNmTN+h9gp4z9HfeR+Iv8i0BqHtPYx0lhyQNayeZ2px3lo3y1/Z76aeKxQg++R2LAvBO6auFM3QIH85XeGh77/LA5rOVR6Xb2VjTw9BsVW4BOFfU2Pb9q1nt7sZjlI6nP+C48OwPGXCE+7ZPxDoXOo7zHqI/eT6Xy0xiHepo5FHrs+a3ruX5/z+NXPlOMcx6Jrxv81wG/Fz+3L0em4oBbfD7FhXwjcNXGnboAC+ROFKYUrHQ+9qYwBVoFK8/0mOL9Rdgj19Py1BAU6B1+HRPWl5RQMFdTi8gp/2p6XVf8Od54Ww52DobehPrUPOYjGXyzi2+yp41dfPgb5be5Q30O0De2nj8MY961xZA7B8Ssn7tv7Yxq/91Xb9bGZepwlHpfYtL14rCvScUAt+T7gGtgfAndNm9+puXCoHV2FfZxKgUnB0gE2XgeapuDXCpOaFoOcQpr68lceNC+up/kx3GueAp6mKdA57GuaAqWaAp37j2+tRes4JMd+Y5DVtNwcJqeMX9uM67o5mOaWg2rkbeU2JgbqTMcr9qlldcyioXFq+tTjbJqu+VrO56x62BYdD9Ti+yg27AuBu6bN79RcONSOrsI+Yv8UaAm1faOW1BOflW7YFwJ3TQTuDVTYRwDro5bUE5+VbtgXAndNBO4NVNhHAOujltQTn5Vu2BcCd00E7g1U2EcA66OW1BOflW7YFwJ3TQTuDVTYRwDro5bUE5+VbtgXAndNBO4NVNhHAOujltQTn5Vu2BcCd00E7g1U2EcA66OW1BOflW7YFwJ3TQTuDVTYRwDro5bUE5+VbtgXAndNBO4NVNhHAOujltQTn5Vu2BcCd00E7g1U2EcA66OW1BOflW7YFwJ3TQTuDVTYRwDro5bUE5+VbtgXAndNBO4NtPaZRqPRljTUwjWwfwTumgjcQIe4FwC03N3dnTTsC4G7JgI30CHuBQA4JgJ3TQRuoEPcCwBwTATumgjcQIe4FwDgmAjcNRG4gQ5xLwDAMRG4a+KJDnSIwA0Ax0TgroknOtAhAjcAHBOBuyae6ECHCNwAcEwE7pp4ogMdInADwDERuGviiQ50iMANAMdE4K6JJzrQIQI3ABwTgbsmnuhAhwjcAHBMBO6aeKIDHSJwA8AxEbhr4okOdIjADQDHROCuiSc60CECN4AW14bYsC8E7pq4U4EO8SAF0OLaEBv2hcBdE3cq0CEepABaXBtiw74QuGviTgU6xIMUQItrQ2zYFwJ3TZvfqXd3dycNqOrTp0/3Hz9+fHqQ6mc1ABDXhtiwLwTumja/U3PhUAOqciGO7Xe/+93jXADV5fqghn0hcNdE4AY68uOPP95/9913v7gffvjhh8e5AKqLtcEN+0LgronADXQmvuXm7TaAKD4r3bAvBO6aCNxAZ+Jbbt5uA4jy81IN+0LgronADXRIBZm32wAynpn7R+CuicANdEhvuXm7DSDjmbl/BO6aCNwbaO0zjUajLWmohWtg/wjcNRG4N1BhHwGsj1pST3xWumFfCNw1Ebg3UGEfAayPWlJPfFa6YV8I3DURuDdQYR8BrI9aUk98VrphXwjcNRG4N1BhHwGsj1pST3xWumFfCNw1Ebg3UGEfAayPWlJPfFa6YV8I3DURuDdQYR8BrI9aUk98VrphXwjcNRG4N1BhHwGsj1oC7A+BuyYC9wYq7COA9VFLgP0hcNdE4N5AhX0EsD5qCbA/BO6aCNwbqLCPANZHLQH2h8Bd0+bV+u7u7qQdHQ9J3MLnz5/v37x5c//8+fP7t2/fPk7FkVBLgP0hcNdEtd4AD8k+6DzMaXsKrZ8+fbp/8eLFLseO6XRuAewLgbsmqvUGeEj248uXL9/eAI+FUi2j8LrH0Pry5UsC94FRS4D9IXDXRLXeAA/JvkwJpXpjTOBGb6glwP4QuGuiWm+Ah2RfpoZSAjd6Qy0B9ofAXRPVegM8JPty5FBK4D42agmwPwTumqjWG+Ah2ZdzoVT/0kf24cOH+9evX98/e/bs29dN1F69evWtH017//7945K/pO+Dez0tq++Gq6+W2Kea/7WRr1+/Pi7xS/pXSfIYxvZN8+NfrNS6Gp/lf+VE49TP6ntozLgtnTcA+0LgrolqvQEekn0ZC6UKoAqY0bt3734RVBVKNU1B2MtrugJrpM+ap2VFy7ufHGAVhh2aRctqfFpW6+TQrfU1z4E8B/u8b5qnfhywNSYtp+W1fv5XThTGNZY8LmxL5wbAvhC4a6Jab4CHZF8cuMdai+flsKwArukx5CrEKqgq6EYO0RqDKQRrmoN55L5jP+5boTjy9DwWh3OHbVOfuW8fmzg+9EPnBsC+ELhrolpvgIdkX8becOtNr0Jri9ZR0zKRQ3Tsz2+QczhXKNbXNGLIdfDNgVi0vLfr+d5e7lta+6ZgrrfXmfuJ+zt2bLA9nRvUUvF/Fnc0BO6aqNYb4CHZl3OhsvUdbtE6alMCt79bnZdt8b8LPsTjdV/+6ofCeNbaN7/1HmtG4O5bPFeoId6nbtgXAndN3KkboED2ZWmo9MNuSuDOIXmM+x2S+xpbvrVv+qzpUyw9NriNofOO49I5zw37QuCuiTt1AxTIviwNlX7YzQncQ2/LI7+Bzn/p0txXDtyt5Vv7ps9DX5OR1rjnHhvchs4NatE5zw37QuCuiTt1AxTIvkwNlfrKRgzMfthNCdz5XwHJ9K9+uB9//WRoPBpvDMz+SknrXw5p7du5/Y1/+ZLA3TedG9Sic54b9oXAXdPmd2ouHGpHV2Ef92RqqNRfZoyh1tfrlMDtf3lETQE5vo1WAI9/iVH9ablWONdnTY99O8zru995eYfxuLz2wWPRLxD+y5f6U/sYlyVw903nBrX43o0N+0LgronAvYEK+7gXCr7+S4pDX/fQMnrrG98qa5qv1/jP9ynwOqTmAByDbm757bSXVWB2oFcg1uf4L5qYg7X+jP96ifdNTT+b/yWU3LS+x6w/zx0bbEvnBrXE+9UN+0LgronAvYEK+7gH+bo71xx0FWRb8/1mujXd9E/3ORyrKcjnN+SmZR3e1bReDuamcKxx6ZcCL+v19bPm5e946xcFB2qtp/1z2J6yL9iezglqyfck18D+ELhrInBvoMI+AlgftaSe+Kx0w74QuGsicG+gwj4CWB+1pJ74rHTDvhC4ayJwb6DCPgJYH7WknvisdMO+ELhrInBvoMI+AlgftaSe+Kx0w74QuGsicG+gwj4CWB+1pJ74rHTDvhC4ayJwb6DCPgJYH7WknvisdMO+ELhrInBvoMI+AlgftaSe+Kx0w74QuGsicG+gwj4CWB+1pJ74rHTDvhC4ayJwb6DCPgJYH7WknvisdMO+ELhrInBvoMI+AlgftaSe+Kx0w74QuGsicG+gtc80Go22pKGWu7u7k4Z9IXDXROAGOsS9AADHROCuicANdIh7AQCOicBdE4Eb6BD3AgAcE4G7JgI30CHuBQA4JgJ3TTzRgQ4RuAHgmAjcNfFEBzpE4AaAYyJw18QTHegQgRsAjonAXRNPdKBDBG4AOCYCd0080YEOEbgB4JgI3DXxRAc6ROAGgGMicNfEEx3oEIEbAI6JwF0TT3SgQwRuADgmAndNPNGBDhG4AeCYCNw18UQHOkTgBtDi2hAb9oXAXRN3KtCh/ECl0Wg02nXa3/zN3zxW2m0QuGsicAMdyg8IGo1Go12nEbixBQI30CE/GAAgcm2IDdP893//97fjReDGFja/U+/u7k4aUB0PUgAtrg2xYRoCN7a0+Z0ai4YbUB33AoCW+KykTsxD4MaWCNxAh7gXALTEZyV1Yh4CN7ZE4AY6xL0AoCU+K6kT8xC4sSUCN9Ah7gUALfFZSZ2Yh8CNLRG4gQ5xLwBoic9K6sQ8BG5sicANdIh7AUBLfFZSJ+YhcGNLBG6gQ9wLAFris5I6MQ+BG1sicAMd4l4A0BKfldSJeQjc2BKBG+jI+/fvvxVh3wv6+V//9V8f5wKoLj4r3TDuL3/5y7ew/enTp2/H69e//vX9x48f7//jP/7jcYnbInDXROAGOvLP//zPJ/eDijMASK4Pajjvb//2b0+O263/z9bxZYrbd999d//jjz8+LoEjI3ADHfmf//mf+7/6q796uhcoxgCi+Kx0w3l6ox2Pmers//7v/z7OvQ29af/Nb37zi3HwQqUOAjfQmfiWm2IMIIrPSjdM8/d///dPx0x1dgvxLTcvVGohcAOd8VtuijGAjGfmcv/1X//1ra6q6TvdW4hvuXmhUguBG+iQ3r5QjAHgulRX//Ef//Hx0zb0lpsXKvUQuIEO6S03xRgArku19Ycffnj8tA295f7DH/7w+AlVELg30NpnGo1GW9KAuVrXEY1Wsd0SgXsDFfYRwPqoJViC6wa4/X2w+V2nfwczt6Oj2AG4BmoJluC6AQoG7ooodgCugVqCJbhuAAJ3CRQ7ANdALcESXDcAgbsEih2Aa6CWYAmuG4DAXQLFDsA1UEuwBNcNQOAugWIH4BqoJViC6wYgcJdAsQNwDdQSLMF1AxC4S6DYAbgGagmW4LoBCNwlUOxwC1+/fr1///79/atXr7jmDorziiW4bgACdwkUu+29fPny23kYay9evPi23Lt37+6/fPnyuOZ+KGg/f/78aX9wPJzXeq7xP4vjuhn36dOnp7rp9vr168e50+j5EdfXs2TsuaNtDmmNJ7Znz559q/d6wYLpdOxuibtuA7c+yWjTG2AXQBXHz58/P865/xawVbxi0Xz79u3j3P2IhRrHw3mtx/dzbHMtWacivWyJx3nqi5dYdxWE9ayx+NzRC5H43Dnnw4cPT/3qZ/F/yVTo1vS5vxhUpuN1S9x1G7j1ScYwhWidDxXAIV5G7c2bN49T94HAfWyc13p8P8c215J1qorHeepLFz1PHIBbb66nPHeGeCy531jr9/hyaAs6VrfEXbeBW59kDJta+PTWwMWsVUB7ReA+Ns5rPb6fY5tryTpV6Vi5/itEx7fVLXpj7a8jap1bBW7xVwj1J87Tsbol7roN3PokY9jUwqci6zcWS4rkVgjcx8Z5rcf3c2xzLVmnKh0r1VHX/3NvjxXO9RWPLQK3t6mG8259nDY/K744Yju6Cvu4F3MKn75O4ms0f5dPxc//Goia3nC0CqLkZfVz63t8CvkaX/yLj1p2qF+Jy2uf4nf+Mu2DHg5+kGg9fWcx0vpeRttV//pZy+7xL5IeTeu84th8P8c215J1qtKxcu3Tz6p/Q2+5VRP9dnmLwO2/c+QxYJyO1S1tftf54ont6Crs417MKXx6a+Fr1H9hRfwXVlwA9adDbC6KeVkFbX1WiwFWBV3FUwHb07WOC6r6yTQvBn2N0eFbLfJ2HbC1DfetYyL609PUFLy1nn7x0LqtXxJwW/m84vh8P8Y215J1qtKxUk2N/5Uzv5gwv92WWwdu1WPPcw3HOB2rW9r8rvMFEtvRVdjHvZhT+FTgfI26oCmoqgjnAKzPWi6+afD6eVkX5lgkVbi1bn6T4u1p+RjQvR85BHu6WqQgrW1E6s/Lxr49LY4PfcjnFcfn+zG2uZasU5WOlcOt62nrDbIDua0duONzRC9X/FxQbR96A49f0vG6pc3vOl88sR1dhX3cizmFrxW4/c9G5QIXl3UI9tdI8rJ+M+EC6uCbA7H5qy2e70KvQpvFcZi3F9/Sm5eNb3A8rfXgwLZ0XlCL78fY5lqyTlU6Vq59rrWall+c6Jng54KsHbhV7+N/wdRn9UvYnk7H7ZY2v+t8scR2dBX2cS/mFD6/tVZzEY3fxR5qXtaF+hxvJxbvSP1pvses4KzPrX+y0MvG7fqXhLEWt+1prQcHtqXzglp8P8Y215J1qtKxirVPLzo0Lb7lVsjN/0Vy7cBNPb6cjuMtbX7X+eKJ7egq7ONezCl88S9NurC6qE7hdc/xmKYG7rHlvWzcrpefWrC9PgW+P/G8ogbfj7HNtWSdqnSsYu2LX73zW269xMgvPAjc/dNxvKXN7zpfPLEdXYV93IuphS/+p8S4rItq/u606e2z52k5tdZXOSK/gdbb8xYV2jgO70NreS+rZl4+fm0k0r7ylZJ9iOcVNfh+jG2uJetUpWOVa5/fcrsG6+12/Hsv4mdDq27Oee5o2UjrDfWLeXQcb2nzu84XT2xHV2Ef92Jq4XOBVYvh+tz6Ws9vw/31k7Flxd+xVsD3upFDtEOxv1LS+os8XlbNPG2of721ib8UeH0KfH/ieUUNvh9jm2vJOlXpWOXaF99yq267dkfXCNxaLq/v7VKPL6fjeEub33W+eGI7ugr7uBdTCl/8Kkn+izKx8KoPh3G/JY79xu+Aq0DHNyJaL76hdrHW+DJNy2HZb9/zW2uHcbXIf9lGf8ZwrZ/zdxG9PgW+P/m84vh8P8Y215J1qtKxatU+1XAf//x2W1zDlwZu1WDV9czbpB5fTsfxlja/63zxxHZ0FfZxD1TQXBT1N7zjm2sVUAVkTdd8Fb4YTKMYpGPTOrFPiUV6bFmNzdtWcXYAVqBu9RuDtUO3Q7ynK0g7wGue+vG82OIvFVrO04f2H9vReUEtvh9jm2vJOhW5rrZefCjwal7rq3yxvmp+fIERnzv55YZpu6r/OZB7m0Njwjw6jre0+V3niye2o6uwj71zwRtrKoYqlgqgraIYqRA6IKtpvRyKTYVSfWs5FeX8ttu0Tc2bsqxoDN4vL6vC7Z/zfqifGMg1/hiqNU7Piw394HzUk+/HJdfAknUqicE2thxyVW+1bNRaT03LTnnuxObtDY1HjeC9nI7fLW1+1+WL59YHYAsV9hHA+qgl9cRnpdtcS9YBjubW98Hmd10sGm5HV2EfAayPWlJPfFa6zbVkHeBobn0fbH7XxaLhdnQV9hHA+qgl9dzd3Z20ubhugNvfB5vfddrh3I6uwj4CWB+1BEtw3QC3vw82v+u0w7kdXYV9BLA+agmW4LoBbn8fbH7XaYdzO7oK+whgfdQSLMF1A9z+Ptj8rtMO53Z0FfYRwPqoJViC6wa4/X3AXbcBih2Aa6CWYAmuG4DAXQLFDsA1UEuwBNcNQOAugWIH4BqoJViC6wYgcJdAsQNwDdQSLMF1AxC4S9BJptFotGs0YK7WdUSjVWy3RLUGOrRFMQAArI/6XhNnHOgQBRkAjon6XhNnHOgQBRkAjon6XhNnHOgQBRkAjon6XhNnHOgQBRlAi2tDbNgXzltNnHGgQxRkAC2uDbFhXzhvNXHGgQ5RkAG0uDbEhn3hvNXEGQc6REEG0OLaEBv2hfNW0+Zn/O7u7qQB1VGQAbS4NsSGfeG81bT5GY9Fg4sQ+An3AoCW+KykTuwT560mAjfQodZ9QaPR5rU//elPj3fUcbT2E/vCeauJwA10qHVf0Gi0eY3AjR5x3moicAMd4l4Alvv973//7f4hcKNHnLeaCNxAh7gXgOUI3OgZ560mAjfQIe4FYDkCN3rGeauJwA10iHsBWI7AjZ5x3moicAMd4l4AliNwo2ect5oI3ECHuBeA5Qjc6BnnrSYCN9Ah7gVgOQI3esZ5q4nADXSIewFYjsCNnnHeaiJwAx3iXgCWI3CjZ5y3mgjcQIe4F4DlCNzoGeetJgI30CHuBWC5Iwdu7B/1vSYCN9Ah7gVgOQI3ekZ9r4nADXTk+++/P7kffvvb3z7OBTAFgRs9c21HLQRuoCMfP348uR/+/Oc/P84FMAWBGz1zbUctm5/xu7u7kwZUFt9y83YbmI/AjZ4RuGvijAOdiW+5ebsNzEfgRs8I3DVxxoEO6S03b7eBZQjc6BmBuybOONAhveXm7TawDIEbPSNw18QZ34BvNhqNRru04RSBGz3j3q2JM74BbjQA10AtaSNwo2cE7po44xvgRgNwDdSSNgI3ekbgrokzvgFuNADXQC1pI3CjZwTumjjjG+BGA3AN1JI2Ajd6RuCuiTO+AW40ANdALWk7cuDmfxa3fwTumjjjG+BGA3AN1JK2Iwduh7XYsC+ct5o44xvgRgNwDdSSNgI3esZ5q4kzvgFuNADXQC1pI3CjZ5y3mjjjG+BGA3AN1JI2Ajd6xnmrafMzHotGlYuwwj4CWB+1pI3AjZ5x3mra/IzHolHlIqywj/Lhw4f7169f3z9//vz+06dPj1O39ebNm2/HX2Mb8vnz5/tXr149XY/6WdNevHhx/+zZs/svX748LrkNjV3j0r6gtiq1ZC4CN3rGeatp8zMei0aVi3CP+/j169f7d+/e3b98+fLpPCl8Koy+f//+2zKaZwrYCtpedi+BW9O1XwrY2mf9wuB9JXD/ks6pz29u8VrAenSscYrAjZ5x3mra/IzHolHlItzbPjo8K2y+ffv2F4FT4S+G8EwhVdN7CdznaB9zWNRn7cet6Vjv4bj5FwA1jVm/qOA2WvccCNzoG+etps3PeCwaVS7CPe2j32L6re8QvwnOb38dxvcUHBUae9DTV3HO0XHby3k+Eh1znCJwo2ect5o2P+OxaFS5CPeyj3pTqaCt8fprI2NaAXFPgVtBW2PtIXB7LARujNExxykCN3rGeatp8zMei0aVi3Av++i31grSUyiUE7gvp+OocezluMnexnsUOuY4ReBGzzhvNW1+xmPRqHIR7mUf/XZbwXupscCtYOvveKvpZ/3FzEzL+S9gaix6866/rBnFf1lEy+rrIeorblfTtH5+E+8xtpq2re0pBKt/HZMW9eftq+nn1ldw8nfeNRb95Ud/79mhPzetI+pTy+tYtX4xUD/xeKlpLPn4ux8tp+X1VSAdG5/zuX8h09tqnecxGq/PifvQvrof/enpcX4U58Xt52OhfYvHWvJx0PnRz1pWP/dO+4VTBG70jPNW0+ZnPBaNKhfhHvYxBp1WsJvK4TIHMYUcTfdXVRR8HPZi0NF8BSB/N1z9aLkcuhyYxCEubld/xnCfxyNaX/Py/iq4O7SpZRqjxuQ+vS9q+S+Yan2NTTROjykHXG8rjzPuQx6n+1PAjsfL68RjHfvR8upL66gPn7PWLz9D3FfruI7ROHScNCbRGNVPPHbqU581vfWLgMact+1joWOtn9V8TfjaicdGTcdB2/f59PHqmcaNUwRu9IzzVtPmZzwWjSoX4R72UWHE52NuiIqGArf7jhzCY5DU+jlkKQjFwO2xKlRFCsl5uw5YrX0aCtziUKcWeds5nHm/875oWty2Q3jcH9G0vKy1jpP4TXE+DgquDqzxFwCPJx/foTGN0fJqrfEO8bHL22mdIx3foTFpnn+JMe2TjkXmX5zi+fJxmLO/vdC4cYrAjZ5x3mra/IzHolHlItzDPjoMqc0JUZnDTO5DoUotcuCNAVDrKyzm9fU20jxWv820/JUSGRqPjAVu0Ty1SOPQtBxy9cZW02Ow037FN7fisedj4W1NHaf61LQcPM0hPc73scj76zHNCaBaXq013iH+RSAHfo8rf6XDYTlvQ8vnaa1+xX3H62foOOyBxo1TBG70jPNW0+ZnPBaNKhfhHvbRoUstv72dw2FmLIhpnoKg38JqHfObTU8fGovDmINWDLXR2HhaQTbyOCKPeS6NT+P0uHMfnjZ1nD5OQ2P3+YzH1scir9Na9hwtr9Ya71TaB/8C0xqX9zuGZR3H/Cbbv+yMtSnHYQ80bpwicKNnnLeaNj/jsWhUuQj3sI/xKxSXBBGHmVYQ0xtMhSU1hS2/hc1Bz8t5PPrZ3/k1jVeh3cuojb3hnBpkI/cbtaaN0bg1BgV1bSf+QhF52tRxnht7K0T7WOR1Wsueo+XVWuM9R9vX8dD2dK6HxqVz7F9w/AuVznH+rrnHP3QssqHt7YHGjVMEbvSM81bT5mc8Fo0qF+Fe9tFBJH/dYQ73kYOYw3EMSwo8mqZ1WhROHbwVvFpvsTUtBu8cuofGI97+UPByn5Gn5a8/tDhca3z+CorD4VC/U8ep46hp8e1v5O3EY+tjkfe3tew5Wl6tNd4ong8dA11bOpdxvaFxifddx1B0PfhYmsc/dCwk9j22vd5p3Dh15MB9d3d30rAvuja5d+vZ/Iz7wovt6Payjw4ualMCpd7eKlRGDjMxUDl45kDkMBWDnoNV5K8dOCCp77zd+LWCqDUe8/aHglerP48ljjny+PWLgJZTuIziMY48beo4vb/qPwdQ8XbiLzhDQdPLDu1Ti5ZXa43XNK54zv2LUX5DPTQuUR/eltYbCtVepjUenYsY/Me21zuNG6eOHLixf65PqGXzM+4LL7aj29M+OtwpyOWvcUSa1wrHDjMx+LjP/PbZ02PQ08857OfwqM+tt/BaNwfc1njM2x8KXpqnFvmXBzXtf3zrrmPiQOgx53F6eu5Xy2nanHF631rj17QcxoeW95g0fyotr9Yar2k7cVvefj6/Y/shDuqtdc3LaJ91nXi/dU50bOM4z22vZxo3ThG40TNdm9y79Wx+xn3hxXZ0e9tHBRGfG4XkGLz1s+Yr4OQ3qwqfCjxaL76JdKCLIV6hyCFTAUjTtZx+dmgSvyWN4TEGRAdeT4shamg8onnefutrCgp2mqfmsVgMgLHF/VN/3rbfxnv/vLyW8TyNT9McFv3LiX72Onmc+tn7oP32PPURxyKa56/n5GOhbWm6WlxnyNixER1bX0PaZ/M0jVnjUdM0Hyf9rL59Tk2fNV/jHxKPRW46XxaPQ/4FcA80bpwicKNnrkWohcC9gT3uo0KOAkkMMQpGCmsxRJnDVG4Wg5X7UNM0Nc0XhSNtW3/G5WMI03oKqmoxNMfwNzaeGHpzG5qvaZH6d3DTOD3uSOHRy2icGm8MfFpHn0VBN0/XfupzbvH4azktPzaWsX5a030usqHlx1rksWp6HKfOm6ZpH8beYJ8LyOpfy/i6UX9xX6Ycz95pvDhF4EbPXGtQC2d8A9xowGUUnvMvNBVRS9oI3OgZgbsmzvgGuNGA5fQGOv8XhqqoJW0EbvSMwF0TZ3wD3GjAMnqrra/jDH3VpBpqSRuBGz0jcNfEGd8ANxowXfxLnGr6/jZ+Qi1pI3CjZ65lqIUzvgFuNGA6/4XX/JceQS0ZQuBGz3Rtcu/WwxnfADcagGuglrQRuNEzAndNnPENcKMBuAZqSRuBGz0jcNfEGd8ANxqAa6CWtBG40TMCd02c8Q1wowG4BmpJG4EbPSNw18QZ3wA3GoBroJa0HTlwO6zFhn3hvNXEGd9ALJQ0Go12ScMpAjd6xnmriTMOdIiCDCxH4EbPOG81ccaBDlGQgeUI3OgZ562mzc/43d3dSQOqoyADyxG40TPOW02bn/FYNLgIgZ9wLwDLEbjRM85bTQRuoEPcC8ByBG70jPNWE4Eb6BD3ArAcgRs947zVROAGOsS9ACxH4EbPOG81EbiBDnEvAMsRuNEzzltNBG6gQ9wLwHIEbvSM81YTgRvoEPcCsByBGz3jvNVE4AY6xL0ALEfgRs84bzURuIEOcS8AyxG40TPOW00EbqBD3AvAcgRu9IzzVhOBG+gQ9wKwHIEbPeO81UTgBjrEvQAsR+BGzzhvNRG4gQ5xLwDLEbjRM85bTQRuoEPcC8ByRw7c2D/qe00EbqBD3AvAcgRu9Iz6XhOBG+gQ9wKwHIEbPaO+10TgBjrEvQAsR+BGz6jvNW1+xu/u7k4aUB0FGViOwI2eUd9r4owDHaIgA8sRuNEz6ntNnHGgQxRkYDkCN3pGfa+JMw505He/+91TMXb77W9/+zgXwBQEbvTon/7pn+6///77p9qun//hH/7hcS6OjsANdOSHH354KsZuf/7znx/nApiCwI0e6XrM9f2Pf/zj41wcHYEb6Ex8y83bbWA+Ajd69Je//OX+N7/5zVN9//Wvf33/f//3f49zcXQEbqAz8S03b7eB+Qjc6FV8y83b7VoI3ECH9Jabt9vAMgRu9MpvuXm7XQ+BG+iQ3nLzdhtYhsCNnum65O12PQRuAMChHDlw8z+L2z+95ebtdj0EbgDAIcTvx7p999139z/++OPjEvuX908NQP+4UwEAh5D/FQg1ve0+krhvbgD6x50KADiM+Jb7aG+3JQZtNwD9404FABxGfMt9tLfbksO22uFon67Zln7PvdXXkraExtzqa27b474f1OZHo0TxAADcjN5yH/HttpR4ZmqfrtkI3PO1+lrS8GTzo1GieAAAbkZvuf/whz88fjqWEs9MhUS177//Obhd0gjc87X6WtLwZPOjUaJ4AABwBaWemZVDJ4H7cDY/GqWKBwAAFyj1zCRwX94I3N3Y/GiUKh4AAFyAZyawTwRuAAB2gmcmsE8EbgAAdoJnJrBPBG5gBbqO3759+/hpmJZ7/fr146fb0LiucZ9N3UcA16P7Ljfc1vv3778d9y9fvjxOafNynz59epxyG8+fP7/4uTJ1HzEdgRtlqADF6+xcIYnLq4BNoT69zrUCdx73lHWGxL6WmruPAK7H915s+Ek8Ji9fvnycOiwuP7WW+YWF2rUCt/uLbWlI9/pLnxEyZx8xHYEb5fg6GyuwMVQuKXyt/i8pgKI+L+1DXEwv1dpHAOvSfZcbfuaQqzYWFl0Hp75MiYbe/l5Sn6/5Npw33H0icKMcFSO1sWtNxUpvSLTMtQL3pde21idwA7XpvssNP1NQdO0eq5ear+WuFbg1jcCNMZvfqTqhuQFrUjFyMRkKjJp3SQHMfav4XXpta30CN1Cb7rvc8DPVbdUlh+5WYNQyqqXXDNz6TODGGAI3ynGB1Z+t603FWi0XQBfwuI77yEVb0xxGvYxbXDYud46WnVpE8zZj0dT2PG1oGfH+u+lzpGlTxw7gOuI96YafqU6pLqlu69i0aqbqo+pdDNyxHnod96EWa10Mo3GZvGxc7hwvOyVw521qPyLtk/bBtb61jMRnmlo0Z+yj9D/eya2oze/UeLLdgDW5wLqg6M/IxbhVAF3AIhU292laJhbovF4s7nG5MVp2SuBWEY3Lab1YbD2WOGb9rGmmZWIfrWOhz1PHDuyZA06uFVu4ewgsueFnOkeuSw6Ukc6l66H+zLVbn3OdzbXO9TCG0bye62xebkirxrb42eHlvF68Nl3PPWZfv3F8WiZuKx+L1j4u8tDHSStq8z3XCc0NWFMsKrrecpFxUWoVQBfRSMvHPkTLxALdWk/ycmO0bH4QtGgssU8VUj9gpDUWF3EXbf3canH7+jx17DgmXWv5GnGL19zeObDEUIM+6Ry5LrmG53romp5DpuhzrrO5D/c7FriltdwQLxufNy2+Ft1nrt3SGkus+95Wq3n7c8Y+6qGPk1bU5nueT7YasKZYYF2EXKw0zwWmVQBj0TIVttinaJlYoFvrSVzOhTS2WET1ORfRMS7Eaq19zjRN87zeuULr5VGbr9t4rfoaUwNm03WT20S6DmNdUu3zdahrNf4ieOvArfn6HJt52fi8OSfeZ3mfx8aiZc/9Qtzax0Ue+jhpRW2+575YYgPWlAusrjlNy8W4VQBd4CIVtlafsQC21pO4nINLbBqD6XMuoqblPE734zHlh8q5sajA5m23eHnU5ustXy8OF+euI+DEw3Vz0ibS9Rbrkj67VumajNfjngJ37Nv9xHodx3duLFo2brultY+LaDu5FbX5nuuE5gasKRdYFx+1WOxaBbBVqFTYcp9aJhbAoQKXlxujZXMRNf+i4OKrsduUwO3Q5H3Vz/GzqO+4fc2fOnYcl6+deM2JrzOuEcz2cN2ctIl0HeZrTvVP12Ku07cO3GO8bKy5Fmuv7ytzzY/ja41Fn72v3lbed/Xh7c8Z+yiNNbei6u45yspFxgXLodVaBTBP82c308+xALpIalv62UUsLzdGy+YiKprmsTv8xD71WfusbWq+x6KxWy7QXia3eCz0eerYcVy+5uL1JLom8zVjutZ8Tanlh3qcn+9L0TQ1bzsuM9Z3vF8lXvNj87ydvC/5PmkdA4/Ny+T6g4aH43TSJtI5yHWpVfNE5yafjzxNn33ufE34WonXltbxuR5bboiXbd0vmu6x+/p2n15P++hlNBZNs9b162Vii/s9Z+yjHvo4aUXV3XOU40LlFh+o+tnFSuJyamMFWM3zXdjcXIBVtDxNhXFouZY87laLD5i8vB82rWLqFvfdvF5eZs7YcXy+HuI15HtE12KmeXFZX68W7ydxX2q6JjVPP2u6mqd52aG+dQ/GfjXd4xubp/70WS0GFs3X9sz3i/4Uj9N/iuuAl8EAHa/cJtCxjS2eL322eE7dfE5irVbz9e35vqbcfL35/Kupj6HlWuJyQ019Wpyu7fgai9dVvG/UWrxeXmbO2M9Sv7kVVXfPAQAXy7+AqcUgGrWWdfNDXT/H4OB1YnhymIjO9e35MbgoWMjYPMljcCjzZ8vj0ucY5EWfY99o0DHMDfvEuXxSd88BABdzGHVg1p85vJrm5QCa5UDqcOv+RUE2h/opfasftzy+sXk5cPttZl7O071ca5wE7gkejuFJwz5xLp/U3XMAwMVy4BYFSk3LxsK45WVa4XYocJ/rW9yfWu5jaN5Q4PZn8xgI3Bd6OIYnDfvEuXxSd88BABdrBW5RsMxh08uqRVo3rq/1vJxaDtGtIHuub81XUDZN07KaPjZP3Hf+HNcRfY5v2VvjJHBPoHOYG/aJc/mk7p4DAC7m8BkDsygka7palMN0XkZh1MF2SCvIyljfOTR7fPpzbJ7k+eJtxWn6HI9Da5wE7gkejuNJwz5xLp/U3XMAwEUUHhUyY4v8pjjPy8HYwVZafao5uObp2VDfCsYaT5zvsDw2z18fcYtvtf3VGbccvmNziHeLb8KRPByfk4Z94lw+qbvnAIDu5IAbW36LjoN6ONcnDfvEuXxSd88BAN0Z+rqF3h7HN8hVtX4RORztU27YJ87lk7p7DgDoisL20Fct+N7zTwjc2BXO5ZO6ew4kenDlf3VgLdrWUICI32H1f16/Rdi41XaAMfk72G7xe96VtY7N4WifcruQ/z7BLa4jb6v1X2Q0LZ471/i1/+vN2JhW9bDNk1ZU3T1HGfkBrs+eLvEvM20duBW2/UDQfI+LwA1AXBNiOxztU24D8l9GVVO41HTXs/j3ArYM3Prs5454TK1lr22zwH13d9qKGr6Kb+Tu4eDnBlyDCosKTP5P1J6uFunzrQJ3iwrirf7lAoI1sE+uXbEdjvYptwa/lMh129NjnXPgvEXgHqKwfatnDDW+P+2r+IZcMGIDLuW3HvFtQqb5sfjq85aBWwXyVoGb+wzYJ927uR2O9im3xG+tFaRbND3W/x4Ct7Z/i2eM9pXA3Z/Tq/jGXDBiAy7lNxxj//lMha9i4PaxAbA/fk7GVpH2+1y9jKGzUuDWdgjc/SFw45CWXEtaPhdDFXT3pRaLtX72dAX7WODG5omme1vxKy6xmX7O4/LDwy3/YjE07jw9PrD0OW8njy3vh5bXdNHbJC+35UMNODLfY7FV4/o3J1S2Aneub/m/iLq+qU5qvVgfx+blbWmc3oabxz70i0Csp7FOy9C4W88Sj2toO3lsrWeJltF6XiYfJ0yz+Z0aT7QbcAkXhlykztE6sWiqqMSCrvmx0LjQiqbH7Q3Ni0Urbku0rdjH0LL6OS7nghm3NzZura9pNrQdF2hzMXdf3q6axuNCrZ/j9nBcOuec6/ni/TuX77nYqnENi/XqnBw4Xfdctzxff0q8tr2stzc2z2NT87YsLidDy+r6iMtpft7e0LhF68dnwNB21GdczjXdfXkd9edrVtuNy2A6AjcOxwVp7kNN68Qil4ueipOLnmj5WHRyyB2aJ3lbomVaY87L6rOLrbgAetq5cWuels/ydnI/4nW9LX+ORXxoP3AsOvc+z35Qu8XroSUuH68VX8tu+qzmeylvp9W0TIvrQm7nxroG7+eSbefxq1XjupPr0xhdQ1rHxzyfA18fvta8vGm+tzc2T/K2TNPymPOy+pzrZ6yp58YtWjbfB3k7Q9eg1o3b18/x+SFab86xx082v1N14nIDLqXrKBaNKbROq4i4oOU+VYQ8PRY7GZsnmp63FYtqFJd1kdSf5wyNW31pWtbaTh67p3u5Vl9D+4Hj0HUwdK3G66MlXpfxOm5dc76P8nWoaa1grT5yOMh8zeagcWs+DnNpndyq0fWg/R67zjKv0zrvviZin/E6VYvG5snQtjQtjzkvO6d+tsYtWv9c4NZ8fc483cu1+srbwzSb36k6cbmtRRcIQWA534hu+SbsiR/U8YF+jpaPRcQBwNeM+szXj5dxi8bm6XMuWDqereszLus+x4rduXFr3TwembIdP2g8vdXX0H7gOHR+dY1kmq7Wur5M10fr/tQ0tUzLKyxEWneo/gxNN1+zOQxtQWM5N95MY8+tGtehOXWmFYJ17DVN5yHXNvMyavlcDc27NHC31o28zNC4dVzyWKduR/3E6a2+8vYwzeZ3qk5cbmvxg6D1oLhUviCPykFMred99jhbD3BTwYgPci3vIuIiFufn4Br7zkVvbJ7kz6Lj2XqAtNZVi9extqFlpoxby2mZrLWdfPx8XLUdafU1tB84Bl1bQ+dX0/1gj9dSpHleJl7DutbitWVaJl7PouVy/dF6ebkWX7N5O1vw/TpnLFo+t4p0/rXvY+c8Xqe+5nysc+3yufB1q+Vj3/G6GZsneVsW+7e8rD/ne0zraRlvyzQt96t18/0xtB39GWm9WPdbfeXtYZrN71SduNzW4ItLLV88l1J/+eY4KgeuNY7jtcVznqmg5IKh5TzN+xmX0WedZxWs1nzNc/Eamyd5vgxdR3lZLadpuQ2NS5/juDXPy+tnF+C8npeL03Tc4nn3MlGl+6GifA1EPu/6M18XoutFzfemrkeL92u85lq0TB6D+z3H16yv+3P8i4BbXs/3o5bT/sQxjM0zHatz+xvFsbhV5XOTrwWdI02P58rXl6f53Piz5/s6ysu7tsrYPMnzTdPyuW4tq8+5aV/l3LhF11RcXlrb8X3qafpTn+N9qWXy8fX2MM/md6pOXG5r8I3pFi+6S/ji10VZgQuLWr4Je+XxxhYLStwntVzY3FRg9KfPteZ7mufb0LzWtvK0oekel8S+1eL+nBu3i6rnjW3HRdrN+yFaLs5TPy7gbte6z9APnVc/2DNfY75u8nKar2vC8+N1K/m6HtpOXCa2oeUjb2PKtanxtq55j1vbcx30feQxjM2L1Ge8585RP7lVpmOaj4evQ8s10echTtMyrl8651om1zRfM2PzWtvK04ame1wS+877E9fJ45Z4H2lcY9vJddz7oT/jdG3D17HbnOsWD+ft8c/NxJPndm26cHRhxAs4FtEojsMXeVzP0yROb82XPN8XcxYvet0cli9wjXto2SjfROonyjeTWkteTjeqf25te2hsuR8tJz6GQ/sBoB++j+MDO4r1T8vFzw4Y/lnzc12yXFszTcs1wyHpHIeRoVpsHmOmaa5f6ivuo/bHYxibF2k/4nLnaPu5Aejf5nfqLYqHC7ELrdpYgWstk6epWMcQmfuL81zYtYw+56Ibpztgx4eJprsvNRfu+Dlq9afmccRp5s+xr7gPnu6+1fID79x+xP70wNLyPif6GUDffA/rHm+J97HvbS+rea5Brl+x3rRoGTUHXNO0XH/U99C4Io/LYxniGpVputaXWEvzsmPzIgI3UEOJwB37jNsZKvaeH4tga5oMTXdRjtNj4LfWNH/ODyc1LT80TVr9aQz67P78OT7EVPQ1LY7X02Jf8SESH3hT9kN/elqcrm3mhyeA/vgeHgq2sX6IltU01Y1Yb1qBe6gGxIBr+ry0ZrhWuf5k7re1XdH8PN3LqsV6LGPzRP3l4zbGfcUGdEvXZ25Fbb7naxcPFfZY5Fws1YYKtufHItiaJuemx4dMDKV+0LgYt7blh1orXLemSSzuLTH0xv2PxyUGYX2OYxsK3FP2I247HhcA+6H7txUcJdfBVs0T168cuF0rIvcR6fNQ/VadGZon7q8VuDUe75uXi2MU1S73r2XyPnisY/MiTace4rB0zedW1OZ7rgKU2zWpkMXCqgJ4blue1wqP+YHSmj4ULF3A1fxg0XqelpsLfytct6ZJ7K8l7n98KPlhoOaHhD/HfRtaf8p+ELiB/YuBM4u1QnzP5/t9KHBrWgzdXj/WONG01hi8vP4cor5ay3jdOCZ9VjPXP6/rvkxj9zEYmxdpWt6/8u7uThv2SfdAbkVtvucuaLFdiwvoWGsVOs+LxbE1Tc5NHwrcLupaz9OGtMJ1a5poe57eeujEYxIfWH7YxfX8Oe7blMA9JG6bwA3sUys4xvqhlmuL1rG4nJr78jq5r1jf8rxWG6otU54HalmsbXm+xpb7tbF55vn6E4GOVW7YJ87lk833PBYjt2vJhd5UCL2t/OCQ1ryh5V2M83QH36HAbfEBEt+s6GePvRWuW9MkbiNOjzxmP+DE44j7EcO7DQXuKfsRHz4EbmC/VCfifY5lVKNjHcWjh2fEScM+cS6fbL7nd3d3J+0aVMhyCI4c/NRywXPQ9PoxxOY+9dnzxH3FcOm3F142/hIQl4t9xzHNCdzi6WretvrzdmNoNn/OYdnTvW4M4Wre9pT9iMsQuIH9Um3ItRDzuB6iQcclN+wT5/LJIffcoc5tKLzm5nAag6FaDNxq8UGTl3UfFueptd4K5T7U3E8MvW459Krl0O1w7+bAbGPbjPL2fSziMbWxPlvzeGAD++VaivlcD1vPAzzQdZUb9olz+aTungMP9AtE/mUFbQoHrV+0UJeCI/+1aj5+UTkjB7QLjpde7PjFD8apvl/9l8Arnsu9q7vnKE9BwQEy/xeB3PJ/Icjy8g7xDiPn+lcb2kbrv8ps9V8HtF/8lwkAq3qocSdtpvhfEfJ/qW21Ma0arP41XW1K/2pDpvxX61tRfb/qtrXfuRVVd89RmgpKflvropmDr4th6+2u+mnN8/T49m+of9Hy54qc+uoh7Gr8cb8A4Koe6uRJm0m1UjU3Up1WDY4czNXy8qJ+WvM8PdbzVv+m5cfetHsc554Dt6D63npOLaLjkVtRdfccZbmwZWOB2KE7FkwtN1Yg1V8MyGP9q9+9BG65akEGgEj1ObcZVEtbL0iGArGfCflFQqvuR+ov1u2xwK3lhvoRj+Hcc+AWPJarUD+5FVV3z1GWimKrGM8N3H7DMSZuZ6h/fR4rxNZT4NaYexkLgINRXc1tBtVZ1dtsKBA7YMbA7XrdelaY1psSuKeEaI+hh8AtOhZXGYuOR25F1d1zlKWi1grVQ4FYRUfTW8U4TjtnqH/1sUbg1rbcWuv5Fwbtn1ocw9g88cOh9VADgIs81JaTNpHqq2pTy1Agdr2LtdnLtp4VQ1r9q1ZOeU64pqreTuH9dMvr+Xmjpr7jLw5j80z9zXneDHrYxkkrqu6eoyQXGhWZLBah3HKwdLFrFaohY/23xpNNDdwu3HHM3o5p3H6QqLDGMYzNizR96sMBACZTrcptItWvoYCrea6FsbXqqvrQvFz7xwz1f+3AncNw63mkz6Zl4xjG5pn7nPJsGqVt5VZU3T1HSWNFxIHYYdNiEbVWgTtnqH8VuylFTctNCdwaUy6g3raLuebHsWu6xzA2L9JY5uw/gMut9T+L64pqbW4TqX7l+meqV7GOm6bleq4+NG1J4I5UO4fGE2k5rTslcGu5oeeU+nFfcezet7F5kZ8Zc/a/6aGPk1ZU3T1HSUsCt6gIap4Lk5dtFaohQ/3rc2s8mYr2lMA9NC5Nd+H3/rSWHZsXEbiB2/O9GdvhaJ9ym0g1bijgql4NHS/VM81zwPSycwLnUP9TQrSD8Lll/QzL4/J0P1+8P61lx+aZpo/Nn+yhj5NWVN09R0kuSnMDt+fFQq7PalON9T/FWODW/rhQ53Ga1s3TvQ9qubCOzRP1R+AGbivel26Ho33KbSLVuFb9k7HA7Xmuz35WzKlxY/2fcy5wa7qW8bjyc6T1fPGyavnZMTZP3F+r9s/y0MdJK6runqOksSLSKljm4hSLrwqgpo29kVDhd7gf6180fWieqK+hwK0xeJ+0nLaTxW3nB1IM42PzIk0f23cA16f7OLfD0T7lNpFq9FCdHAvErpvx2aB+NK31gkY0PdbGc4G7VUftXOD2ul4uPovEzyjRMnF+fPaMzYs8XctfRGPKrai6e46yWsVFhgqPC1mr+LjAtgqppsfiPdS/aNrQQ8K0jdYyHp95O3FMKuJxXc2L87UfLsJj88xFP+4fgPXpvsvtcLRPuU2U62Hkep2p3ml6rHvmY5zrdqtmD/Uv6nvsBYVramsZrRtrsLcTx6SxeF33Fefrs+r12LwoPzNwuelX8Up0onMD1qRiFYuXqLC0rsXYhjjgji0/pf+hYqzC2Fo+tvygcFEdmq/9j+OO88fmWethA2B9vi9jOxztU24z6JjEADlUo2PLz4RI88aWn9K/mupyi2psa/nYYkAWPS+G5ms7mh/H7flj8yKNaeiZhGXmXcUr8AmPDViTwyiWUzFuFWkA68rPy0PWMv3LK7nN4ECJZXhGroPAjZIoyMspaLfeegNYH8/MafRf4OJbbkzHC5V1ELhRlgI3wXEeFWHuUWA7+XnJ/ThMx4bgOI+eibyMWgeBG6WpGPM9tWn0tohfUIBt8cycRzVr6LvT+CUFbX5BWQ+BGwCAneCZCewTgRsAgJ3gmQnsE4EbAICd4JkJ7BOBGwCAneCZCewTgRsAgJ3gmQnsU5eBO7Yl7u7umn3NbepniVZfS9oS7Hu7vzmNfZ+v1deStgT73u5vTmPf52v1taTNdY0+dkn7eY22hK6RVl9z28JrrdnXkrbE1vt+IA9HYVut4hHbEjyE2v3Naez7fK2+lrQl2Pd2f3Ma+z5fq68lbYmq+36NPnZJ+3mNtoSukVZfc9vCa63Z15K2xNb7fiAPR2FbreIR2xI8hNr9zWns+3ytvpa0Jdj3dn9zGvs+X6uvJW2Jqvuu8X7//fcX9bFL2s9rtCV0jbT6mtsWXmvNvpa0Ja617w/X7Le+lh6DA3g4CtuKRaPVluAh1O5vTmPf58vHcGlbgvPe7m9OY9/na/W1pC3Bvl/Wx+5oP6/RltA10uprblt4rTX7WtKWuNa+x1bU5nsei0arLUEhbvc3p7Hv87X6WtKWYN/b/c1p7Pt8rb6WtCXY98v62B3t5zXaErpGWn3NbQuvtWZfS9oS19r32Iqqu+cAAADADRC4AQAAgBURuAEAAIAVEbgBAACAFRG4AQAAgBURuAEAAIAVEbgBAACAFRG4AQAAgBURuAEAAIAVEbgBAACAFRG4AQAAgBURuAEAAIAVEbgBAACAFRG4AQAAgBURuAEAAIAVEbgBAACAFRG4AQAAgBURuAEAQL/++q8f0spDXJnS/vjHx5UeeNp//ufjhBX83d/9cvtzxPViW3O82MzDmQUAAOhcDKX//u+PEx/9y7/8NN2BW396Wc1bU/yFYIpXr35ePv6CoH3ydAX5a9C+3zrAa/9wYuLVAQAAsCGHUbUcuEXh8tZvuGVO4I5hu/WLQPxF4dLg6r5uGbj9iw9OcFQAAED/HETVYuB2MNW0GLhvZWrgjm+wx5aN/bV+sZgiBvdbBW6HbTWc4KgAAID+OcypxSCqgBrFsKnmEJ6/b50DcCuYxhCplrclUwN3fLs99pWROE7/MpHHrrHG/jxN8nS3f/u3X35Wn7HfPKa4X2qSx2F5ep4PAjcAANiBGOQcuBX0WiE4BuX41juGSAdMT8uB0yHS8no2NXC3tt0SA3PcN49HzeG6NU3HpjVd9LOnq/k4Dh2D1r61pqmf1nQ84agAAID+OczlFkOpTQnc1poWg6n6kqFAOTVoxuX85rrl0sAtUwJ3DNfxeDmES2vfLj0ORXFUAABA/xzm1K7xhtvOBUivf265PD2LITiG3WyLwK199PRLj1eejm84KgAAoH8Oc2r5LWx2jcBtMYy2lpsaNGOQHgvcMSzHN+GtED0UrIemTwncfqMvc47X1ONQFEcFAAD0z2FOLQbu1tczrhG4tY04/dKgGcPu2LJxmbiftwrcS4/X1ONQFEcFAAD0z2FOLQbRlksDdwymfuPrz3E5mRM047jim2QbetMscwJ3fJs+JXDHccXlpx4viWPBCY4KAADon8Oc2tqBOwdffR4KlHMCtwyNLb5RP/fWXqE4Lu9pFgO3lnN/MXB7nnjfYgiX2I/E4+JpFo+PttPah8LS0QIAAOhIDLS5xZBpORSq5cDs1ppmcbsKjzngytj657T2S9PGxHW07bz9eDzisg72MXDn9fVzi+er6TjkcUdxusM8vklHCgAAAIeUAzduhsANAABQAYF7MwRuAACACgjcmyFwAwAAHF0M27FpOlb3cKQBAAAArIXADQAAAKyIwA0AAACsiMANAAAArIjADQAAAKyIwA0AAACsiMANAAAArIjADQAAAKyIwA0AAACsiMANAAAArIjADQAAAKyIwA0AAACsiMANAAAArIjADQAAAKyIwA0AAACsiMANAAAArIjADQAAAKyIwA0AAACs5v7+/wE07ZuLNWCObAAAAABJRU5ErkJggg==)\n",
    "\n",
    "**Main Ideas:**\n",
    "\n",
    "In Jane street competition, Yirun have used a supervised autoencoder MLP approach and his teammates use XGBoost. Their final submission is a simple blend of these two models. Here, We explain Yirun's approach in detail.\n",
    "\n",
    "The supervised autoencoder approach was initially proposed in [Bottleneck encoder + MLP + Keras Tuner](https://www.kaggle.com/aimind/bottleneck-encoder-mlp-keras-tuner-8601c5) and was adapted to this competition [here](https://www.kaggle.com/yamqwe/bottleneck-encoder-mlp-keras-tuner), where one supervised autoencoder is trained separately before cross-validation (CV) split. Yirun have realised that this training may cause label leakage because the autoencoder has seen part of the data in the validation set in each CV split and it can generate label-leakage features to overfit. So, his approach was to train the supervised autoencoder along with MLP in one model in each CV split. The training processes and explanations are given in the notebook and the following statements.\n",
    "\n",
    "**The Model:**\n",
    "\n",
    "- Use autoencoder to create new features, concatenating with the original features as the input to the downstream MLP model\n",
    "- Train autoencoder and MLP together in each CV split to prevent data leakage\n",
    "- Add target information to autoencoder (supervised learning) to force it to generate more relevant features, and to create a shortcut for backpropagation of gradient\n",
    "- Add Gaussian noise layer before encoder for data augmentation and to prevent overfitting\n",
    "- Use swish activation function instead of ReLU to prevent ‘dead neuron’ and smooth the gradient\n",
    "- Batch Normalisation and Dropout are used for MLP\n",
    "- Use Keras Tuner to find the optimal hyperparameter set\n",
    "\n",
    "\n",
    "\n",
    "This notebook follows the ideas presented in my \"Initial Thoughts\" [here][1]. Some code sections have been reused from Chris' great notebook series on SIIM ISIC melanoma detection competition [here][3]\n",
    "\n",
    "[1]: https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/284903\n",
    "[2]: https://www.kaggle.com/yamqwe/let-s-talk-validation-grouptimeseriessplit\n",
    "[3]: https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.029987,
     "end_time": "2021-11-27T08:16:39.425053",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.395066",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "____\n",
    "**Credits:**\n",
    "The following notebook is heavily based on multiple notebook of the past Jane street market prediction competition. If you find it useful, spare some upvotes to the originals. They earned it! \n",
    "- \"Jane Street: Supervised Autoencoder MLP\" (JaneStreet 1st 🏆)  by Yirun Zhang - https://www.kaggle.com/gogo827jz/jane-street-supervised-autoencoder-mlp\n",
    "\n",
    "____"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "347324aa-490a-4556-bcc4-36379bdbec7c",
    "_uuid": "03824ce3-fc08-4d19-926d-27edbb773466",
    "papermill": {
     "duration": 0.030949,
     "end_time": "2021-11-27T08:16:39.486079",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.45513",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"outline\">Libraries 📚</span>\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "f5bdcf7b-aa74-4204-9ff2-f20042b819e0",
    "_uuid": "78d84509-539f-4a94-a546-e228b28ea70a",
    "papermill": {
     "duration": 0.030041,
     "end_time": "2021-11-27T08:16:39.54644",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.516399",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Code starts here ⬇"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:43.394682Z",
     "iopub.status.busy": "2022-01-25T19:20:43.393303Z",
     "iopub.status.idle": "2022-01-25T19:20:48.640718Z",
     "shell.execute_reply": "2022-01-25T19:20:48.639961Z",
     "shell.execute_reply.started": "2022-01-25T19:20:43.394648Z"
    },
    "papermill": {
     "duration": 7.079069,
     "end_time": "2021-11-27T08:16:46.655729",
     "exception": false,
     "start_time": "2021-11-27T08:16:39.57666",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import traceback\n",
    "import gresearch_crypto\n",
    "import tensorflow as tf\n",
    "from scipy.stats import pearsonr\n",
    "import pandas as pd, numpy as np\n",
    "from sklearn.metrics import r2_score\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030531,
     "end_time": "2021-11-27T08:16:46.718422",
     "exception": false,
     "start_time": "2021-11-27T08:16:46.687891",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"config\">Configuration 🎚️</span>\n",
    "<hr >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030208,
     "end_time": "2021-11-27T08:16:46.779977",
     "exception": false,
     "start_time": "2021-11-27T08:16:46.749769",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In order to be a proper cross validation with a meaningful overall CV score, **you need to choose the same** `INC2021`, `INC2020`, `INC2019`, `INC2018`, `INC2017`, `INCCOMP`, `INCSUPP`, and `DEPTH_NETS`, `WIDTH_NETS` **for each fold**. If your goal is to just run lots of experiments, then you can choose to have a different experiment in each fold. Then each fold is like a holdout validation experiment. When you find a configuration you like, you can use that configuration for all folds.\n",
    "* DEVICE - is GPU or TPU\n",
    "* SEED - a different seed produces a different triple stratified kfold split.\n",
    "* FOLDS - number of folds. Best set to 3, 5, or 15 but can be any number between 2 and 15\n",
    "* LOAD_STRICT - This controls whether to load strict at proposed [here](https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm)\n",
    "* INC2021 - This controls whether to include the extra historical prices during 2021.\n",
    "* INC2020 - This controls whether to include the extra historical prices during 2020.\n",
    "* INC2019 - This controls whether to include the extra historical prices during 2019.\n",
    "* INC2018 - This controls whether to include the extra historical prices during 2018.\n",
    "* INC2017 - This controls whether to include the extra historical prices during 2017.\n",
    "* INCCOMP - This controls whether to include the original data of the competition.\n",
    "* INCSUPP - This controls whether to include the supplemented train data that was released with the competition.\n",
    "* BATCH_SIZES - is a list of length FOLDS. These are batch sizes for each fold. For maximum speed, it is best to use the largest batch size your GPU or TPU allows.\n",
    "* EPOCHS - is a list of length FOLDS. These are maximum epochs. Note that each fold, the best epoch model is saved and used. So if epochs is too large, it won't matter.\n",
    "* DEPTH_NETS - is a list of length FOLDS. These are the Network Depths to use each fold. The number refers to the number of layers. So a number of `1` refers to 1 layer, and `2` refers to 2 layers, etc.\n",
    "* WIDTH_NETS - is a list of length FOLDS. These are the Network Widths to use each fold. The number refers to the number of units per layer. So a number of `32` refers to 32 per layer, and `643` refers to 64 layers, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:48.643422Z",
     "iopub.status.busy": "2022-01-25T19:20:48.643163Z",
     "iopub.status.idle": "2022-01-25T19:20:48.652116Z",
     "shell.execute_reply": "2022-01-25T19:20:48.650622Z",
     "shell.execute_reply.started": "2022-01-25T19:20:48.643386Z"
    },
    "papermill": {
     "duration": 0.041476,
     "end_time": "2021-11-27T08:16:46.851866",
     "exception": false,
     "start_time": "2021-11-27T08:16:46.81039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DEVICE = \"TPU\" #or \"GPU\"\n",
    "\n",
    "SEED = 42\n",
    "\n",
    "# CV PARAMS\n",
    "FOLDS = 5\n",
    "GROUP_GAP = 130\n",
    "MAX_TEST_GROUP_SIZE = 180\n",
    "MAX_TRAIN_GROUP_SIZE = 280\n",
    "\n",
    "# LOAD STRICT? YES=1 NO=0 | see: https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm\n",
    "LOAD_STRICT = True\n",
    "\n",
    "# WHICH YEARS TO INCLUDE? YES=1 NO=0\n",
    "INC2021 = 0\n",
    "INC2020 = 0\n",
    "INC2019 = 0\n",
    "INC2018 = 0\n",
    "INC2017 = 0\n",
    "INCCOMP = 1\n",
    "INCSUPP = 0\n",
    "\n",
    "# BATCH SIZE AND EPOCHS\n",
    "BATCH_SIZES = [4096] * FOLDS\n",
    "EPOCHS = [1] * FOLDS\n",
    "\n",
    "# WHICH NETWORK ARCHITECTURE TO USE?\n",
    "DEPTH_NETS = [2, 2, 2, 2, 2] \n",
    "WIDTH_NETS = [256, 128, 64, 64, 32]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:48.654681Z",
     "iopub.status.busy": "2022-01-25T19:20:48.654347Z",
     "iopub.status.idle": "2022-01-25T19:20:48.86434Z",
     "shell.execute_reply": "2022-01-25T19:20:48.86352Z",
     "shell.execute_reply.started": "2022-01-25T19:20:48.654645Z"
    },
    "papermill": {
     "duration": 0.061279,
     "end_time": "2021-11-27T08:16:46.943859",
     "exception": false,
     "start_time": "2021-11-27T08:16:46.88258",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DEVICE == \"TPU\":\n",
    "    print(\"connecting to TPU...\")\n",
    "    try:\n",
    "        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "        print('Running on TPU ', tpu.master())\n",
    "    except ValueError:\n",
    "        tpu = None\n",
    "    if tpu:\n",
    "        try:\n",
    "            print(\"initializing  TPU ...\")\n",
    "            tf.config.experimental_connect_to_cluster(tpu)\n",
    "            tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "            print(\"TPU initialized\")\n",
    "        except: print(\"failed to initialize TPU\")\n",
    "    else: DEVICE = \"GPU\"\n",
    "\n",
    "if DEVICE != \"TPU\": strategy = tf.distribute.get_strategy()\n",
    "if DEVICE == \"GPU\": print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
    "AUTO     = tf.data.experimental.AUTOTUNE\n",
    "REPLICAS = strategy.num_replicas_in_sync"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030375,
     "end_time": "2021-11-27T08:16:47.005222",
     "exception": false,
     "start_time": "2021-11-27T08:16:46.974847",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"loading\">Data Loading 🗃️</span>\n",
    "<hr>\n",
    "\n",
    "The data organisation has already been done and saved to Kaggle datasets. Here we choose which years to load. We can use either 2017, 2018, 2019, 2020, 2021, Original, Supplement by changing the `INC2021`, `INC2020`, `INC2019`, `INC2018`, `INC2017`, `INCCOMP`, `INCSUPP` variables in the preceeding code section. These datasets are discussed [here][1].\n",
    "\n",
    "[1]: https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285726\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-01-25T19:20:48.867755Z",
     "iopub.status.busy": "2022-01-25T19:20:48.867178Z",
     "iopub.status.idle": "2022-01-25T19:21:15.169716Z",
     "shell.execute_reply": "2022-01-25T19:21:15.168989Z",
     "shell.execute_reply.started": "2022-01-25T19:20:48.867718Z"
    },
    "papermill": {
     "duration": 25.330417,
     "end_time": "2021-11-27T08:17:12.367164",
     "exception": false,
     "start_time": "2021-11-27T08:16:47.036747",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datatable as dt\n",
    "extra_data_files = {0: '../input/cryptocurrency-extra-data-binance-coin', 2: '../input/cryptocurrency-extra-data-bitcoin-cash', 1: '../input/cryptocurrency-extra-data-bitcoin', 3: '../input/cryptocurrency-extra-data-cardano', 4: '../input/cryptocurrency-extra-data-dogecoin', 5: '../input/cryptocurrency-extra-data-eos-io', 6: '../input/cryptocurrency-extra-data-ethereum', 7: '../input/cryptocurrency-extra-data-ethereum-classic', 8: '../input/cryptocurrency-extra-data-iota', 9: '../input/cryptocurrency-extra-data-litecoin', 11: '../input/cryptocurrency-extra-data-monero', 10: '../input/cryptocurrency-extra-data-maker', 12: '../input/cryptocurrency-extra-data-stellar', 13: '../input/cryptocurrency-extra-data-tron'}\n",
    "\n",
    "# Uncomment to load the original csv [slower]\n",
    "# orig_df_train = pd.read_csv(data_path + 'train.csv') \n",
    "# supp_df_train = pd.read_csv(data_path + 'supplemental_train.csv')\n",
    "# df_asset_details = pd.read_csv(data_path  + 'asset_details.csv').sort_values(\"Asset_ID\")\n",
    "\n",
    "orig_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_train.jay').to_pandas()\n",
    "df_asset_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\n",
    "supp_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas()\n",
    "assets_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\n",
    "asset_weight_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Weight'].tolist()[idx] for idx in range(len(assets_details))}\n",
    "asset_name_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Asset_Name'].tolist()[idx] for idx in range(len(assets_details))}\n",
    "\n",
    "def load_training_data_for_asset(asset_id, load_jay = True):\n",
    "    dfs = []\n",
    "    if INCCOMP: dfs.append(orig_df_train[orig_df_train[\"Asset_ID\"] == asset_id].copy())\n",
    "    if INCSUPP: dfs.append(supp_df_train[supp_df_train[\"Asset_ID\"] == asset_id].copy())\n",
    "    \n",
    "    if load_jay:\n",
    "        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.jay').to_pandas())\n",
    "        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.jay').to_pandas())\n",
    "        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.jay').to_pandas())\n",
    "        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.jay').to_pandas())\n",
    "        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.jay').to_pandas())\n",
    "    else: \n",
    "        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'))\n",
    "        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'))\n",
    "        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'))\n",
    "        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'))\n",
    "        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'))\n",
    "    df = pd.concat(dfs, axis = 0) if len(dfs) > 1 else dfs[0]\n",
    "    df['date'] = pd.to_datetime(df['timestamp'], unit = 's')\n",
    "    if LOAD_STRICT: df = df.loc[df['date'] < \"2021-06-13 00:00:00\"]    \n",
    "    df = df.sort_values('date')\n",
    "    return df\n",
    "\n",
    "def load_data_for_all_assets():\n",
    "    dfs = []\n",
    "    for asset_id in list(extra_data_files.keys()): dfs.append(load_training_data_for_asset(asset_id))\n",
    "    return pd.concat(dfs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030782,
     "end_time": "2021-11-27T08:17:12.429606",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.398824",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"features\">Feature Engineering 🔬</span>\n",
    "<hr>\n",
    "\n",
    "This notebook uses upper_shadow, lower_shadow, high_div_low, open_sub_close, seasonality/datetime features first shown in this notebook [here][1] and successfully used by julian3833 [here][2].\n",
    "\n",
    "Additionally we can decide to use external data by changing the variables `INC2021`, `INC2020`, `INC2019`, `INC2018`, `INC2017`, `INCCOMP`, `INCSUPP` in the preceeding code section. These variables respectively indicate whether to load last year 2021 data and/or year 2020, 2019, 2018, 2017, the original, supplemented data. These datasets are discussed [here][3]\n",
    "\n",
    "Consider experimenting with different feature engineering and/or external data. The code to extract features out of the dataset is taken from julian3833' notebook [here][2]. Thank you julian3833, this is great work.\n",
    "\n",
    "[1]: https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition\n",
    "[2]: https://www.kaggle.com/julian3833\n",
    "[3]: TBD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.171345Z",
     "iopub.status.busy": "2022-01-25T19:21:15.171103Z",
     "iopub.status.idle": "2022-01-25T19:21:15.181332Z",
     "shell.execute_reply": "2022-01-25T19:21:15.180701Z",
     "shell.execute_reply.started": "2022-01-25T19:21:15.171311Z"
    },
    "papermill": {
     "duration": 0.047873,
     "end_time": "2021-11-27T08:17:12.509409",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.461536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\n",
    "def lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n",
    "\n",
    "# Credit: https://www.kaggle.com/swaralipibose/64-new-features-with-autoencoders/code\n",
    "def get_features(df, row = False):\n",
    "    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n",
    "    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n",
    "    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n",
    "    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n",
    "    df_feat['trade'] = df_feat['Close'] - df_feat['Open']\n",
    "    df_feat['gtrade'] = df_feat['trade'] / df_feat['Count']\n",
    "    df_feat['shadow1'] = df_feat['trade'] / df_feat['Volume']\n",
    "    df_feat['shadow3'] = df_feat['upper_Shadow'] / df_feat['Volume']\n",
    "    df_feat['shadow5'] = df_feat['lower_Shadow'] / df_feat['Volume']\n",
    "    df_feat['diff1'] = df_feat['Volume'] - df_feat['Count']\n",
    "    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n",
    "    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n",
    "    df_feat['mean3'] = (df_feat['trade'] + df_feat['gtrade']) / 2\n",
    "    df_feat['mean4'] = (df_feat['diff1'] + df_feat['upper_Shadow']) / 2\n",
    "    df_feat['mean5'] = (df_feat['diff1'] + df_feat['lower_Shadow']) / 2\n",
    "    return df_feat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.03185,
     "end_time": "2021-11-27T08:17:12.573351",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.541501",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"modelconf\">Building the Model ⚙️</span>\n",
    "<hr>\n",
    "\n",
    "This is a common model architecute. Consider experimenting with different backbones, custom heads, losses, and optimizers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.031589,
     "end_time": "2021-11-27T08:17:12.636713",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.605124",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Competition Metric**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.182775Z",
     "iopub.status.busy": "2022-01-25T19:21:15.182396Z",
     "iopub.status.idle": "2022-01-25T19:21:15.205603Z",
     "shell.execute_reply": "2022-01-25T19:21:15.204945Z",
     "shell.execute_reply.started": "2022-01-25T19:21:15.182741Z"
    },
    "papermill": {
     "duration": 0.045505,
     "end_time": "2021-11-27T08:17:12.713855",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.66835",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Numpy Version\n",
    "def corr(a, b, w):\n",
    "    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n",
    "    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n",
    "\n",
    "# TF Version\n",
    "def tf_cov(x, y, w): return (tf.reduce_sum(w * (x - tf.reduce_mean(x * w)) * (y - tf.reduce_mean(y * w))) / tf.reduce_sum(w))\n",
    "def tf_comp_metric(a, b, w): return tf_cov(a, b, w) / tf.sqrt(tf_cov(a, a, w) * tf_cov(b, b, w))\n",
    "def nn_comp_metric(w): \n",
    "    def wcorr(x, y): return tf_comp_metric(x, y ,w)\n",
    "    return wcorr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.031629,
     "end_time": "2021-11-27T08:17:12.777656",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.746027",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**The Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.207213Z",
     "iopub.status.busy": "2022-01-25T19:21:15.206688Z",
     "iopub.status.idle": "2022-01-25T19:21:15.22261Z",
     "shell.execute_reply": "2022-01-25T19:21:15.221844Z",
     "shell.execute_reply.started": "2022-01-25T19:21:15.207168Z"
    },
    "papermill": {
     "duration": 0.052809,
     "end_time": "2021-11-27T08:17:12.862198",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.809389",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_model(fold, dim = 128, weight = 1.0):\n",
    "   \n",
    "    n_layers = DEPTH_NETS[fold]\n",
    "    hidden_units = WIDTH_NETS[fold]\n",
    "    dropout_rates = 0.002\n",
    "    \n",
    "    inp = tf.keras.layers.Input(shape = (dim, ))\n",
    "    x0 = tf.keras.layers.BatchNormalization()(inp)\n",
    "\n",
    "    encoder = tf.keras.layers.GaussianNoise(dropout_rates)(x0)\n",
    "    encoder = tf.keras.layers.Dense(hidden_units)(encoder)\n",
    "    encoder = tf.keras.layers.BatchNormalization()(encoder)\n",
    "    encoder = tf.keras.layers.Activation('swish')(encoder)\n",
    "\n",
    "    decoder = tf.keras.layers.Dropout(dropout_rates)(encoder)\n",
    "    decoder = tf.keras.layers.Dense(dim, name = 'decoder')(decoder)\n",
    "\n",
    "    x_ae = tf.keras.layers.Dense(hidden_units)(decoder)\n",
    "    x_ae = tf.keras.layers.BatchNormalization()(x_ae)\n",
    "    x_ae = tf.keras.layers.Activation('swish')(x_ae)\n",
    "    x_ae = tf.keras.layers.Dropout(dropout_rates)(x_ae)\n",
    "\n",
    "    out_ae = tf.keras.layers.Dense(1, activation = 'linear', name = 'ae_action')(x_ae)\n",
    "\n",
    "    x = tf.keras.layers.Concatenate()([x0, encoder])\n",
    "    x = tf.keras.layers.BatchNormalization()(x)\n",
    "    x = tf.keras.layers.Dropout(dropout_rates)(x)\n",
    "\n",
    "    if n_layers > 2:\n",
    "        for i in range(2, n_layers):\n",
    "            x = tf.keras.layers.Dense(hidden_units[i])(x)\n",
    "            x = tf.keras.layers.BatchNormalization()(x)\n",
    "            x = tf.keras.layers.Activation('swish')(x)\n",
    "            x = tf.keras.layers.Dropout(dropout_rates)(x)\n",
    "\n",
    "    out = tf.keras.layers.Dense(1, activation = 'linear', name = 'action')(x)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs = inp, outputs = [decoder, out_ae, out])\n",
    "    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001),\n",
    "                  loss = {'decoder': tf.keras.losses.MeanSquaredError(),\n",
    "                          'ae_action': tf.keras.losses.MeanSquaredError(),\n",
    "                          'action': tf.keras.losses.MeanSquaredError(),\n",
    "                         },\n",
    "                  metrics = {'decoder': tf.keras.metrics.MeanAbsoluteError(),\n",
    "                             'ae_action': tf.keras.metrics.MeanSquaredError(),\n",
    "                             'action': nn_comp_metric(weight),\n",
    "                            },\n",
    "                 )    \n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.032247,
     "end_time": "2021-11-27T08:17:12.92683",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.894583",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"sched\">LR Scheduler ⏱️</span>\n",
    "<hr>\n",
    "\n",
    "This is a common train schedule in kaggle competitions. The learning rate starts near zero, then increases to a maximum, then decays over time. Consider changing the schedule and/or learning rates. Note how the learning rate max is larger with larger batches sizes. This is a good practice to follow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.223999Z",
     "iopub.status.busy": "2022-01-25T19:21:15.223755Z",
     "iopub.status.idle": "2022-01-25T19:21:15.23452Z",
     "shell.execute_reply": "2022-01-25T19:21:15.233775Z",
     "shell.execute_reply.started": "2022-01-25T19:21:15.223967Z"
    },
    "papermill": {
     "duration": 0.042625,
     "end_time": "2021-11-27T08:17:13.002811",
     "exception": false,
     "start_time": "2021-11-27T08:17:12.960186",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_lr_callback(batch_size = 8):\n",
    "    lr_start   = 0.000005\n",
    "    lr_max     = 0.00000125 * REPLICAS * batch_size\n",
    "    lr_min     = 0.000001\n",
    "    lr_ramp_ep = 5\n",
    "    lr_sus_ep  = 0\n",
    "    lr_decay   = 0.8\n",
    "    def lrfn(epoch):\n",
    "        if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n",
    "        elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n",
    "        else: lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n",
    "        return lr\n",
    "    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n",
    "    return lr_callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.238137Z",
     "iopub.status.busy": "2022-01-25T19:21:15.237791Z",
     "iopub.status.idle": "2022-01-25T19:21:15.271779Z",
     "shell.execute_reply": "2022-01-25T19:21:15.271022Z",
     "shell.execute_reply.started": "2022-01-25T19:21:15.238111Z"
    },
    "papermill": {
     "duration": 0.07369,
     "end_time": "2021-11-27T08:17:13.112796",
     "exception": false,
     "start_time": "2021-11-27T08:17:13.039106",
     "status": "completed"
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\n",
    "from sklearn.utils.validation import _deprecate_positional_args\n",
    "\n",
    "# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\n",
    "class GroupTimeSeriesSplit(_BaseKFold):\n",
    "    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n",
    "    Provides train/test indices to split time series data samples\n",
    "    that are observed at fixed time intervals according to a\n",
    "    third-party provided group.\n",
    "    In each split, test indices must be higher than before, and thus shuffling\n",
    "    in cross validator is inappropriate.\n",
    "    This cross-validation object is a variation of :class:`KFold`.\n",
    "    In the kth split, it returns first k folds as train set and the\n",
    "    (k+1)th fold as test set.\n",
    "    The same group will not appear in two different folds (the number of\n",
    "    distinct groups has to be at least equal to the number of folds).\n",
    "    Note that unlike standard cross-validation methods, successive\n",
    "    training sets are supersets of those that come before them.\n",
    "    Read more in the :ref:`User Guide <cross_validation>`.\n",
    "    Parameters\n",
    "    ----------\n",
    "    n_splits : int, default=5\n",
    "        Number of splits. Must be at least 2.\n",
    "    max_train_size : int, default=None\n",
    "        Maximum size for a single training set.\n",
    "    Examples\n",
    "    --------\n",
    "    >>> import numpy as np\n",
    "    >>> from sklearn.model_selection import GroupTimeSeriesSplit\n",
    "    >>> groups = np.array(['a', 'a', 'a', 'a', 'a', 'a',\\\n",
    "                           'b', 'b', 'b', 'b', 'b',\\\n",
    "                           'c', 'c', 'c', 'c',\\\n",
    "                           'd', 'd', 'd'])\n",
    "    >>> gtss = GroupTimeSeriesSplit(n_splits=3)\n",
    "    >>> for train_idx, test_idx in gtss.split(groups, groups=groups):\n",
    "    ...     print(\"TRAIN:\", train_idx, \"TEST:\", test_idx)\n",
    "    ...     print(\"TRAIN GROUP:\", groups[train_idx],\\\n",
    "                  \"TEST GROUP:\", groups[test_idx])\n",
    "    TRAIN: [0, 1, 2, 3, 4, 5] TEST: [6, 7, 8, 9, 10]\n",
    "    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a']\\\n",
    "    TEST GROUP: ['b' 'b' 'b' 'b' 'b']\n",
    "    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] TEST: [11, 12, 13, 14]\n",
    "    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b']\\\n",
    "    TEST GROUP: ['c' 'c' 'c' 'c']\n",
    "    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\\\n",
    "    TEST: [15, 16, 17]\n",
    "    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b' 'c' 'c' 'c' 'c']\\\n",
    "    TEST GROUP: ['d' 'd' 'd']\n",
    "    \"\"\"\n",
    "    @_deprecate_positional_args\n",
    "    def __init__(self,\n",
    "                 n_splits=5,\n",
    "                 *,\n",
    "                 max_train_size=None\n",
    "                 ):\n",
    "        super().__init__(n_splits, shuffle=False, random_state=None)\n",
    "        self.max_train_size = max_train_size\n",
    "\n",
    "    def split(self, X, y=None, groups=None):\n",
    "        \"\"\"Generate indices to split data into training and test set.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like of shape (n_samples, n_features)\n",
    "            Training data, where n_samples is the number of samples\n",
    "            and n_features is the number of features.\n",
    "        y : array-like of shape (n_samples,)\n",
    "            Always ignored, exists for compatibility.\n",
    "        groups : array-like of shape (n_samples,)\n",
    "            Group labels for the samples used while splitting the dataset into\n",
    "            train/test set.\n",
    "        Yields\n",
    "        ------\n",
    "        train : ndarray\n",
    "            The training set indices for that split.\n",
    "        test : ndarray\n",
    "            The testing set indices for that split.\n",
    "        \"\"\"\n",
    "        if groups is None:\n",
    "            raise ValueError(\n",
    "                \"The 'groups' parameter should not be None\")\n",
    "        X, y, groups = indexable(X, y, groups)\n",
    "        n_samples = _num_samples(X)\n",
    "        n_splits = self.n_splits\n",
    "        n_folds = n_splits + 1\n",
    "        group_dict = {}\n",
    "        u, ind = np.unique(groups, return_index=True)\n",
    "        unique_groups = u[np.argsort(ind)]\n",
    "        n_samples = _num_samples(X)\n",
    "        n_groups = _num_samples(unique_groups)\n",
    "        for idx in np.arange(n_samples):\n",
    "            if (groups[idx] in group_dict):\n",
    "                group_dict[groups[idx]].append(idx)\n",
    "            else:\n",
    "                group_dict[groups[idx]] = [idx]\n",
    "        if n_folds > n_groups:\n",
    "            raise ValueError(\n",
    "                (\"Cannot have number of folds={0} greater than\"\n",
    "                 \" the number of groups={1}\").format(n_folds,\n",
    "                                                     n_groups))\n",
    "        group_test_size = n_groups // n_folds\n",
    "        group_test_starts = range(n_groups - n_splits * group_test_size,\n",
    "                                  n_groups, group_test_size)\n",
    "        for group_test_start in group_test_starts:\n",
    "            train_array = []\n",
    "            test_array = []\n",
    "            for train_group_idx in unique_groups[:group_test_start]:\n",
    "                train_array_tmp = group_dict[train_group_idx]\n",
    "                train_array = np.sort(np.unique(\n",
    "                                      np.concatenate((train_array,\n",
    "                                                      train_array_tmp)),\n",
    "                                      axis=None), axis=None)\n",
    "            train_end = train_array.size\n",
    "            if self.max_train_size and self.max_train_size < train_end:\n",
    "                train_array = train_array[train_end -\n",
    "                                          self.max_train_size:train_end]\n",
    "            for test_group_idx in unique_groups[group_test_start:\n",
    "                                                group_test_start +\n",
    "                                                group_test_size]:\n",
    "                test_array_tmp = group_dict[test_group_idx]\n",
    "                test_array = np.sort(np.unique(\n",
    "                                              np.concatenate((test_array,\n",
    "                                                              test_array_tmp)),\n",
    "                                     axis=None), axis=None)\n",
    "            yield [int(i) for i in train_array], [int(i) for i in test_array]\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\n",
    "from sklearn.utils.validation import _deprecate_positional_args\n",
    "\n",
    "# modified code for group gaps; source\n",
    "# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\n",
    "class PurgedGroupTimeSeriesSplit(_BaseKFold):\n",
    "    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n",
    "    Allows for a gap in groups to avoid potentially leaking info from\n",
    "    train into test if the model has windowed or lag features.\n",
    "    Provides train/test indices to split time series data samples\n",
    "    that are observed at fixed time intervals according to a\n",
    "    third-party provided group.\n",
    "    In each split, test indices must be higher than before, and thus shuffling\n",
    "    in cross validator is inappropriate.\n",
    "    This cross-validation object is a variation of :class:`KFold`.\n",
    "    In the kth split, it returns first k folds as train set and the\n",
    "    (k+1)th fold as test set.\n",
    "    The same group will not appear in two different folds (the number of\n",
    "    distinct groups has to be at least equal to the number of folds).\n",
    "    Note that unlike standard cross-validation methods, successive\n",
    "    training sets are supersets of those that come before them.\n",
    "    Read more in the :ref:`User Guide <cross_validation>`.\n",
    "    Parameters\n",
    "    ----------\n",
    "    n_splits : int, default=5\n",
    "        Number of splits. Must be at least 2.\n",
    "    max_train_group_size : int, default=Inf\n",
    "        Maximum group size for a single training set.\n",
    "    group_gap : int, default=None\n",
    "        Gap between train and test\n",
    "    max_test_group_size : int, default=Inf\n",
    "        We discard this number of groups from the end of each train split\n",
    "    \"\"\"\n",
    "\n",
    "    @_deprecate_positional_args\n",
    "    def __init__(self,\n",
    "                 n_splits=5,\n",
    "                 *,\n",
    "                 max_train_group_size=np.inf,\n",
    "                 max_test_group_size=np.inf,\n",
    "                 group_gap=None,\n",
    "                 verbose=False\n",
    "                 ):\n",
    "        super().__init__(n_splits, shuffle=False, random_state=None)\n",
    "        self.max_train_group_size = max_train_group_size\n",
    "        self.group_gap = group_gap\n",
    "        self.max_test_group_size = max_test_group_size\n",
    "        self.verbose = verbose\n",
    "\n",
    "    def split(self, X, y=None, groups=None):\n",
    "        \"\"\"Generate indices to split data into training and test set.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like of shape (n_samples, n_features)\n",
    "            Training data, where n_samples is the number of samples\n",
    "            and n_features is the number of features.\n",
    "        y : array-like of shape (n_samples,)\n",
    "            Always ignored, exists for compatibility.\n",
    "        groups : array-like of shape (n_samples,)\n",
    "            Group labels for the samples used while splitting the dataset into\n",
    "            train/test set.\n",
    "        Yields\n",
    "        ------\n",
    "        train : ndarray\n",
    "            The training set indices for that split.\n",
    "        test : ndarray\n",
    "            The testing set indices for that split.\n",
    "        \"\"\"\n",
    "        if groups is None:\n",
    "            raise ValueError(\n",
    "                \"The 'groups' parameter should not be None\")\n",
    "        X, y, groups = indexable(X, y, groups)\n",
    "        n_samples = _num_samples(X)\n",
    "        n_splits = self.n_splits\n",
    "        group_gap = self.group_gap\n",
    "        max_test_group_size = self.max_test_group_size\n",
    "        max_train_group_size = self.max_train_group_size\n",
    "        n_folds = n_splits + 1\n",
    "        group_dict = {}\n",
    "        u, ind = np.unique(groups, return_index=True)\n",
    "        unique_groups = u[np.argsort(ind)]\n",
    "        n_samples = _num_samples(X)\n",
    "        n_groups = _num_samples(unique_groups)\n",
    "        for idx in np.arange(n_samples):\n",
    "            if (groups[idx] in group_dict):\n",
    "                group_dict[groups[idx]].append(idx)\n",
    "            else:\n",
    "                group_dict[groups[idx]] = [idx]\n",
    "        if n_folds > n_groups:\n",
    "            raise ValueError(\n",
    "                (\"Cannot have number of folds={0} greater than\"\n",
    "                 \" the number of groups={1}\").format(n_folds,\n",
    "                                                     n_groups))\n",
    "\n",
    "        group_test_size = min(n_groups // n_folds, max_test_group_size)\n",
    "        group_test_starts = range(n_groups - n_splits * group_test_size,\n",
    "                                  n_groups, group_test_size)\n",
    "        for group_test_start in group_test_starts:\n",
    "            train_array = []\n",
    "            test_array = []\n",
    "\n",
    "            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n",
    "            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n",
    "                train_array_tmp = group_dict[train_group_idx]\n",
    "\n",
    "                train_array = np.sort(np.unique(\n",
    "                                      np.concatenate((train_array,\n",
    "                                                      train_array_tmp)),\n",
    "                                      axis=None), axis=None)\n",
    "\n",
    "            train_end = train_array.size\n",
    "\n",
    "            for test_group_idx in unique_groups[group_test_start:\n",
    "                                                group_test_start +\n",
    "                                                group_test_size]:\n",
    "                test_array_tmp = group_dict[test_group_idx]\n",
    "                test_array = np.sort(np.unique(\n",
    "                                              np.concatenate((test_array,\n",
    "                                                              test_array_tmp)),\n",
    "                                     axis=None), axis=None)\n",
    "\n",
    "            test_array  = test_array[group_gap:]\n",
    "\n",
    "\n",
    "            if self.verbose > 0:\n",
    "                    pass\n",
    "\n",
    "            yield [int(i) for i in train_array], [int(i) for i in test_array]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.032105,
     "end_time": "2021-11-27T08:17:13.17731",
     "exception": false,
     "start_time": "2021-11-27T08:17:13.145205",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"training\">Training 🏋️</span>\n",
    "<hr>\n",
    "\n",
    "Our model will be trained for the number of FOLDS and EPOCHS you chose in the configuration above. Each fold the model with lowest validation loss will be saved and used to predict OOF and test. Adjust the variable `VERBOSE`. The variable `VERBOSE=1 or 2` will display the training and validation loss for each epoch as text. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.031822,
     "end_time": "2021-11-27T08:17:13.302287",
     "exception": false,
     "start_time": "2021-11-27T08:17:13.270465",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Let's take a look at our CV**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2022-01-25T19:21:15.273599Z",
     "iopub.status.busy": "2022-01-25T19:21:15.273146Z"
    },
    "papermill": {
     "duration": 167.596868,
     "end_time": "2021-11-27T08:20:00.931193",
     "exception": false,
     "start_time": "2021-11-27T08:17:13.334325",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10):\n",
    "    cmap_cv = plt.cm.coolwarm\n",
    "    jet = plt.cm.get_cmap('jet', 256)\n",
    "    seq = np.linspace(0, 1, 256)\n",
    "    _ = np.random.shuffle(seq)   # inplace\n",
    "    cmap_data = ListedColormap(jet(seq))    \n",
    "    for ii, (tr, tt) in enumerate(list(cv.split(X=X, y=y, groups=group))):\n",
    "        indices = np.array([np.nan] * len(X))\n",
    "        indices[tt] = 1\n",
    "        indices[tr] = 0        \n",
    "        ax.scatter(range(len(indices)), [ii + .5] * len(indices), c=indices, marker='_', lw=lw, cmap=cmap_cv, vmin=-.2, vmax=1.2)\n",
    "    ax.scatter(range(len(X)), [ii + 1.5] * len(X), c=y, marker='_', lw=lw, cmap=plt.cm.Set3)\n",
    "    ax.scatter(range(len(X)), [ii + 2.5] * len(X), c=group, marker='_', lw=lw, cmap=cmap_data)\n",
    "    yticklabels = list(range(n_splits)) + ['target', 'day']\n",
    "    ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels, xlabel='Sample index', ylabel=\"CV iteration\", ylim=[n_splits+2.2, -.2], xlim=[0, len(y)])\n",
    "    ax.set_title('{}'.format(type(cv).__name__), fontsize=15)\n",
    "    return ax\n",
    "\n",
    "asset_id = 0\n",
    "df = load_training_data_for_asset(asset_id)\n",
    "df_proc = get_features(df)\n",
    "df_proc['date'] = df['date'].copy()\n",
    "df_proc['y'] = df['Target']\n",
    "df_proc = df_proc.dropna(how=\"any\")\n",
    "X = df_proc.drop(\"y\", axis=1)\n",
    "y = df_proc[\"y\"]\n",
    "groups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n",
    "X = X.drop(columns = 'date')\n",
    "\n",
    "fig, ax = plt.subplots(figsize = (12, 6))\n",
    "cv = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size=MAX_TRAIN_GROUP_SIZE, max_test_group_size=MAX_TEST_GROUP_SIZE)\n",
    "plot_cv_indices(cv, X, y, groups, ax, FOLDS, lw=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 4874.677043,
     "end_time": "2021-11-27T09:41:15.642029",
     "exception": false,
     "start_time": "2021-11-27T08:20:00.964986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\n",
    "VERBOSE = 2\n",
    "\n",
    "def get_Xy_and_model_for_asset(asset_id):\n",
    "    df = load_training_data_for_asset(asset_id)\n",
    "    df_proc = get_features(df)\n",
    "    df_proc['date'] = df['date'].copy()\n",
    "    df_proc['y'] = df['Target']\n",
    "    df_proc = df_proc.dropna(how=\"any\")\n",
    "    X = df_proc.drop(\"y\", axis=1)\n",
    "    y = df_proc[\"y\"]\n",
    "    groups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n",
    "    X = X.drop(columns = 'date')\n",
    "    oof_preds = np.zeros(len(X))\n",
    "    scores, models = [], []\n",
    "    \n",
    "    for fold, (train_idx, val_idx) in enumerate(PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size = MAX_TRAIN_GROUP_SIZE, max_test_group_size = MAX_TEST_GROUP_SIZE).split(X, y, groups)):\n",
    "        # GET TRAINING, VALIDATION SET\n",
    "        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n",
    "        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n",
    "\n",
    "        # DISPLAY FOLD INFO\n",
    "        if DEVICE == 'TPU':\n",
    "            if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "        print('#'*25); print('#### FOLD',fold+1)\n",
    "        print('#### Training WIDTH %s DEPTH %s | batch_size %s' % (WIDTH_NETS[fold], DEPTH_NETS[fold], BATCH_SIZES[fold]*REPLICAS))\n",
    "\n",
    "        # BUILD MODEL\n",
    "        K.clear_session()\n",
    "        with strategy.scope(): model = build_model(fold, dim = x_train.shape[1], weight = asset_weight_dict[asset_id])\n",
    "\n",
    "        # SAVE BEST MODEL EACH FOLD\n",
    "        sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5' % fold, monitor = 'val_loss', verbose = 0, save_best_only = True, save_weights_only = True, mode = 'min', save_freq = 'epoch')\n",
    "\n",
    "        # TRAIN\n",
    "        history = model.fit( x_train, y_train, epochs = EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], validation_data = (x_val, y_val), verbose=VERBOSE )\n",
    "        model.load_weights('fold-%i.h5' % fold)\n",
    "\n",
    "        # PREDICT OOF\n",
    "        pred = model.predict(x_val, verbose = VERBOSE)[2]\n",
    "        models.append(model)\n",
    "\n",
    "        # REPORT RESULTS\n",
    "        try: mse = mean_squared_error(np.nan_to_num(y_val), np.nan_to_num(pred))\n",
    "        except: mse = 0.0\n",
    "        scores.append(mse)\n",
    "        oof_preds[val_idx] = pred[:, 0]\n",
    "        w_score = corr(np.nan_to_num(y_val), np.nan_to_num(pred.flatten()), np.array([asset_weight_dict[asset_id]] * len(y_val)))\n",
    "        print('#### FOLD %i OOF MSE %.3f | WCORR: %.3f' % (fold + 1, mse, w_score))\n",
    "\n",
    "    df = df_proc\n",
    "    df['oof_preds'] = np.nan_to_num(oof_preds)\n",
    "    print('\\n\\n' + ('-' * 80) + '\\n' + 'Finished training %s. Results:' % asset_name_dict[asset_id])\n",
    "    print('Model: r2_score: %s | pearsonr: %s | wcorr: %s ' % (r2_score(df['y'], df['oof_preds']), pearsonr(df['y'], df['oof_preds'])[0], corr(df['y'].values, df['oof_preds'].values, np.array([asset_weight_dict[asset_id]] * len(df['y'].values)))))\n",
    "    print('Predictions std: %s | Target std: %s' % (df['oof_preds'].std(), df['y'].std()))\n",
    "    \n",
    "    try: plt.close()\n",
    "    except: pass   \n",
    "    df2 = df.reset_index().set_index('date')\n",
    "    fig = plt.figure(figsize = (12, 6))\n",
    "    # fig, ax_left = plt.subplots(figsize = (12, 6))\n",
    "    ax_left = fig.add_subplot(111)\n",
    "    ax_left.set_facecolor('azure')    \n",
    "    ax_right = ax_left.twinx()\n",
    "    ax_left.plot(df2['y'].rolling(3 * 30 * 24 * 60).corr(df2['oof_preds']).iloc[::24 * 60], color = 'crimson', label = \"Target WCorr\")\n",
    "    ax_right.plot(df2['Close'].iloc[::24 * 60], color = 'darkgrey', label = \"%s Close\" % asset_name_dict[asset_id])   \n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "    plt.xlabel('Time')\n",
    "    plt.title('3 month rolling pearsonr for %s' % (asset_name_dict[asset_id]))\n",
    "    plt.show()\n",
    "    \n",
    "    return scores, oof_preds, models, y\n",
    "\n",
    "models, scores, targets, oof_preds = {}, {}, {}, {}\n",
    "for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n",
    "    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n",
    "    cur_scores, cur_oof_preds, cur_models, cur_targets = get_Xy_and_model_for_asset(asset_id)\n",
    "    scores[asset_id], oof_preds[asset_id], models[asset_id], targets[asset_id] = np.mean(cur_scores), cur_oof_preds, cur_models, cur_targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.365578,
     "end_time": "2021-11-27T09:41:16.169668",
     "exception": false,
     "start_time": "2021-11-27T09:41:15.80409",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"codebook\">Calculate OOF MSE</span>\n",
    "The OOF (out of fold) predictions are saved to disk. If you wish to ensemble multiple models, use the OOF to determine what are the best weights to blend your models with. Choose weights that maximize OOF CV score when used to blend OOF. Then use those same weights to blend your test predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 164.095221,
     "end_time": "2021-11-27T09:44:00.427663",
     "exception": false,
     "start_time": "2021-11-27T09:41:16.332442",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# COMPUTE OVERALL OOF MSE\n",
    "print('Overall MEAN OOF MSE %s' % np.mean(list(scores.values())))\n",
    "\n",
    "# SAVE OOF TO DISK \n",
    "y_pred, y_true, weights = [], [], []\n",
    "for asset in oof_preds:\n",
    "    df_oof = pd.DataFrame(dict(asset_id = asset, oof_preds=oof_preds[asset]))\n",
    "    df_oof.to_csv(str(asset) + '_oof.csv',index=False)\n",
    "    y_pred += oof_preds[asset].tolist()\n",
    "    y_true += targets[asset].tolist() \n",
    "    weights += ([asset_weight_dict[asset]] * len(oof_preds[asset].tolist()))\n",
    "    print('%s score: %s' % (asset_name_dict[asset], corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten()))))\n",
    "    \n",
    "print('Overall score %s' % corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.169498,
     "end_time": "2021-11-27T09:44:00.763711",
     "exception": false,
     "start_time": "2021-11-27T09:44:00.594213",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\" id=\"submit\">Submit To Kaggle 🇰</span>\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 26.926705,
     "end_time": "2021-11-27T09:44:27.860629",
     "exception": false,
     "start_time": "2021-11-27T09:44:00.933924",
     "status": "completed"
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_df_test = []\n",
    "\n",
    "env = gresearch_crypto.make_env()\n",
    "iter_test = env.iter_test()\n",
    "\n",
    "for i, (df_test, df_pred) in enumerate(iter_test):\n",
    "    for j , row in df_test.iterrows():\n",
    "        try:\n",
    "            if row['Asset_ID'] in models:\n",
    "                cur_models = models[row['Asset_ID']]\n",
    "                x_test = get_features(row)\n",
    "                y_pred = np.mean(np.concatenate([np.expand_dims(model.predict(x_test.to_frame().T)[2][0], axis = 0) for model in cur_models], axis = 0), axis = 0)\n",
    "            else: y_pred = 0.0\n",
    "        except: \n",
    "            y_pred = 0.0\n",
    "            traceback.print_exc()\n",
    "        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n",
    "    all_df_test.append(df_test)\n",
    "    env.predict(df_pred)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.1671,
     "end_time": "2021-11-27T09:44:28.19687",
     "exception": false,
     "start_time": "2021-11-27T09:44:28.02977",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <span class=\"title-section w3-xxlarge\">References</span>\n",
    "\n",
    "<span id=\"f1\">1.</span> [Initial baseline notebook](https://www.kaggle.com/julian3833)<br>\n",
    "<span id=\"f2\">2.</span> [Competition tutorial](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)<br>\n",
    "<span id=\"f3\">3.</span> [Competition Overview](https://www.kaggle.com/c/g-research-crypto-forecasting/overview)</span><br>\n",
    "<span id=\"f4\">4.</span> [My Initial Ideas for this competition](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/284903)</span><br>\n",
    "<span id=\"f5\">5.</span> [My post notebook about cross validation](https://www.kaggle.com/yamqwe/let-s-talk-validation-grouptimeseriessplit)</span><br>\n",
    "<span id=\"f5\">6.</span> [Chris original notebook from SIIM ISIC](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)</span><br>\n",
    "\n",
    "<span class=\"title-section w3-large w3-tag\">WORK IN PROGRESS! 🚧</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>\n",
    "<center><img src=\"https://i.ibb.co/2M2MsgH/upvote7.png\" height=400 width=400></center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}