3-Intro a DeepLearning (redes).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Cargamos-librerias\" data-toc-modified-id=\"Cargamos-librerias-1\">Cargamos librerias</a></span><ul class=\"toc-item\"><li><span><a href=\"#metricas-de-evaluacion-(sigmas)-+-funciones-de-utilidad\" data-toc-modified-id=\"metricas-de-evaluacion-(sigmas)-+-funciones-de-utilidad-1.1\">metricas de evaluacion (sigmas) + funciones de utilidad</a></span></li><li><span><a href=\"#Datos-de-entrenamiento!\" data-toc-modified-id=\"Datos-de-entrenamiento!-1.2\">Datos de entrenamiento!</a></span></li><li><span><a href=\"#preprocesamiento-para-X-y-Y\" data-toc-modified-id=\"preprocesamiento-para-X-y-Y-1.3\">preprocesamiento para X y Y</a></span></li></ul></li><li><span><a href=\"#Tensorflow\" data-toc-modified-id=\"Tensorflow-2\">Tensorflow</a></span></li><li><span><a href=\"#Keras\" data-toc-modified-id=\"Keras-3\">Keras</a></span><ul class=\"toc-item\"><li><span><a href=\"#Red-sencillita\" data-toc-modified-id=\"Red-sencillita-3.1\">Red sencillita</a></span></li><li><span><a href=\"#Calidad-de-vida:-liveloss!\" data-toc-modified-id=\"Calidad-de-vida:-liveloss!-3.2\">Calidad de vida: liveloss!</a></span></li><li><span><a href=\"#Calidad-de-vida:-agregando-un-tensorboard\" data-toc-modified-id=\"Calidad-de-vida:-agregando-un-tensorboard-3.3\">Calidad de vida: agregando un tensorboard</a></span></li><li><span><a href=\"#Trucos:-Regularizadores-de-redes\" data-toc-modified-id=\"Trucos:-Regularizadores-de-redes-3.4\">Trucos: Regularizadores de redes</a></span></li><li><span><a href=\"#Actividad:-Crea-una-red-profunda\" data-toc-modified-id=\"Actividad:-Crea-una-red-profunda-3.5\">Actividad: Crea una red profunda</a></span></li><li><span><a href=\"#trucos:-Dropout\" data-toc-modified-id=\"trucos:-Dropout-3.6\">trucos: Dropout</a></span></li><li><span><a href=\"#Trucos:-BatchNormalization-y-Activacion!\" data-toc-modified-id=\"Trucos:-BatchNormalization-y-Activacion!-3.7\">Trucos: BatchNormalization y Activacion!</a></span></li><li><span><a href=\"#Keras-Functional\" data-toc-modified-id=\"Keras-Functional-3.8\">Keras Functional</a></span></li><li><span><a href=\"#Truco:-Skip-connections\" data-toc-modified-id=\"Truco:-Skip-connections-3.9\">Truco: Skip connections</a></span></li><li><span><a href=\"#Optimizador\" data-toc-modified-id=\"Optimizador-3.10\">Optimizador</a></span></li><li><span><a href=\"#Controlando-el-learning-rate\" data-toc-modified-id=\"Controlando-el-learning-rate-3.11\">Controlando el learning rate</a></span></li><li><span><a href=\"#Actividad:-Crea-una-red-poderosa!\" data-toc-modified-id=\"Actividad:-Crea-una-red-poderosa!-3.12\">Actividad: Crea una red poderosa!</a></span></li></ul></li><li><span><a href=\"#Pytorch\" data-toc-modified-id=\"Pytorch-4\">Pytorch</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](extra/atlas.png)\n",
    "# Cargamos librerias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:47.151173Z",
     "start_time": "2018-08-24T08:17:46.814786Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:47.725328Z",
     "start_time": "2018-08-24T08:17:47.152976Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sc\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from IPython.display import display\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## metricas de evaluacion (sigmas) + funciones de utilidad\n",
    "\n",
    "![http://i.imgur.com/Hflz2lG.jpg](http://i.imgur.com/Hflz2lG.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:34:42.166891Z",
     "start_time": "2018-08-24T08:34:42.157723Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "\n",
    "    \n",
    "def AMSScore(s,b): \n",
    "    return np.sqrt (2.*( (s + b + 10.)*np.log(1.+s/(b+10.))-s))\n",
    "\n",
    "def eval_model(Y_true_train,Y_pred_train,w_train,Y_true_test,Y_pred_test,w_test):\n",
    "    ratio =  float(len(X_train)) /float(len(X_test))\n",
    "    TruePositive_train = w_train*(Y_true_train==1.0)*(1.0/ratio)\n",
    "    TrueNegative_train = w_train*(Y_true_train==0.0)*(1.0/ratio)\n",
    "    TruePositive_valid = w_test*(Y_true_test==1.0)*(1.0/(1-ratio))\n",
    "    TrueNegative_valid = w_test*(Y_true_test==0.0)*(1.0/(1-ratio))\n",
    "    s_train = sum ( TruePositive_train*(Y_pred_train==1.0) )\n",
    "    b_train = sum ( TrueNegative_train*(Y_pred_train==1.0) )\n",
    "    s_test = sum ( TruePositive_valid*(Y_pred_test==1.0) )\n",
    "    b_test = sum ( TrueNegative_valid*(Y_pred_test==1.0) )\n",
    "    score_train = AMSScore(s_train,b_train)\n",
    "    score_test = AMSScore(s_test,b_test)\n",
    "    print('--- Resultados --')\n",
    "    print('- AUC train: {:.3f} '.format(roc_auc_score(Y_train,Y_train_pred)))\n",
    "    print('- AUC test : {:.3f} '.format(roc_auc_score(Y_test,Y_test_pred)))\n",
    "    print('- AMS train: {:.3f} sigma'.format(score_train))\n",
    "    print('- AMS test : {:.3f} sigma'.format(score_test))\n",
    "    return score_train, score_test\n",
    "\n",
    "def plot_roc(clf,Y_test,Y_test_prob):\n",
    "    fpr, tpr, thresholds = roc_curve(Y_test, Y_test_prob)\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    plt.plot(fpr, tpr, lw=1, alpha=0.3, label=str(clf.__class__.__name__))\n",
    "    plt.plot(np.linspace(0,1,100),np.linspace(0,1,100), lw=2, alpha=0.3, label='Suerte')\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.xlim([0,1])\n",
    "    plt.ylim([0,1])\n",
    "    plt.tight_layout()\n",
    "    return\n",
    "\n",
    "def keras_graph(model):\n",
    "    from IPython.display import SVG\n",
    "    from keras.utils.vis_utils import model_to_dot\n",
    "    return SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datos de entrenamiento!\n",
    "\n",
    "Quieres saber mas? Visita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.192476Z",
     "start_time": "2018-08-24T08:17:48.223762Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df=pd.read_csv('datos/training.csv')\n",
    "print(df.shape)\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## preprocesamiento para X y Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.327953Z",
     "start_time": "2018-08-24T08:17:49.193912Z"
    }
   },
   "outputs": [],
   "source": [
    "Y = df['Label'].replace(to_replace=['s','b'],value=[1,0]).values\n",
    "weights = df['Weight'].values\n",
    "X = df.drop(['EventId','Label','Weight'],axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.531551Z",
     "start_time": "2018-08-24T08:17:49.381349Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,Y_train,Y_test,w_train,w_test = train_test_split(X,Y,weights,train_size=0.3)\n",
    "print(X_train.shape,Y_train.shape,w_train.shape)\n",
    "print(X_test.shape,Y_test.shape,w_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow\n",
    "[![](extra/tf_logo.jpg)](https://www.tensorflow.org/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T07:38:39.497500Z",
     "start_time": "2018-08-24T07:38:38.473420Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras\n",
    "\n",
    "[![](extra/keras-logo.png)](https://keras.io/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:18:17.391905Z",
     "start_time": "2018-08-24T08:18:16.014973Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras import backend as K\n",
    "import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Red sencillita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:18:55.181558Z",
     "start_time": "2018-08-24T08:18:44.314653Z"
    }
   },
   "outputs": [],
   "source": [
    "K.clear_session()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(units=10, input_dim=X_train.shape[-1], kernel_initializer='normal', activation='relu'))\n",
    "model.add(Dense(1, kernel_initializer='normal', activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:04:10.725399Z",
     "start_time": "2018-08-24T08:04:04.945904Z"
    }
   },
   "outputs": [],
   "source": [
    "pcut=0.5\n",
    "Y_train_pred = model.predict(X_train).flatten() > pcut\n",
    "Y_test_pred = model.predict(X_test).flatten() > pcut\n",
    "Y_train_prob= model.predict_proba(X_train).flatten()\n",
    "Y_test_prob = model.predict_proba(X_test).flatten()\n",
    "eval_model(Y_train,Y_train_pred,w_train,Y_test,Y_test_pred,w_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calidad de vida: liveloss!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T07:42:02.516816Z",
     "start_time": "2018-08-24T07:41:44.849722Z"
    }
   },
   "outputs": [],
   "source": [
    "from livelossplot import PlotLossesKeras\n",
    "\n",
    "model.fit(X_train, Y_train,\n",
    "          epochs=5, batch_size=32,\n",
    "          validation_data=(X_test, Y_test),\n",
    "          callbacks=[PlotLossesKeras()],\n",
    "          verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:24:25.433970Z",
     "start_time": "2018-08-24T08:24:25.327676Z"
    }
   },
   "outputs": [],
   "source": [
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calidad de vida: agregando un tensorboard\n",
    "Para poder ver tu graph y metrics corre tensorboard\n",
    "```\n",
    "tensorboard --logdir Graph/\n",
    "```\n",
    "podras visualizar todo en la liga [http://localhost:6006](http://localhost:6006)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T07:51:12.631378Z",
     "start_time": "2018-08-24T07:49:27.930434Z"
    }
   },
   "outputs": [],
   "source": [
    " tb_call = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True)\n",
    "\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32,callbacks=[tb_call])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trucos: Regularizadores de redes\n",
    "\n",
    "**L2** pesos mas \"equitativos\"\n",
    "**L1** pesos mas \"sparse\", muchos 0's."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:59:09.423193Z",
     "start_time": "2018-08-24T08:58:55.107420Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras import regularizers\n",
    "\n",
    "K.clear_session()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(units=10, input_dim=X_train.shape[-1],\n",
    "                kernel_regularizer=regularizers.l2(0.01),\n",
    "                activity_regularizer=regularizers.l1(0.01),\n",
    "                activation='relu'))\n",
    "model.add(Dense(1, kernel_initializer='normal', activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actividad: Crea una red profunda\n",
    "\n",
    "Sugerencias:\n",
    "* Agrega mas capas\n",
    "* Hay alguna relacion entre las dimensiones de las capas?\n",
    "* Como calculas el numero de parametros a entrenar?\n",
    "* Cual es el efecto del learning rate?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## trucos: Dropout\n",
    "\n",
    "Matar conexiones de manera aleatoria puede mejorar las conexiones.\n",
    "\n",
    "Mas info en https://keras.io/layers/core/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T07:55:16.004669Z",
     "start_time": "2018-08-24T07:55:07.388156Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Activation, Dropout\n",
    "\n",
    "K.clear_session()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(units=10, input_dim=X_train.shape[-1], kernel_initializer='normal', activation='relu'))\n",
    "model.add(Dropout(0.2)) # 20% de las conexiones se apagan\n",
    "model.add(Dense(1, kernel_initializer='normal', activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trucos: BatchNormalization y Activacion!\n",
    "\n",
    "Algunas activaciones https://keras.io/layers/advanced-activations/\n",
    "\n",
    "Batch Normalization https://keras.io/layers/normalization/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:03:56.679496Z",
     "start_time": "2018-08-24T08:01:25.474946Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Activation, Dropout, BatchNormalization\n",
    "\n",
    "K.clear_session()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(units=100, input_dim=X_train.shape[-1], kernel_initializer='normal'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('tanh'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1, kernel_initializer='normal', activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras Functional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:35:10.930191Z",
     "start_time": "2018-08-24T08:34:55.006939Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense\n",
    "\n",
    "K.clear_session()\n",
    "\n",
    "inputs = Input(shape=(X_train.shape[-1],))\n",
    "x = Dense(64, activation='relu')(inputs)\n",
    "x = Dense(64, activation='relu')(x)\n",
    "pred = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "model = Model(inputs=inputs, outputs=pred)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)\n",
    "pcut=0.5\n",
    "Y_train_pred = model(X_train).flatten() > pcut\n",
    "Y_test_pred = model(X_test).flatten() > pcut\n",
    "Y_train_prob= model(X_train).flatten()\n",
    "Y_test_prob = model(X_test).flatten()\n",
    "eval_model(Y_train,Y_train_pred,w_train,Y_test,Y_test_pred,w_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Truco: Skip connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:25:19.434913Z",
     "start_time": "2018-08-24T08:25:06.632221Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Concatenate\n",
    "\n",
    "K.clear_session()\n",
    "\n",
    "inputs = Input(shape=(X_train.shape[-1],))\n",
    "x1 = Dense(64, activation='relu')(inputs)\n",
    "x2 = Dense(64, activation='relu')(x1)\n",
    "skipped = Concatenate()([x1,x2])\n",
    "pred = Dense(1, activation='sigmoid')(skipped)\n",
    "\n",
    "model = Model(inputs=inputs, outputs=pred)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=5, batch_size=32)\n",
    "pcut=0.5\n",
    "Y_train_pred = model(X_train).flatten() > pcut\n",
    "Y_test_pred = model(X_test).flatten() > pcut\n",
    "Y_train_prob= model(X_train).flatten()\n",
    "Y_test_prob = model(X_test).flatten()\n",
    "eval_model(Y_train,Y_train_pred,w_train,Y_test,Y_test_pred,w_test)\n",
    "keras_graph(model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizador\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "![](extra/saddle_point_evaluation_optimizers.gif)\n",
    "\n",
    "Imagenes de Sebastian Ruder (http://ruder.io)\n",
    "\n",
    "![](extra/beales_20171117_00-02-20_2d.gif)\n",
    "\n",
    "Imagenes de https://github.com/wassname/viz_torch_optim'\n",
    "\n",
    "\n",
    "Y hay muchos mas en camino! (http://www.fast.ai/2018/07/02/adam-weight-decay/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:33:37.789587Z",
     "start_time": "2018-08-24T08:32:15.479848Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Concatenate\n",
    "from keras import optimizers\n",
    "\n",
    "lr = 0.01\n",
    "K.clear_session()\n",
    "for opt in [optimizers.SGD(lr=lr),\n",
    "            optimizers.SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True),\n",
    "            optimizers.Adam(lr=lr),\n",
    "            optimizers.Adagrad(lr=lr)\n",
    "           ]:\n",
    "    print('===={}==='.format(str(opt.__class__)))\n",
    "    inputs = Input(shape=(X_train.shape[-1],))\n",
    "    x1 = Dense(64, activation='relu')(inputs)\n",
    "    x2 = Dense(64, activation='relu')(x1)\n",
    "    skipped = Concatenate()([x1,x2])\n",
    "    pred = Dense(1, activation='sigmoid')(skipped)\n",
    "    model = Model(inputs=inputs, outputs=pred)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "    model.fit(X_train, Y_train, epochs=5, batch_size=32)\n",
    "    pcut=0.5\n",
    "    Y_train_pred = model(X_train).flatten() > pcut\n",
    "    Y_test_pred = model(X_test).flatten() > pcut\n",
    "    Y_train_prob= model(X_train).flatten()\n",
    "    Y_test_prob = model(X_test).flatten()\n",
    "    eval_model(Y_train,Y_train_pred,w_train,Y_test,Y_test_pred,w_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Controlando el learning rate\n",
    "\n",
    "![image.png](extra/learn_rate.png)\n",
    "\n",
    "Imagen de [cs231n](http://cs231n.stanford.edu/)\n",
    "\n",
    "![](extra/lr_finder.png)\n",
    "\n",
    "![](extra/Lanscape_lr.png)\n",
    "\n",
    "![](extra/cyclic_lr.png)\n",
    "\n",
    "* Cyclical Learning Rates for Training Neural Networks (https://arxiv.org/abs/1506.01186)\n",
    "* Setting the learning rate of your neural network. (https://www.jeremyjordan.me/nn-learning-rate/)\n",
    "* SNAPSHOT ENSEMBLES: TRAIN 1, GET M FOR FREE (https://arxiv.org/pdf/1704.00109.pdf)\n",
    "* SGDR: STOCHASTIC GRADIENT DESCENT WITH WARM RESTARTS (https://arxiv.org/pdf/1608.03983.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T09:03:13.188922Z",
     "start_time": "2018-08-24T09:02:58.664235Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import LearningRateScheduler\n",
    "class LossHistory(keras.callbacks.Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.losses = []\n",
    "        self.lr = []\n",
    "    def on_epoch_end(self, batch, logs={}):\n",
    "        self.losses.append(logs.get('loss'))\n",
    "        self.lr.append(step_decay(len(self.losses)))\n",
    "        \n",
    "def step_decay(epoch):\n",
    "    initial_lrate = 0.1\n",
    "    drop = 0.5\n",
    "    epochs_drop = 1.0\n",
    "    lrate = initial_lrate * np.power(drop,  \n",
    "           np.floor((1+epoch)/epochs_drop))\n",
    "    return lrate\n",
    "\n",
    "loss_history = LossHistory()\n",
    "lrate = LearningRateScheduler(step_decay)\n",
    "callbacks = [loss_history, lrate]\n",
    "\n",
    "\n",
    "hist = model.fit(X_train, Y_train,\n",
    "          epochs=5, batch_size=32,callbacks=callbacks)\n",
    "\n",
    "plt.plot(loss_history.losses,loss_history.lr)\n",
    "plt.xlabel('learning rate')\n",
    "plt.ylabel('loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actividad: Crea una red poderosa!\n",
    "\n",
    "Sugerencias:\n",
    "* Usa los trucos que vimos\n",
    "* Cual es el efecto de entrenar con un learning rate bajo, alto?\n",
    "* Ojo con le optimizador y el learning rate, es super importante!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch\n",
    "\n",
    "\n",
    "[![](extra/pytorch-logo.png)](https://pytorch.org/)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T09:43:38.406384Z",
     "start_time": "2018-08-24T09:43:38.400298Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T09:45:01.935004Z",
     "start_time": "2018-08-24T09:45:01.914747Z"
    }
   },
   "outputs": [],
   "source": [
    "class Net(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(30, 50)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.dout = nn.Dropout(0.2)\n",
    "        self.fc2 = nn.Linear(50, 100)\n",
    "        self.prelu = nn.PReLU(1)\n",
    "        self.out = nn.Linear(100, 1)\n",
    "        self.out_act = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, input_):\n",
    "        a1 = self.fc1(input_)\n",
    "        h1 = self.relu1(a1)\n",
    "        dout = self.dout(h1)\n",
    "        a2 = self.fc2(dout)\n",
    "        h2 = self.prelu(a2)\n",
    "        a3 = self.out(h2)\n",
    "        y = self.out_act(a3)\n",
    "        return y\n",
    "    \n",
    "net = Net()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=0.001)\n",
    "criterion = nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T12:58:40.945947Z",
     "start_time": "2018-08-24T12:58:08.229410Z"
    }
   },
   "outputs": [],
   "source": [
    "e_losses = []\n",
    "num_epochs = 5\n",
    "batch_size=128\n",
    "for e in range(num_epochs):\n",
    "    print(e,end='..')\n",
    "    losses=[]\n",
    "    for beg_i in range(0, X.shape[0], batch_size):\n",
    "        x_batch = torch.from_numpy(X)[beg_i:beg_i + batch_size, :]\n",
    "        y_batch = torch.from_numpy(Y.reshape(-1,1))[beg_i:beg_i + batch_size, :]\n",
    "        x_batch = Variable(x_batch).float()\n",
    "        y_batch = Variable(y_batch).float()\n",
    "\n",
    "        opt.zero_grad()\n",
    "        # (1) Forward\n",
    "        y_hat = net(x_batch)\n",
    "        # (2) Compute diff\n",
    "        loss = criterion(y_hat, y_batch)\n",
    "        # (3) Compute gradients\n",
    "        loss.backward()\n",
    "        # (4) update weights\n",
    "        opt.step()        \n",
    "        losses.append(loss.data.numpy())\n",
    "    e_losses.append(np.mean(losses))\n",
    "\n",
    "plt.plot(e_losses)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:anaconda3]",
   "language": "python",
   "name": "conda-env-anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "209px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}