12_custom_models_and_training_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 12 – Custom Models and Training with TensorFlow**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 12, as well as code examples from Appendix C_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a href=\"https://colab.research.google.com/github/ageron/handson-ml3/blob/main/12_custom_models_and_training_with_tensorflow.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
    "  </td>\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://kaggle.com/kernels/welcome?src=https://github.com/ageron/handson-ml3/blob/main/12_custom_models_and_training_with_tensorflow.ipynb\"><img src=\"https://kaggle.com/static/images/open-in-kaggle.svg\" /></a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This project requires Python 3.7 or above:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "assert sys.version_info >= (3, 7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And TensorFlow ≥ 2.8:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "assert tf.__version__ >= \"2.8.0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using TensorFlow like NumPy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensors and Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]]) # matrix\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scalars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Keras's low-level API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You may still run across code that uses Keras's low-level API:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K = tf.keras.backend\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But since Keras does not support multiple backends anymore, you should instead use TF's low-level API directly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(tf.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensors and NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Type Conversions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  0.],\n",
       "       [ 8., 10.,  1.]], dtype=float32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[:, 2].assign([0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   0.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(\n",
    "    indices=[[0, 0], [1, 2]], updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to use scatter_update()\n",
    "sparse_delta = tf.IndexedSlices(values=[[1., 2., 3.], [4., 5., 6.]],\n",
    "                                indices=[1, 0])\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code in this section and all the following sections  in appendix C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Other Data Structures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code in this section is in Appendix C."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([4, 6, 5, 2], dtype=int32)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.length(p, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.strings.unicode_decode(p, \"UTF8\")\n",
    "r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(6,), dtype=int32, numpy=array([ 67, 111, 102, 102, 101, 101], dtype=int32)>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232]]>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r[1:3]  # extra code – a slice of a ragged tensor is a ragged tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857], [65, 66], [], [67]]>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2 = tf.ragged.constant([[65, 66], [], [67]])\n",
    "tf.concat([r, r2], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233, 68, 69, 70], [67, 111, 102, 102, 101, 101, 71], [99, 97, 102, 102, 232], [21654, 21857, 72, 73]]>\n"
     ]
    }
   ],
   "source": [
    "r3 = tf.ragged.constant([[68, 69, 70], [71], [], [72, 73]])\n",
    "print(tf.concat([r, r3], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4, 6), dtype=int32, numpy=\n",
       "array([[   67,    97,   102,   233,     0,     0],\n",
       "       [   67,   111,   102,   102,   101,   101],\n",
       "       [   99,    97,   102,   102,   232,     0],\n",
       "       [21654, 21857,     0,     0,     0,     0]], dtype=int32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.to_tensor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sparse tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],\n",
    "                    values=[1., 2., 3.],\n",
    "                    dense_shape=[3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 1., 0., 0.],\n",
       "       [2., 0., 0., 0.],\n",
       "       [0., 0., 0., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.sparse_tensor.SparseTensor at 0x7f84a6749f10>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s * 42.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsupported operand type(s) for +: 'SparseTensor' and 'float'\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    s + 42.0\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 30.,  40.],\n",
       "       [ 20.,  40.],\n",
       "       [210., 240.]], dtype=float32)>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to multiply a sparse tensor and a dense tensor\n",
    "s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])\n",
    "tf.sparse.sparse_dense_matmul(s, s4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indices[1] = [0,1] is out of order. Many sparse ops require sorted indices.\n",
      "    Use `tf.sparse.reorder` to create a correctly ordered copy.\n",
      "\n",
      " [Op:SparseToDense]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-17 10:32:40.424119: W tensorflow/core/framework/op_kernel.cc:1692] OP_REQUIRES failed at sparse_to_dense_op.cc:162 : Invalid argument: indices[1] = [0,1] is out of order. Many sparse ops require sorted indices.\n",
      "    Use `tf.sparse.reorder` to create a correctly ordered copy.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# extra code – when creating a sparse tensor, values must be given in \"reading\n",
    "#              order\", or else `to_dense()` will fail.\n",
    "s5 = tf.SparseTensor(indices=[[0, 2], [0, 1]],  # WRONG ORDER!\n",
    "                     values=[1., 2.],\n",
    "                     dense_shape=[3, 4])\n",
    "try:\n",
    "    tf.sparse.to_dense(s5)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 2., 1., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to fix the sparse tensor s5 by reordering its values\n",
    "s6 = tf.sparse.reorder(s5)\n",
    "tf.sparse.to_dense(s6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tensor Arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = tf.TensorArray(dtype=tf.float32, size=3)\n",
    "array = array.write(0, tf.constant([1., 2.]))\n",
    "array = array.write(1, tf.constant([3., 10.]))\n",
    "array = array.write(2, tf.constant([5., 7.]))\n",
    "tensor1 = array.read(1)  # returns (and zeros out!) tf.constant([3., 10.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 1.,  2.],\n",
       "       [ 3., 10.],\n",
       "       [ 5.,  7.]], dtype=float32)>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to disable clear_after_read\n",
    "array2 = tf.TensorArray(dtype=tf.float32, size=3, clear_after_read=False)\n",
    "array2 = array2.write(0, tf.constant([1., 2.]))\n",
    "array2 = array2.write(1, tf.constant([3., 10.]))\n",
    "array2 = array2.write(2, tf.constant([5., 7.]))\n",
    "tensor2 = array2.read(1)  # returns tf.constant([3., 10.])\n",
    "array2.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to create and use a tensor array with a dynamic size\n",
    "array3 = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)\n",
    "array3 = array3.write(0, tf.constant([1., 2.]))\n",
    "array3 = array3.write(1, tf.constant([3., 10.]))\n",
    "array3 = array3.write(2, tf.constant([5., 7.]))\n",
    "tensor3 = array3.read(1)\n",
    "array3.stack()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.sparse_tensor.SparseTensor at 0x7f84a1c77400>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = tf.constant([[1, 5, 9]])\n",
    "b = tf.constant([[5, 6, 9, 11]])\n",
    "u = tf.sets.union(a, b)\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 5), dtype=int32, numpy=array([[ 1,  5,  6,  9, 11]], dtype=int32)>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 5), dtype=int32, numpy=\n",
       "array([[ 1,  5,  6,  9, 11],\n",
       "       [ 0, 10, 13,  0,  0]], dtype=int32)>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = tf.constant([[1, 5, 9], [10, 0, 0]])\n",
    "b = tf.constant([[5, 6, 9, 11], [13, 0, 0, 0]])\n",
    "u = tf.sets.union(a, b)\n",
    "tf.sparse.to_dense(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 5), dtype=int32, numpy=\n",
       "array([[ 1,  5,  6,  9, 11],\n",
       "       [-1, 10, 13, -1, -1]], dtype=int32)>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to use a different default value: -1 in this case\n",
    "a = tf.constant([[1, 5, 9], [10, -1, -1]])\n",
    "b = tf.constant([[5, 6, 9, 11], [13, -1, -1, -1]])\n",
    "u = tf.sets.union(a, b)\n",
    "tf.sparse.to_dense(u, default_value=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[2, 3, 7],\n",
       "       [7, 0, 0]], dtype=int32)>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to use `tf.sets.difference()`\n",
    "set1 = tf.constant([[2, 3, 5, 7], [7, 9, 0, 0]])\n",
    "set2 = tf.constant([[4, 5, 6], [9, 10, 0]])\n",
    "tf.sparse.to_dense(tf.sets.difference(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[5, 0],\n",
       "       [0, 9]], dtype=int32)>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to use `tf.sets.difference()`\n",
    "tf.sparse.to_dense(tf.sets.intersection(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1,), dtype=bool, numpy=array([ True])>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – check whether set1[0] contains 5\n",
    "tf.sets.size(tf.sets.intersection(set1[:1], tf.constant([[5, 0, 0, 0]]))) > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Queues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = tf.queue.FIFOQueue(3, [tf.int32, tf.string], shapes=[(), ()])\n",
    "q.enqueue([10, b\"windy\"])\n",
    "q.enqueue([15, b\"sunny\"])\n",
    "q.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=int32, numpy=10>,\n",
       " <tf.Tensor: shape=(), dtype=string, numpy=b'windy'>]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.dequeue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "q.enqueue_many([[13, 16], [b'cloudy', b'rainy']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(3,), dtype=int32, numpy=array([15, 13, 16], dtype=int32)>,\n",
       " <tf.Tensor: shape=(3,), dtype=string, numpy=array([b'sunny', b'cloudy', b'rainy'], dtype=object)>]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.dequeue_many(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAD8CAYAAACiqQeGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABH9ElEQVR4nO3dd3gUVffA8e8JBEJHIAhIU+kiXYogXaWoqKg/RLGLgI1XsOtre21YaYqIHQugoggoCoTeBKnSpUhVegiQQnJ+f9xFAwSyIbuZ3c35PM8+ye7OzpzJbPbsnbn3XFFVjDHGGBMaorwOwBhjjDH/ssRsjDHGhBBLzMYYY0wIscRsjDHGhBBLzMYYY0wIscRsjDHGhBBLzMaEERFpLSIqIqVyaHu3iUhCTmzLGONYYjYmyETkYxEZn8HjjXxJtrIHYRljQpQlZmMMIpLP6xiMMY4lZmNCREanqUWksu+xRics3lRElohIoogsEpGGJ6zrYhGZLiKHRWSbiLwrIkXTPT/N99jrIrILmJ2FOO8RkfUikuz7eXcGz6/1xbZLRCaJSF7fcxeKyBQRiReRgyKyVETaZOXvZEyks8RsTHh6HXgUaARsACaISEFwyQ/4GRgH1AWuBeoBH56wjpsBAS4BbvFnoyJyDTAEeBuoDQwE3hGRK33PNwKGAs8B1YH2wE/pVvEFsANoDNQHngUS/dxnY3KFvF4HYEwu0SGDTlTZ+WL8gqpOAhCR24GtQHdgBPAwMEpV3zi2sIj0BhaLSGlV/dv38EZV7ZfF7fYHPlPVIb77a32t9UeBH4CKwCFgnKoeBDYDS9O9vhLwuqqu9t1fn8XtGxPxrMVsTM6YgWu1pr91z8b65h77RVUTgOVALd9DDYGbRSTh2I1/T1Wfn24di85guzU5+bT3rHTb/gWXjDeKyOcicquIFEm37JvACBGZKiJPikiNM4jBmIhmidmYnHFYVdenv+Faueml+X5Kuseiz2BbUbiWc710t7pAVWBJuuUOncG6ATKakk4BfK3kBsANwJ/A48BqESnne/5ZXBL/DrgYWCYid5xhHMZEJEvMxoSOXb6fZdM9Vu8UyzY99ouIFMJd713le+g34IITvwj4bkeyGeMqoMUJj7UAVh67o6pHVXWqqj4O1AEKAVeke36dqg5S1c7AB8Bd2YzJmIhi15iNCR3rgS3AsyLyGFAZeOoUyz7l6029HfgvkIzrWAXwKjBPRIYB7wEHgRrAlap6TzZjfA0YIyKLcB3MOgA34TqYISJX4E6XzwD2Am2AIsAqESmA67Q2BtgEnI1L6vOzGZMxEcUSszEhQlVTRKQb8A6uw9QS4AngpOIkwGPAG7iez78DV6jqId96lolIS+B/wHQgD67n9tgAxPidiNyP6wT2Nu56ch9V/cG3yH7gatyXhYLAH8BdqjrTN1b6LOAToAywx7dv/bMblzGRRFQzulxkjDHGGC/YNWZjjDEmhPidmEUkj4gsPkXNXxGRQb4qQMtEpEFgwzTGGGNyh6y0mB/k316fJ+qIG4pRFegJvJvNuIwxxphcya/ELCLlgc64sZEZ6QJ8qs48oLiIlD3FssYYY4w5BX9bzG8Dj/BvAYQTnYMb5nHMVt9jxhhjjMmCTIdL+cYl/q2qi0Sk9akWy+Cxk7p7i0hP3KluYmJiGlasWNH/SMNMWloaUVGn/94TfeAAqTExpOXPn0NRBY4/+xfOInX/tmzZgqqS2//3wlk47p+kpiJHj/r1WReO+5cVa9eu3a2qsadbxp9xzM2Bq0SkExADFBWRkap6c7pltgIV0t0vjyt8cBxVHQ4MB6hevbquWbPGj82Hp2nTptG6devMF0xOBhGIPpPKi97xe//CVKTuX+vWrdm/fz9LlizxOpSgidRjd0zY7d/WrZCQADX8K4sedvuXRSKyObNlMv1aoqqPq2p5Va0MdAOmnpCUwU0vd4uvd3ZT4ICq7jiToHOdW2+FKVO8jsIYY4Jj2TKYMMHrKMLKGVf+EpFeAKo6DJgIdMKVFDwM3B6Q6HKDjz6CmBivozDGmODo1MndjN+ylJhVdRowzff7sHSPK3BvIAPLNWJi4P33oXFjqFvX62iMMSZwPvoItm+HJ5/0OpKwYrWyQ0G5clCwoNdRGGNMYP3f/8HevV5HEXYsMYeCzp1h/36Ij4eiRb2Oxhhjsm/pUkhKcmcDTZZEbp/0cPP009YJzBgTOXbsgM2ZdkA2GbAWc6gYNMgNmzLGmOzo3RvGjXPXds9k9sAtW+C229zro6LcGb1XX83a51NKCnTokPVtG8BazKFDBIYNg++/9zoSY0w4u/FG+O23M3993rwuEa9aBYsXw/z58O23WVvHq6/CG2+ceQy5nLWYQ8nFF8NZZ3kdhTEmnLVsmb3Xly3rbgD58kGdOq4VnRWPPw5HjmQvjlzMWsyhpE4dd+rJrssYY0LBnj3w3Xdw+eX+v2b8eJgzBwoXDlpYkc5azKFm3DgoVQoqVfI6EmNMbpaUBNddB337Qs2a/r8uJsaKJmWTJeZQc999XkdgjMntUlPhppugfn3o18//1+3eDW3aQJ48wYstF7BT2aHo/ffhtde8jsIYE4lGjXKdTTO6xce7Ze65B4oUyXoHrgED4NNPAx9zLmMt5lB05ZVQoIDXURhjwtFdd8FPP7nfy5d3w5ZGjPj3+ZYtYe7cf+/v3Qs9ekDTpq7A0ezZ8MEHULu2azED3HEHPPBA5tseMADS0gK3L7mUJeZQVKYMrFzphis0bep1NMaYcJI+CWckfa/r+Hho184l4TFj3GPNm5/Z+Oe33oKLLoIWLbL+WnMcO5UdqrZsgY0bvY7CGBOpDh1ysz5FRbme1Nmt19+0KVSuHJDQcjtrMYeqY8MTUlIgOtrbWIwxkSUxEbp0gYQEiItz15OzY9UqN9yzUKHAxJfLWYs5lH3xBTz0kNdRGGPCxak6dR27gfuy37WrK7n5yy+BKWo0YgQsWJD99RjAjxaziMQAM4D8vuW/VtVnTlimNfA9cOzc67eq+nxAI82Nrr7ajSM0xhh/ZHZtODXVlexcswZmzIDY2MBs18pvBpQ/LeYkoK2q1gXqAR1EJKMeSTNVtZ7vZkk5EAoWhLVr4ZNPvI7EGBMOtmxxnblq1oQLLoBHHjk+WffuDRMnwosvwp9/wrx57rZ8+Zlv8+abYcmSbIeeG/jbYT3TFrOqKpDguxvtu51Bl73jbd1agAMHoFix7K4pwhUsaEOnjDH+OTYBRaNGkJwMl17qJqDo2tUl6K++cjWsu3U7/nVXXumqDp6Jp5+Gc8/NfuwRbv16dxj8IepHt3gRyQMsAqoAQ1X10ROebw18A2wFtgP9VfX3DNbTE+jp7jVsWLnydF5+eTllyiT6F20YSUhIoHCgasWqkm/3bpIDddopAAK6fyEoUvevb9++pKamMnjwYK9DCZpIPXbHZGX/qgwaRGK5cmwN0iWxUrNmsa9ePVID+PeOxOO3fHkxnnqqNvHx0YAsUtVGp32Bqvp9A4oDcUDtEx4vChT2/d4JWJfZuvLlq6+gWrq06rx5GnHi4uICt7IFC1SvvDJw6wuAgO5fCIrU/WvVqpXWrVvX6zCCKlKP3TF+79/u3arly6uuXBmcQNLSVPv2Vd2zJ6CrjbTjN3Kkar58qqDasaMqsFAzyY9Z6pWtqvuBaUCHEx6PV9UE3+8TgWgRKXW6dVWseJhLL4W//4bWrf8d224y0KiRzdNsjPHfmU5AkRUpKa6oSIkSwVl/mFOFZ591l+CTk900CP5eLcg0MYtIrIgU9/1eAGgPrD5hmTIiri++iDT2rXfPaTccpUyYAD17uiF1N9wAL798ZgVnIp4IbN0Kd9/tdSTGmFB3phNQZMXRo3Dhha6cpzlJYqI7BM895+q3DBoEgwe7LgD+8GexssAnvuvMUcBoVR0vIr0AVHUYcB3QW0SOAkeAbqqZp9joaBg2DKpXh/794YknXCfk995z83ObdMqWdd9eVP8dj2iMMSc60wkosiJvXvj1V1db2xxn1y645hpXcrxwYdffrnPnrK0j0xazqi5T1fqqWkdVa6tvKJSqDvMlZVR1iKpeoKp1VbWpqs7xNwARV0Pj229dB+SPP4bLLrMvYifJm9ed85850+tIjDGh6tgEFAsXuhZzvXquuRZIqanw5JOQP39g1xsBVq92lUlnz3bzh8yalfWkDCFUkvPqq13OufJKmD7d7dyECVC1qteRhZDERHj7bbj4Yv/PiRhjco8znYAiK5KS4Jxz7LTmCaZMccOhDhyAhg3hhx/+nSskq0KqJGeDBjB/PtStC+vWueQ8Y4bXUYWQIkXcqQWbhNwY45U9e6BPH7ukls4HH7jZNQ8ccI3M6dPPPClDiCVm+Lf5f8UV7nR2+/Y27/ZxkpLcKaqEhMyXNcaYQNqxw/X2tjmXAfdnePRRNwX20aPw8MPwzTfZn8sj5BIzuAvm333nevqnpMCtt7riMvZewF3X+f5790cyxpicVLasK+EZFZKpI0cdPgzXXw8DBriTmMOHu98D8acJ2b9unjxuiNzQoe73//0Pund3l1lzvYoV3R8mOdnrSIwxucXGja5paKew2bEDWrVyVxaLFYOffgrsaNaQTczH9Onj5vAuUgRGjYK2bV1RklxNBPbvdxc0jDEmJ5QuDXfc4XUUnlu2DJo0cR3fzz0X5s51l1wDKeQTM7iL6rNnu4bi3Lnuj7JypddReezYcAU7v2+MCbY9e9wMVBdf7HUknpo40XV837IFmjVzZ/WDUVgtLBIzuCIz8+dD48awaZP7o/zyi9dReaxrV1i82OsojDGR7o8/4McfvY7CU0OHuuG8CQlucq6pU91JhGAIm8QMUKYMxMW5ToHx8dCxo7vgnmtNmOAGzBljTLCkpbkW0XPPeR2JJ1JT4cEHXa3rtDT473/hiy8gJiZ42wyrxAyuOtioUfD44+4Pds89rpxnaqrXkXkgXz4YONCd5zfGmGAYPtxlo1zo4EHo0sUVT4uOdkN3n3su+P3fwrJ8VFQUvPSSqwrWs6crCbt+PXz+efbHj4WdRo3cxXdjjAmGu+7KlR1Nt2xxp66XLnUTaI0dCy1b5sy2w67FnN7tt8PPP8NZZ7mhvS1bwvbtXkeVw5o3d+PJNm/2OhJjTKSZMMFNVlGypNeR5KhFi1wn46VLoVo1178pp5IyhHliBmjTxvXUPv98+O03dylkyRKvo8phY8da7VJjTODlyZPrSgB/951Lwjt2uHmD5s6FKlVyNoawT8zgpo2cNw8uuQS2bYMWLdzY51zj3nuhRw+bzNoYEzg7drip/ho39jqSHKEKr78O117rqnrddhtMmuROY+e0iEjMAKVKueFTN98Mhw65C/YDB+aiXDV2bPAmRTfG5D5PPukyUy6QkgK9erla16quD9OHH3o3gVamnb9EJAaYAeT3Lf+1qj5zwjICDAQ6AYeB21T1t8CHe3r587tec9WquU6EffvC2rUuQUf8LIlt2rgaccYYEwgffOB1BDli/35X83ryZDcE6tNP3X0v+dNiTgLaqmpdoB7QQUSanrBMR6Cq79YTeDeQQWaFiJvw4osvXKJ+5x3Xsy4+3quIckjx4m7k+5gxXkdijAl3PXu6oiIRXhd740ZXzGzyZFcsZNo075My+JGY1Tk2x2C073biCeIuwKe+ZecBxUUkG7NRZt+NN7rKLLGxrsB48+a5pOPytm1eR2CMCXc33QSVKnkdRVAdK++8ahVccIHred2kiddROaJ+XIQVkTzAIqAKMFRVHz3h+fHAK6o6y3d/CvCoqi48YbmeuBY1sbGxDUePHh2QnTid7dtjeOKJC9m8uRBnnZXM//63nFq1DgZ9uwkJCRT2aGrGvAkJHA3ytr3cv5wQqfvXt29fUlNTGTx4sNehBE2kHrtjgr1/JRYsYF/9+mh0dNC2cTo5cfymTi3NK6/UICUlikaN9vLMM79TuHDOVKlq06bNIlVtdNqFVNXvG1AciANqn/D4BKBFuvtTgIanW1e1atU0p+zbp9q+vSqoxsSojhkT/G3GxcUFfyMZ2bxZtV491bS0oG7Gs/3LIZG6f61atdK6det6HUZQReqxOyao+5ecrHrjjaqHDgVvG5kI5v6lpam+8ILLBaDaq5dqSkrQNpchYKFmkmuz1CtbVfcD04AOJzy1FaiQ7n55IGRKfRQv7mYFuftuN5/z9dfDyy9HaI/tihVhwYKIvzZkjAkCEddBp2BBryMJuKQkuPVW1wdJBN56y/VBCsWOwZkmZhGJFZHivt8LAO2B1ScsNg64RZymwAFV3RHoYLMjOhreew9ee80dlCeegDvvhORkryMLgrQ0N24sMdHrSIwx4WLHDjcpTgROJbtnD1x6KXz2mfvO8d13btROqLZf/GkxlwXiRGQZ8Cvwi6qOF5FeItLLt8xEYAOwHngf6BOUaLNJxE148c03UKAAfPQRXH457N3rdWQBlj8/dO/uioobY4w/ypZ1PWYj7HNj7Vpo2hRmzoRy5dzPq67yOqrT86dX9jJVra+qdVS1tqo+73t8mKoO8/2uqnqvqp6vqhfqCZ2+Qs0117iDU7as6x7frJmbBCOidOrkuh0mJXkdiTEm1P3xhyv4EGE1sadPd0l5/XqoX99d5WvQwOuoMhdZX42yoGFD1z2+bl33japJE5esI8qYMbBpk9dRGGNCXXQ0VKiQ+XJh5OOP3enrfftcLYsZM+Ccc7yOyj+5NjGDex/OnAmdO7vT2e3bw8iRXkcVQEOGuDJouXKyamOMX/76y82Xe+21XkcSEGlprpro7be7Upv/+Y+rWBxOI+hydWIGKFLETRn54IOuI1iPHq6cZ8T02P7Pf+Crr7yOwhgTqn7+Gd71rFhjQB054opLvfSSmxTrnXfgzTfDb4KsEOwonvPy5IG334aqVeGBB+CFF9w1iQ8/dLVTw9rTT7sJq40xJiM9engdQUD89ZebvGj+fNfgGjPGde4NR7m+xZzevfe66SKLFIEvv4S2bWHXLq+jyqaSJd35+u++8zoSY0yo6d8fJkzwOops+/13109o/nxXymHOnPBNymCJ+SQdO8Ls2e7gHqulunKl11FlU+HCUKyY11EYY0LNQw+5WRzC2M8/u13YvNlNHT1/PtSu7XVU2WOJOQMXXugO7kUXHT/7SNhq2BAuucQNiTDGGIBRo9x1vDC+1DVsmBsZGh/vKjpOmwZlyngdVfZZYj6FMmXcQe7aFQ4cgA4d4P33vY4qG+bPh+ef9zoKY0yoWLMmdEtfZSI11TX2e/d2vz/+uOvjWqCA15EFhiXm0yhYEEaPhscecwe/Z094+OEwrVjXvDl88onXURhjQsG+fW74SenSXkeSZQkJbmTXW2+54dcffuh6YUdSwbII2pXgiIpyE1588IErdv76664VfeiQ15Gdgb/+ciPubVyzMblXUpIrh5WQ4HUkWbZtG7RsCePGuTPwP//sxitHGkvMfrrjDvcmKF7cdXBu1Qq2h8z8WX4qXRreeCP8BvUZYwInf35YsSK8Km4Aixe7zl2LF8P557vOua1bex1VcFhizoI2bWDePPemWLTI9dheutTrqLJAxPVse/ddq6FtTG60YYOb/zY62utIsuSHH1z/1e3b3c9586B6da+jCh5LzFlUvbp7U7RoAVu3uku3YTUMUMRdX9q/3+tIjDE5rUwZNylxmFB1xZ+6dHGXD2++GX75BUqV8jqy4LLEfAZKlXLDp26+2b1ZrroKBg0KozKeTzzhui8ePux1JMaYnLJ+Paxa5VoVYeDoUVf06T//cZ+tzz8Pn37qzsRHOkvMZyh/fvcmee4510v7wQfh/vvdmyks9OsHU6Z4HYUxJqds2AC//eZ1FH6Jj4crrnBX3fLnhy++cNWFw3R0V5ZlWitbRCoAnwJlgDRguKoOPGGZ1sD3wEbfQ98em7c5kom4EQdVqriegUOHuhoeo0Z5HZkf3nsvssYXGGNO7dAhuOwyr6Pwy86d+Wne3PVPK1XKTTIU5sXJssyfT+ajQD9VrQk0Be4VkVoZLDdTVev5bhGflNPr3h2mTnVvop9+ctedd+4M8fMtUVGue/krr3gdiTEm2B58EL791usoMjV/PvTp05AVK6BGDXc/tyVl8CMxq+oOVf3N9/tBYBUQJtNN55zmzd2bqEYN902vT5+GLFjgdVSZaNwYunXzOgpjTLC9+y5ceaXXUZzW11+74U/79uWjXTs3HOq887yO6l+9e8M55+TM6XTRLPRYEpHKwAygtqrGp3u8NfANsBXYDvRX1d8zeH1PoCdAbGxsw9GjR2cj9NCUkJCXZ565gN9+O4t8+VJ54onVtGoVulNU5T1wgOLLl7M7ix1CEhISKBxm4yCzIlL3r2/fvqSmpjJ48GCvQwmaSD12x2Rp/1SpMngwf3bvTnKIdmVWhS++qMiIES4LX3bZnzz88Eby5g2t3rRLlxajQoXDdO3anLi4aWe8njZt2ixS1UanXUhV/boBhYFFwLUZPFcUKOz7vROwLrP1VatWTSNVcrJq587b1L3lVF95RTUtzeuoTmH7dtXHHsvyy+Li4gIfSwiJ1P1r1aqV1q1b1+swgipSj90xWdq/tDTVsWNVU1KCFU62JCWp3n67+5wUUR0wQHXq1DiPozo9yO7rWaiZ5Ee/ev+ISDSuRfy5qp50oUJV41U1wff7RCBaRELz61kOiI6Gfv3WMmCAO+3x2GNw112QnOx1ZBkoW9bVHD1wwOtIjDGBpOoG/Xbp4uoJh5i9e92cyR995EZvfvONm4vg2KniTZvc75s2+be+rC4fyjJNzCIiwAfAKlV98xTLlPEth4g09q13TyADDTci7k32zTfuTffhh26Gqn37vI4sA0lJ7nrzwYNeR2KMCZRdu2DkyJAssLB+PTRr9u80jTNmwDXXeB1V6PCnxdwc6AG0FZElvlsnEeklIr18y1wHrBCRpcAgoJuvyZ7rXXONe9OVKQNxce7NuH6911GdIH9+WL4cihTxOhJjTCCkpkKJEq7YQogNi5w505UzXrsW6tSBBQug0emvuOY6/vTKnqWqoqp19N/hUBNVdZiqDvMtM0RVL1DVuqraVFXnBD/08NGokXvz1anjpkBt2hRmzfI6qhPky+ea+IsWeR2JMSa7JkxwNbFDzMiR0L69O43dqZP7HKxQwf/XjxrlzkZmdIuPz/z14SK0vkpFsAoV3JuwUyfYswfatXNv0pBy/fVQtarXURhjsuvKK2HgwMyXyyGq8Mwz0KOH62tz//2ucEhWT9K1bOmGUR27TZjgTgx06gRFiwYn9mPuugvKl3e/ly/v7gdL6PUIiGBFirg3Y79+rrZ2jx6wbh08+2yIlJpr3NjV0l250jXrjTHhZ9gwN9tOmzZeRwJAYqKbNvfLL91Z9bffdon5TJQt627gWsjt2kHt2jBmDPz9d8BCztCIEcFdf3qWmHNY3rzui2zVqq4Yz/PPu+T84YcQE+N1dMDmze48kyVmY8JT/fpu7vUQsGsXXH01zJnjpn8eNcq1brPr0CG3nqgoGD8eChbM/DUHDsCOHZkvV6NG9uPLLkvMHrnvPlfV5v/+z32T3LzZVciMjfU4sA4d3M89e6BkSW9jMcZkzdSproZlCHzLX7UKOneGjRvdpbzx410/m+xKTHQjwBISXIdaf0+Hjxnj32X3UOi2bNeYPdSpE8ye7d60c+a4noqrVnkdFa7b+BVXhMY71BjjH1X4+OOQGPY4ZYobgbJxIzRs6MoVByIpp6RA166wfbsbon3WWf6/9q67jpV8Ov0tI6fqcJb+FkiWmD1Wp4570150kXsTN2sWArMxVqnixjSExIVvY4xfkpLc8CiPT7uNGOFOvB044IaLTp/+73Xh7EhNhRtvdCNbJk/O2d0804R+piwxh4CyZd1A+65d3Zu5Qwd4/32Pg4qKcv9VO3d6HIgxJlObNrmZdDw8y5WWBo8+6k4XHz0KjzziJqYoVCgw6+/dGyZOhBdfhD//hHnz3G358sCs/3S2bHEdzWrWhAsucPsWzD+1JeYQUbAgjB7t3thHj0LPnu7gp6V5FFBUlKslGqKF740x6VSu7L7de3SW6/BhN9pywADXwXX4cHj11cDVNlGFr76CI0fchHjNmv17e/LJwGzjdPLmdfuzahUsXuzOcgZzFk3r/BVCoqLc9MhVq0KvXvDaa+5y72efBe5bZ5Y0aeLOq5cv74ZfGGNCz9ixbiTFnXd6svkdO+Cqq2DhQihWzJUhbtcusNvwuoBI+mFa+fK5S5BbtgRve9ZiDkF33gmTJkHx4u5/rlUr19nBE9u3ux7axpjQ1KCBZzUtly1z398XLoRzz3VFPwKdlEPNnj1uBM3llwdvG5aYQ1Tbtv9OFL5okXvzL13qQSA9ergxzXat2ZjQ8/PP7nRa3bo5vumJE91l7S1b3Ait+fPdNdhIlpQE110HffsGd18tMYewGjXcm715c9i6FVq0cCXoctykSTlzIccYkzUzZ8L+/Tm+2SFDXNXPhATXU3rKFM87gwddaircdJOr39KvX3C3ZYk5xJUq5YYGdO/u/gmuugoGD87hIEKim7gx5jh798ILL7jhjTkkNRUeeMCV1ExLg//+Fz7/PDj1TCpXdp2+KlcOzvJZdc89rpjJG28EZ/3pWWIOAzExbsKLZ591/wzH/jGOHs2hAERcuZ0WLVwtPGOMt/buhUsucRU3csjBg67i1uDBrgPUZ5/Bc8/ljnIHs2fDBx+4a+n160O9em6+g2DJtFe2iFQAPgXKAGnAcFUdeMIyAgwEOgGHgdtU9bfAh5t7ibjZWapUcQXhhwyBP/5wQwiCPasK4MZzvf++R93DjTHHKVECliyB6Ogc2dyWLa4Y4LJlrlLv2LHue0FukdNDxP1pMR8F+qlqTaApcK+I1DphmY5AVd+tJ/BuQKM0/7jpJlcOt1Qp+PFH14j9888c2njNmvDJJ7B6dQ5t0BhzolKzZrnZb3IoKS9c6CaeW7YMqlVzRT1yU1L2QqaJWVV3HGv9qupBYBVwzgmLdQE+VWceUFxEAlCEzWSkeXP3z1Gjhqt607gx/PprDm28cOEc2pAxJiP7GjRws9/kgLFj3RzIO3dC69ZupEgOXtLOtbJ0jVlEKgP1gfknPHUOkH649VZOTt4mgM4/30180bYt/PWXG+sczEo0/+jaFSpWpGCONdONMf/4/HOi9+8PesEfVVfgqGtXV23r9tvd4IwSJYK6WePjd+UvESkMfAP0VdUTa7BkdPn/pDPyItITd6qb2NhYpk2b5n+kYSYhISFH9u/xx4WYmGpMnFiWrl2hZ88/6NZtS1A7ZJRYsIASCxYwrWLF4G3EYzl1/HLa/v37SU1Njch9OyZSjx1AuUWLSLjwwqDu39GjwttvV2XChHIA3H33Bm688U/mzAnaJo8TycfPb6qa6Q2IBiYBD53i+feAG9PdXwOUPd06q1WrppEsLi4ux7aVlqY6YMC/85zceadqUlJwtxkXFxf8jXgoJ49fTmrVqpXWrVvX6zCCKlKPnS5frqrB3b99+1TbtXOfIzExqqNHB21TpxSxx88HWKiZ5NxMT2X7elx/AKxS1TdPsdg44BZxmgIHVHVHdr80GP+IwMMPuxq1BQq4bv0dO8K+fcHbZlRSkqs2FAJzvxoT8XbvduMkgzhGcsOGf6edPftsNyfG9dcHbXPmNPy5xtwc6AG0FZElvlsnEeklIr18y0wENgDrgfeBPsEJ15zOtde6uU/LlHE9t5s1c0OqgiEtf353kbtIkeBswBjjpKa6MUpTp7ppjoJgzhxXeXf1ajet4fz5rgyw8UamR1lVZ5HxNeT0yyhwb6CCMmfuoovcP9UVV7ge202auILrLVoEYWNnnQVvveX+ky+7LAgbMMbw8cfuG/ZLLwVl9V9+6Tp3JSW5iRlGjXKzRBnvWOWvCFSxIsya5U5n79njZnv5/PMgbaxlSzcHmjEmOG67Dfr3D/hqVV1Fz+7dXVLu3RvGj7ekHAosMUeookVh3Di47z5IToabb3YlPQNevaZhQ3eRe9y4AK/YGMN//gObNgV8nFJSEtx6q6t1LeJOfA0dGrQz5SaLLDFHsLx5XV3bQYMgKsrVtb35Zlf2OqCOHHFlgYwxgXXZZVC+fEBXuXs3XHqpq3VdqBB8/72bxjA31LwOF5aYc4H773cN2sKF4YsvoH172LUrgBuoXBmeesp168zJgrLGRKpDh9w/a8eOkD9/wFa7Zo3r5DVzJpQr535eeWXAVm8CxBJzLtG5s7vuXL68mynlWA/MgFGFO++EzZsDuFJjcqlduwI+pGLatH9HatSvDwsWuJ8m9FhizkXq1nX/jA0busZt06ZuzGJAiLjhHJUru+Edxpgzs327G/P49NMBW+VHH7mz4vv2uTndZ8yAc6xocsiyxJzLlC3rxjpfcw0cOAAdOsCIEQFauYhb2TPPBGiFxuRCw4e7+VwDIC0NnnjCTRWbkgIPPeRq6ttcNKHN+uDlQoUKwddfw+OPw4ABcPfdsG4dvPyy6ySWLTfcYF07jTlTqakBGz5x5IjreT1mDOTJ4+Zw79Ur89cZ71mLOZeKioJXX4X333d5dMAAV37v8OFsrrhoUffV/NhXdGOMfw4dgnr13D9hNrtI//UXtGnjknLRojBxoiXlcGKJOZe76y746SdXVODbb930kTuyW+W8aFF3rjxPnoDEaEyuUKiQ6/RRsGC2VrNihav4N38+VKrkOntaYb7wYonZ0K6dmwD9vPNg4UL3T710aTZWKOLGYEye7M6RG2NO79tv3bXl0qWztZpJk6B5czc44lhyrl07QDGaHGOJ2QBQsybMmwcXXwxbtrja2hMnZnOlO3e6mqDGmNNr3NgNk8iGd991wyLj491lqbg4N0uUCT+WmM0/YmPdmbTu3SEhwTV6hwzJxgpvucV94CxfHrAYjYk4H37oTmOfYc351FTX27pPH/f7E0+4Tt0FCgQ4TpNjLDGb48TEwMiRbsRTWpqrGpataWA3bXLjMa0imDEnU3Xnnc+wP0ZCgpvu9a23IDrajVd+8cUAjK4wnrLDZ04i4kZsjBwJ+fK5ettdusDBg2ewsvPOc/NOHjliydmY9A4edDUyn3vOdZjMom3b3ORu48a5GVh//tlNRGXCX6aJWUQ+FJG/RWTFKZ5vLSIHRGSJ7/bfwIdpvHDTTe7UdsmS7npzixbw559nuLJbbnE1AY0xzqJFbrziGVi82F0lWrwYqlRx/UNatw5seMY7/lSC+BgYAnx6mmVmquoVAYnIhJQWLVzPzs6d3QRSTZrADz+cwYo++cRdRzPGwP79LpOeQTadPbskL73khjtfconr0F2qVKADNF7KtMWsqjOAvTkQiwlR55/vhlO1aeM6WrdsCTNmZPGToFAh9/W+R4/gBGlMOOnSBVauzNJLVN215Kefrs3hw+5f6ZdfLClHIlE/rvuJSGVgvKqeNCJORFoD3wBbge1Af1X9/RTr6Qn0BIiNjW04evToM4075CUkJFA4wgrSpqQIb71VjR9/LAtAz55/0K3bFr+LFMnRoxTYto3DlSoFMcrAiMTjB9C3b19SU1MZPHiw16EETcgfO1UkJQXNl8/vl6SmCoMGVWHcODfzxB13bOTmmzdH5BzKIX/8sqlNmzaLVLXRaRdS1UxvQGVgxSmeKwoU9v3eCVjnzzqrVaumkSwuLs7rEIIiLU31lVdU3fd31bvuUk1OzuJKnntOdd26oMQXKJF6/Fq1aqV169b1OoygCuljN2OG6i23ZOkl+/erXnaZ+3/Ln1/16ad/D1JwoSGkj18AAAs1k/yY7dkGVDU+3e8TReQdESmlqruzu24TekTg0UchMXEFr7xSmxEj3BSSX3/teob6pWFDVwPUmNzm4ouhQgW/F9+0Ca64An7/3dUZ+P57SEr6G6gVtBCN97I9XEpEyoi4Eyoi0ti3Tiv3FOFatdrN9OmustDUqe7zxu953Tt3doOkv/02qDEaE1Ieesj9k1Su7Nfi8+e7zpa//+4q882fD82aBTdEExr8GS71JTAXqC4iW0XkThHpJSLH5iq5DlghIkuBQUA3X3PdRLjGjWHBArjwQli92lUUnD3bzxcnJsKqVUGNz5iQcvnlULGiX4uOGeM6bP/9N7RvD3PmwLnnBjc8EzoyPZWtqjdm8vwQ3HAqkwtVrAizZkG3bvDjj9C2ras+1L17Ji+sVAmefNLNllG9uis5Zkwk2rgRZs50Y/kzoQqvvOLKaoKbK33oUFfVy+QeVvnLZFvRoq760L33QnKyK0zy3HN+FvoaNsxqaZvIlpzsV8nN5GQ3jfkTT7i+HK+/Du+9Z0k5N8p25y9jAPLmdRNeVK8Offu6kp7r1sGIEZk0ht991/3cuRPKlMmBSI3JQZMnuyo91aufdrG9e13N6+nT3XTMn38OV1+dMyGa0GMtZhNQ99/vWs+FC7sPl/btYdeuTF60Zo07F25dE0wkUYXRo2H36QeorFvnOnVNnw5ly8KMGZaUcztLzCbgOnd2153Ll3edwZo2dZ3DTql6dVeU++hR11vbmHB34IA7CzR8uPtHOIWZM93/x9q1ULeu63ndsGEOxmlCkiVmExR167oe2w0bunHOzZq5YVWnlCePOwf+zTc5FaIxwTN5spuW7TQ++wzatXOnsTt3dkk6C0OcTQSzxGyCpmxZd3ru6qtdzf7LL4cPPjjNC158Ea67zk5pm/C2dy907erezxlQhf/+13XSTklx851//z0UKZLDcZqQZYnZBFWhQq4R/PDD7kz1XXfBY4+d4ox18eKwb58bwJmcnMORGhMAR464WV7i48mokHViohu18MILEBXlGtUDB/rVadvkIpaYTdBFRcGAAe5yW9688OqrcMMNbtq6k5Qo4caIZKHAvzEh4ehRNwTht9/cGMIT7Nrlxvl/+aXrHDl+PNx3nwdxmpBnidnkmLvvdkVIihVzrejWrWHHjgwWrFHDFd8eOjSnQzTmzD3/vBsfmMGXylWrXHnNuXPddeTZs6FjRw9iNGHBxjGbHNW+vftw6twZfv3VfVhNmODKeh7noovcBThjwoGqm90lg/4Rkye7rhMHDri39fffu/4XxpyKtZhNjqtZE+bNcz21t2yB5s1dS/o4lSrB+ee7D7tMxoEa46l166BTJ1cZ5IR5hN9/Hzp0cEm5a1eYNs2SssmcJWbjidKl3fCpbt3g4EE3td1JZ65FoH59yJ/fkxiN8UuVKjBo0HGdvdLS4JFHoGdPSE113y9Hj3a525jMWGI2nomJgS++cENH0tJcR5gHH3QfZP/o1s01N957z7M4jTml225zleuqVv3noUOH3Knr115znR1HjHATU0TZp63xk71VjKdE3IQXn33m+swMGgRdurhW9D/y53c9Xo0JNX36uBazz/bt0KoVjB3rRv9NmgR33uldeCY8WWI2IeHmm10nmZIlXWewFi3c9WcAYmPd1FXTprlZ443x2vjxrincuLFrFuNmMG3SBBYtgvPOc50c27b1OE4TljJNzCLyoYj8LSIrTvG8iMggEVkvIstEpEHgwzS5wSWXuE5h1arBsmXuM2/hwnQL7NxpHcFMaLjwQmjw70fdsS+TW7e6zozz5rlRf8acCX9azB8DHU7zfEegqu/WE3g3+2GZ3KpKFdfSaNPG5eGWLd1pQcBdb27Z0g2CPu5CtDE5ZM8eeOghNxjZl5gHD4arroKEBOje3Z35iY31OE4T1jJNzKo6A9h7mkW6AJ+qMw8oLiI2IMCcsRIl4Kef4PbbXYXDrl1dRxpVXEKeMsUV3zYmpxUs6M5XR0Vx9Kib5vSBB1znxWefhZEjM5l/3Bg/iPoxYYCIVAbGq2rtDJ4bD7yiqrN896cAj6rqwgyW7YlrVRMbG9tw9OjR2Ys+hCUkJFD4hDGNkSQn9k8VvvyyIu+/fx4AnTtvp2/fdeTNq0QlJlL099/ZH6Q58iL1+PXt25fU1FQGZzLzUTgL1rGr8OWX/N2uHUmlS3P4cB6ef74W8+eXJDo6jYcfXs2ll/4d8G1mJFLfm8dE+v61adNmkao2Ou1CqprpDagMrDjFcxOAFunuTwEaZrbOatWqaSSLi4vzOoSgysn9GzNGNSZGFVTbtVPdt09V161Tvf/+oG0zUo9fq1attG7dul6HEVRBOXZpaaoffaR64IBu3qx64YXu/ViypOrMmYHf3OlE6nvzmEjfP2ChZpIfA9EreyuQfhbR8sD2AKzXGMCNCZ0+Hc4+253FbtYMNkT5ijps3OgqLxkTLJMmuTfebbexcG1RmjSB5cuhenWYP991+jImkAKRmMcBt/h6ZzcFDqhqRlMTGHPGGjd2H4K1a8Pq1e4y35w5wIwZrgusMcFSsCAULMi337q+hzt3us6Jc+e6qrHGBJo/w6W+BOYC1UVkq4jcKSK9RKSXb5GJwAZgPfA+0Cdo0ZpcrVIlNytPhw5u1FTbtvBlvluhRw+YNctNdmtMoGzZAgMHoi0uYcCsi+na1XVGvOMO1znxrLO8DtBEqkxnl1LVGzN5XoF7AxaRMadRtCj88IMr3fnOO254yrp18PRfXyElSkCtWl6HaCJF3rwcLVaS3j1dLRFwpTUfeeS4stjGBJxV/jJhJ29eGDIE3n7bfUA+8wzcEj+EpPNqwsSJGU69Z4zfUlLg+efZp8XpMPJmRoxwQ6C+/tpNRmFJ2QSbJWYTlkRcq/n776FQITd+9Iq2h0n8ZBQcPux1eCbM7U49ixZt8zFliut0OH26G09vTE6wxGzC2pVXusvL55wDk+cWovaiT1i7Rl1PWmOy6oUXWDRuGzXfuZ+Va/Jw4YWu02Hjxl4HZnITS8wm7NWrBwsWuAqJf/wBN7beweaPp3odlglDs3ZXp2P3s9i923UynDXLdTo0JidZYjYRoVw5N3KqSxf47WBVqnz9Kt++tBri4rwOzYQBfeddRt7yM5cMuoFdycXo08d1Mixa1OvITG5kidlEjEKF3PwW/fu76ZvffHI3o9/cSlqa15GZUJaUBM/80IgnPqtBVBQMHOg6F+bNdMyKMcFhidlElDx53IQX770H8/K04P/G9+CN5t+SOPNXr0MzISh+5Dg+rDGAF366iL2FKvL9925SCut5bbxkidlEpJ494ccfoVgxmDYvP3f2jmbnTq+jMqFkzWql/dPNGLTpSs45x11PvuIKr6Myxo8CI15IS0tj9+7d7N+/n9QwnXe3WLFirFq1yuswgubE/cuTJw/FixenVKlSREWFxve9Sy91ZTs7d+7Mpt+hQY3nufLj66h2tRUhye0WDZnLHw8N5deUkTRoEMuUH1w/BWNCQUgm5q1btyIiVK5cmejoaCQMzysdPHiQIkWKeB1G0KTfP1UlJSWFv/76i61bt1KxYkWPo/tXrVpuuMvVV8PkuRcxsEdZho9xPW5N7vTJ8CR69m1MudQydOkCn3/u+icYEypCo2lzgkOHDnHOOeeQL1++sEzKuY2IkC9fPs455xwOHTrkdTgnKV0apk6F4t06siWhOIc6Xsdnz23wOiyTw9LS4NU+m6l5zyWkpArX9T+Xb76xpGxCT0i2mIGQOR1q/BfKxywmxrWMqlYV/vfCkyx9tjKL9xzltbfykieP19GZYDtyBB667k+GTaxEqaifGPZuFD17eh2VMRkL3U9SYwIsKgqefx4e+rQ+BfIepcfgi7ip414OHvQ6MhNMO3fCVc33cNvE6ylZJJkvfiphSdmENEvMJtfp0QN+nJKPG4v/xKhfSnBdky1s2eJ1VCYYVixXHq4zicmLS3BjpbnMmJePSy/1OipjTs8Ss8mVWraE8b+eTZ0qh3lx1TW0veggixZ5HZUJpEmToP3Fh+mw61PaXHSIeQuibFZQExb8Sswi0kFE1ojIehF5LIPnW4vIARFZ4rv9N/ChGhNYVapA3PyCPNJqAev/KswrF4/ju7E2ZWQkeG9QEgs6PsOBhCh++L/PmTC9MKVLex2VMf7JNDGLSB5gKNARqAXcKCIZfe+cqar1fLfnAxxn2GjdujX33Xef5+vIzL59+zj77LP5448/Ml32uuuu48033wxqPF4pUQJ++jmKXjclcHnyOLpfm8iHj6y2KZ3DVGpyKt++DL0fjGa7luHRx6L44gsoUMDryIzxnz8t5sbAelXdoKrJwFdAl+CGZYLtpZdeolOnTpx//vmZLvvMM8/wv//9jwMHDuRAZDkvXz5457Mi7HppBJfxE7e9Vov9d35BSmJ4FrfJrRK2x7O4wlW88fOlNMqzhGaf9ObZl/MTwoMFjMmQP8OlzgHSd43ZCjTJYLlmIrIU2A70V9XfT1xARHoCPQFiY2OZNm1ahhssVqwYB8O0q2xqairJycmkpqae8T4cW0eg/wbJycnky5ePw4cPM2LECEaNGuXXNipXrkzlypUZMWIEPX3dWU+1f4mJiac8rqGuWTO44Iql/DkeGm18n1/P3kD8e/2IKRM5za1j1fTC9RidyoEle6j9yOOcnbKOtRTm4T6Lia0YT4TtJgAJCQkRd/zSi/T984uqnvYGXA+MSHe/BzD4hGWKAoV9v3cC1mW23mrVqumprFy58pTPhbpWrVpp7969tV+/flqyZEmNjY3Vfv36aWpq6j/P33vvvce95tZbb9XOnTsft4577rlHH3jgAS1evLgWL15c+/fv/886VFXT0tL01Vdf1fPOO09jYmK0du3a+tlnn50US69evbRfv35aqlQpbdSokaqqjhkzRkuUKKFpaWn/LPvqq68qcNLt6aefVlXV5557Tps3b/7P8vHx8Rnufzgfu2NWDp2qe6SEKujG6Kr65+Q1XocUMK1atdK6det6HUZA/fpanO6RkqqgG6Kr6djXx3gdUlDFxcV5HUJQRfr+AQs1k/zoz0merUCFdPfL41rF6ZN7vKom+H6fCESLSKkz/K6QIRFvbmfi888/J0+ePMyZM4chQ4bw9ttvM2rUqCyvIy0tjblz5/Lee+8xfPhw3n777X+ef+qpp/jggw8YOnQoK1eu5PHHH+eee+5hwoQJx61n5MiRqCozZ87k008/BWDmzJk0bNjwuKpqvXv3ZseOHf/c+vXrR5kyZbjlllsAaNy4MQsWLODIkSNn9kcJI8tK/s0rt9zGhnw1qJyyjrPaN2T+8z95HZY5gaYp0694jXoPt6eE7mFhbAemvfUYvx1Y5nVoxmSLP4n5V6CqiJwrIvmAbsC49AuISBnxfcqLSGPfevcEOthwUatWLZ566imqVavGDTfcQJs2bZgyZUqW1lG2bFkGDRpEjRo1uOGGG3j44Yf/6YB16NAh3nzzTUaMGEGHDh0499xz6d69O3fffTdDhw49bj3nnnsub7zxBjVq1KBmzZoAbN68mbJlyx63XJEiRShTpgxlypThk08+4csvv2TatGlUqVIFgHLlypGSksL27cd9J4tI7777Lj8vmULJPxYw/+yrKEwCFz3TiamXPENqsl13DgUJ2+P5tfL1tJrwCHlJZXaTh2iwbTyfjPmEcePGZb4CY0JYpolZVY8C9wGTgFXAaFX9XUR6iUgv32LXASt815gHAd18TfaAUfXmdibq1Klz3P1y5crx999/Z2kdTZs2Pa5F26xZM7Zt20Z8fDwrV64kMTGRDh06ULhw4X9u77777km9rBs2bHjSuo8cOUJMTEyG23355ZcZNGgQcXFxVK9e/Z/HC/i6teaGFvMxxcoX4aKtY5nV/lkA2s56ntWxl7B7+Q5vA8vlVo9exr5K9Wi85RsOUJR5D39D83lvEBVttVVNZPCrVrbv9PTEEx4blu73IcCQwIYWvqKjo4+7LyKkpaUBrp70id9ZUlJSsrT+Y+v64YcfTprJ6cRtF8qgQn+pUqXYt2/fSY+/+OKLDBs2jOnTp//TUj5m7969gOu0l5tE5Y2ixS/PsOSNFpz78HVcED+X3fXq8tvLX9DgkfZeh5eraJoy/abhNPmqLwVIZFO+ahz9bjxNO1b1OjRjAsoGEuSw2NhYduw4vsW1dOnSk5abP3/+cQl83rx5lCtXjqJFi1KrVi3y58/P5s2bqVKlynG3SpUqZRpD/fr1Wbly5XGPvfDCC7z33nvHnb5Ob8WKFZQrV46zzz7b312NKPX6tePIb6tYVKwtpdJ2Ue/Ry5hy0WMkHkjyOrRcYd/aXSwofw2tv+pFARKZXf12zt6xhCqWlE0EssScw9q2bcuPP/7IuHHjWLNmDQ899BBbMijUvH37dvr27cuaNWv4+uuvee211/jPf/4DuOvB/fv3p3///nz44YesX7+eJUuWMGzYMIYPH55pDJdffjmrVq1izx7XDeDFF19k4MCBfPXVVxQqVIidO3eyc+dOEhMT/3nNzJkz6ZDLJzEuU68Mdf/6mRktn0IR2i18lV2xtVg7arHXoUW0+c/+iNSoTpMd33OAosy/fyTNV39IgRKRM4zNmPRCdtrHSHXHHXewbNky7rjjDgD69OnDNddcw+7du49b7qabbiI1NZUmTZogItx5553/JGZwLdyzzz6b119/nd69e1O0aFHq1avHI488kmkMF154IY0bN+arr76iT58+DBgwgPj4eJo3b37ccpMnT6Zdu3YkJiYyduxYJk2aFIC/QOj7+uuvmT17dobP5c2fh5bTX2DlB50occ/1VEjZQEq3xswa/gRNfniK6ILRGb7OZN3+Tfv5vc19NN/0OQCrCzWk0E/f0KTFqc8Kne7YGRM2MhtPFaxbpI5jPuZU43xDxY8//qjVqlXTo0ePZrrskCFD9NJLLz3usUgex6zq31jKhL8SdOoF9/3TV3BFgYa6/JNFwQ8uG8JlHPOCx7/VHVFlVUFTyKszrnhVjyZl/l5VjfxxsLZ/4Y0AjWM2EahDhw7ce++9bN26NdNlo6OjGTx4cA5EFRo+/vhjfvop83HLhUoXos2KwSx6dTLb81TggiOLqHVrI2bUe4CD2+JzINLIs2XeNuaUu46LXr6WMmk7WFbkYrZOWMolPzxCnnyZ97r299gZE8osMediDzzwgF+dxXr27Hnc0KlIl9UP94aPtKP4tt+ZXf8+FKHl0sEkVazCrPu+Ii3VZsPwR+L+RKa3fZbYZudz8Y5vOExB4q5+mwv2zKRyJ//narTEbCKBJWZjAqDg2UVo/ttg1n+1iOWFmlAqbRctht7I78Wbs2z4XK/DC1mapsx/bCy7YmvSKu45YkhiQflriZ+/ijZjHyRPtH1EmdzH3vXGBFD1/6tHrX1zmH3rcHZFlebChLnUuedi5lb8PzZPzXyKzdxk8cAZLCt2CU1evZYKRzexLv8FLH59Co23fEOZxhUzX4ExESpkE7PahLhhx46Zkyc6iuYf302BreuZ2exhjhBDsy2jqdCuKjOq3cmmqRu8DtFTqz5byMJSHajftxV1E2azV0oy49q3qLxvCfX7tfU6PGM8F5KJOTo6OleVfowUR44cOanyWG5WuGwRLpkzgH3z1zHv3G7u+vO6D6nQripzz+3OxklrvQ4xx2ia8tsbcfxWoj01b7mIRnsmcYhCzGz9NPm2bqDlN32JLmCjN42BEE3MpUuXZtu2bRw+fNhaYWFAVTl8+DDbtm2jdOnSXoeTbRMnTuSVV14J2PrKNS5P0w1fsnXyGmZVvQ1BabbpSyp1qMGykm1Y+OoUNC0y3+dHj6Qw5z+j2VjoAhr0b0uDfVNIIh/TG/Ujae1mLol7nsLligZse4E+dsZ4ISS/ohYt6v5Rt2/fnuU60qEiMTHxlBNFRIIT9y86Opqzzz77n2MXzgoWLBiUY1epXRUqrf2ILdOeYmOvV2i65hPq7J0Gj01jy9PnsuHqftR7rTvFKp0V8G3ntD9nbuaPx4bTYO5QLtYDAMRLURa37U/d9/rQ6vySQdlusI6dMTkpJBMzuOQczh/y06ZNo379+l6HETSRvH/vvPMOa9eupXXr1kFZf4XW51Nh9fvsXvUSKx4YTq2pQ6iQspEKY+4jacxDLCp7Kck976PhI+3JVzBk/0VPcnBbPCte/J5CX46g9v6ZVMSdBdgUXZUt1z5Io6G306pkwaDGEOxjZ0xOCJ//emNyyOjRo9m/f3/Qt1OqZiytf3mSlEMPM++Jb8j72Uc02DeZhjsmwHMTSHiuMAurdiXP9ddS+z+XUqhU6NWG3r9hL8te/5m8Y8dw0c5xNOMoACnkZV7lGyj00D3UufcSKkdJJmsKjJw6dsYEk1+JWUQ6AAOBPMAIVX3lhOfF93wn4DBwm6r+FuBYjYlI0YXy0XTgjTDwRrbP+5P1z35G5biPqZi8novXfQIvfcKRl2KYH9uO5IvbUPrGdlS5to4nY3wTDySxcuRvHBj1E6WW/EKtg/NpiZuGNA1hRZFm7O7Yg3ovXs/FVUrleHzGRIJME7OI5AGGApcCW4FfRWScqqafN7AjUNV3awK86/tpjMmCck0rUu6nJ0GfYM3YlWwfOpZz53xO5cTVNNk1Ab6fAN/DfoqztVgt9lzQigLNG1Dy0gZUvKQS0TGZl630V/KBI2z4aS07p65Ef11I+bVTqXTodxrwb7+PZKJZV6Qhey67kSpP3EDtBucEbPvG5Fb+tJgbA+tVdQOAiHwFdAHSJ+YuwKe+At3zRKS4iJRV1R0nr84YkykRql97AdWvvQB4it1LtrJ6WBx5fxxPxW1zKZe6heIH5sCcOTAHeM0lyZ15SrO1ZB0Ol6hAamwZosqUJl/ZkkSXPgvJF03CtgOkJCayfPhcUvYfInnfIZL3H0b37kO2/kneHVuI3r+LivEriE3dSQ2UGieE9kf+WvxdtTl5rupMtV5tqVGhiAd/IGMilz+J+Rwg/YTBWzm5NZzRMucAp0zMW7ZsiegOGvv376d48eJehxE0kbx/S5Ys4ejRo6H5/jwXOPc8Ug6W48iuBNL2HyT68H7ypR4hmhRI3QZ/b4O/gdUnv3y97+f991yc6aYUSJICHM0bQ2qhokQVL0pM6aK+8cZrYfZamP1W4PYtAEL62AVIJP/vQeTvnz/8ScwZ9do4cdClP8sgIj2Bnr67SdOnT1/hx/bDVSlgd6ZLha+I37/p06dH6v6Vmu7vsdMjkHIE9u+D/cCmYIYVMJF87CAX/O8R2fuX6YxA/iTmrUCFdPfLA9vPYBlUdTgwHEBEFqpqIz+2H5Zs/8JbJO9fJO8b2P6Fu9ywf5kt40+3zl+BqiJyrojkA7oB405YZhxwizhNgQN2fdkYY4zJukxbzKp6VETuAybhhkt9qKq/i0gv3/PDgIm4oVLrccOlbg9eyMYYY0zk8mscs6pOxCXf9I8NS/e7AvdmcdvDs7h8uLH9C2+RvH+RvG9g+xfucv3+iU0SYYwxxoSOkJxdyhhjjMmtQiIxi0h/EVERiagafiLygogsE5ElIvKziJTzOqZAEpHXRGS1bx/Hikhxr2MKFBG5XkR+F5E0EYmYHqIi0kFE1ojIehF5zOt4AklEPhSRv0UkIodhikgFEYkTkVW+9+aDXscUKCISIyILRGSpb9+e8zqmYBCRPCKyWETGn245zxOziFTAlfv80+tYguA1Va2jqvWA8cB/PY4n0H4BaqtqHWAt8LjH8QTSCuBaYIbXgQRKuvK6HYFawI0iUsvbqALqY6CD10EE0VGgn6rWBJoC90bQ8UsC2qpqXaAe0ME3wifSPAisymwhzxMz8BbwCBkUJAl3qhqf7m4hImwfVfVnVT3quzsPN349IqjqKlVd43UcAfZPeV1VTQaOldeNCKo6A9jrdRzBoqo7jk0OpKoHcR/wEVGcXJ0E391o3y2iPi9FpDzQGRiR2bKeJmYRuQrYpqpLvYwjmETkRRHZAtxE5LWY07sD+NHrIMxpnap0rgkzIlIZqA/M9ziUgPGd5l2CKyj7i6pGzL75vI1rhKZltmDQ52MWkclAmQyeehJ4Args2DEE0+n2T1W/V9UngSdF5HHgPuCZHA0wmzLbP98yT+JOs32ek7Fllz/7FmH8Kp1rQpuIFAa+AfqecFYurKlqKlDP11dlrIjUVtWI6C8gIlcAf6vqIhFpndnyQU/Mqto+o8dF5EJcSf6lbjpnygO/iUhjVd0Z7LgC5VT7l4EvgAmEWWLObP9E5FbgCqCdhtnYuywcu0jhV+lcE7pEJBqXlD9X1W+9jicYVHW/iEzD9ReIiMQMNAeuEpFOQAxQVERGqurNGS3s2alsVV2uqqVVtbKqVsZ9aDQIp6ScGRGpmu7uVWQ430/4EpEOwKPAVap62Ot4TKb8Ka9rQpS4FswHwCpVfdPreAJJRGKPjeoQkQJAeyLo81JVH1fV8r5c1w2YeqqkDKHR+SuSvSIiK0RkGe6UfcQMb/AZAhQBfvENCRuW2QvChYhcIyJbgWbABBGZ5HVM2eXrqHesvO4qYLSq/u5tVIEjIl8Cc4HqIrJVRO70OqYAaw70ANr6/t+W+FpgkaAsEOf7rPwVd435tEOKIplV/jLGGGNCiLWYjTHGmBBiidkYY4wJIZaYjTHGmBBiidkYY4wJIZaYjTHGmBBiidkYY4wJIZaYjTHGmBBiidmYXEREpqYrTpEoItd7HZMx5nhWYMSYXEhEegNtgBt9kwcYY0JE0CexMMaEFhG5BegIdLWkbEzoscRsTC7iO3V9E9BFVVO8jscYczJLzMbkEr45YfsAV6hqotfxGGMyZteYjcklRGQPsBc45HtosKp+4GFIxpgMWGI2xhhjQogNlzLGGGNCiCVmY4wxJoRYYjbGGGNCiCVmY4wxJoRYYjbGGGNCiCVmY4wxJoRYYjbGGGNCiCVmY4wxJoT8P8n4sjAWCCY7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# extra code – shows what the Huber loss looks like\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(8, 3.5))\n",
    "z = np.linspace(-4, 4, 200)\n",
    "z_center = np.linspace(-1, 1, 200)\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z ** 2 / 2, \"r:\", linewidth=1)\n",
    "plt.plot(z_center, z_center ** 2 / 2, \"r\", linewidth=2)\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"k--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"k--\")\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.text(2.1, 3.5, r\"$\\frac{1}{2}z^2$\", color=\"r\", fontsize=15)\n",
    "plt.text(3.0, 2.2, r\"$|z| - \\frac{1}{2}$\", color=\"b\", fontsize=15)\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To test our custom loss function, let's create a basic Keras model and train it on the California housing dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# extra code – loads, splits and scales the California housing dataset, then\n",
    "#              creates a simple Keras model\n",
    "\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 969us/step - loss: 0.3970 - mae: 0.7423 - val_loss: 0.3721 - val_mae: 0.6864\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 702us/step - loss: 0.2330 - mae: 0.5302 - val_loss: 0.2730 - val_mae: 0.5552\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8470e12a90>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving/Loading Models with Custom Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_loss/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_model_with_a_custom_loss\")  # extra code – saving works fine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss\",\n",
    "                                   custom_objects={\"huber_fn\": huber_fn})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 981us/step - loss: 0.1904 - mae: 0.4699 - val_loss: 0.2363 - val_mae: 0.5045\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 743us/step - loss: 0.1773 - mae: 0.4514 - val_loss: 0.2182 - val_mae: 0.4884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f84a7329c70>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold ** 2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 996us/step - loss: 0.1950 - mae: 0.4469 - val_loss: 0.2734 - val_mae: 0.4741\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 744us/step - loss: 0.1909 - mae: 0.4434 - val_loss: 0.2507 - val_mae: 0.4685\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8430545700>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_loss_threshold_2/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_threshold_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2\",\n",
    "                                   custom_objects={\"huber_fn\": create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 996us/step - loss: 0.1880 - mae: 0.4395 - val_loss: 0.2452 - val_mae: 0.4571\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 755us/step - loss: 0.1858 - mae: 0.4374 - val_loss: 0.2243 - val_mae: 0.4526\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8480b07250>"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(tf.keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra code – creates another basic Keras model\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 985us/step - loss: 0.4997 - mae: 0.7514 - val_loss: 0.5202 - val_mae: 0.6936\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 753us/step - loss: 0.2781 - mae: 0.5435 - val_loss: 0.3794 - val_mae: 0.5651\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f84408d09d0>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_loss_class/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class\")  # extra code – saving works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_class\",\n",
    "                                   custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 981us/step - loss: 0.2206 - mae: 0.4783 - val_loss: 0.3241 - val_mae: 0.5093\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 760us/step - loss: 0.2018 - mae: 0.4574 - val_loss: 0.2909 - val_mae: 0.4934\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8470937610>"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows that loading worked fine, the model can be used normally\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.loss.threshold  # extra code – the treshold was loaded correctly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Custom Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z):\n",
    "    return tf.math.log(1.0 + tf.exp(z))\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights):  # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = tf.keras.layers.Dense(1, activation=my_softplus,\n",
    "                              kernel_initializer=my_glorot_initializer,\n",
    "                              kernel_regularizer=my_l1_regularizer,\n",
    "                              kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.1668 - mae: 0.7430 - val_loss: inf - val_mae: inf\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 750us/step - loss: 0.7359 - mae: 0.5977 - val_loss: 2.6252 - val_mae: 0.5870\n",
      "INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5646 - mae: 0.5293 - val_loss: 0.9063 - val_mae: 0.5070\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 759us/step - loss: 0.4981 - mae: 0.4975 - val_loss: 0.7695 - val_mae: 0.4918\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f83d1b07550>"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – show that building, training, saving, loading, and training again\n",
    "#              works fine with a model containing many custom parts\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1, activation=my_softplus,\n",
    "                          kernel_initializer=my_glorot_initializer,\n",
    "                          kernel_regularizer=my_l1_regularizer,\n",
    "                          kernel_constraint=my_positive_weights)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.save(\"my_model_with_many_custom_parts\")\n",
    "model = tf.keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    }\n",
    ")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyL1Regularizer(tf.keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 992us/step - loss: 1.1668 - mae: 0.7430 - val_loss: inf - val_mae: inf\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 752us/step - loss: 0.7359 - mae: 0.5977 - val_loss: 2.6252 - val_mae: 0.5870\n",
      "INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n",
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 980us/step - loss: 0.5646 - mae: 0.5293 - val_loss: 0.9063 - val_mae: 0.5070\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 783us/step - loss: 0.4981 - mae: 0.4975 - val_loss: 0.7695 - val_mae: 0.4918\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f84a6b929d0>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – again, show that everything works fine, this time using our\n",
    "#              custom regularizer class\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1, activation=my_softplus,\n",
    "                          kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                          kernel_constraint=my_positive_weights,\n",
    "                          kernel_initializer=my_glorot_initializer),\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.save(\"my_model_with_many_custom_parts\")\n",
    "model = tf.keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    }\n",
    ")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra code – once again, lets' create a basic Keras model\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 596us/step - loss: 1.3734 - huber_fn: 0.5275\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 556us/step - loss: 0.7705 - huber_fn: 0.3166\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8470edd910>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – train the model with our custom metric\n",
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: if you use the same function as the loss and a metric, you may be surprised to see slightly different results. This is in part because the operations are not computed exactly in the same order, so there might be tiny floating point errors. More importantly, if you use sample weights or class weights, then the equations are a bit different:\n",
    "* the `fit()` method keeps track of the mean of all batch losses seen so far since the start of the epoch. Each batch loss is the sum of the weighted instance losses divided by the _batch size_ (not the sum of weights, so the batch loss is _not_ the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Streaming metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = tf.keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a streaming metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(tf.keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs)  # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        sample_metrics = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(sample_metrics))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Extra material** – the rest of this section tests the `HuberMetric` class and shows another implementation subclassing `tf.keras.metrics.Mean`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that the `HuberMetric` class works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\",\n",
    "              metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 587us/step - loss: 0.4997 - huber_metric_10: 0.4997\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 574us/step - loss: 0.2781 - huber_metric_10: 0.2781\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f83d0f59280>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_metric/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_model_with_a_custom_metric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\n",
    "    \"my_model_with_a_custom_metric\",\n",
    "    custom_objects={\n",
    "        \"huber_fn\": create_huber(2.0),\n",
    "        \"HuberMetric\": HuberMetric\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 625us/step - loss: 0.2206 - huber_metric_10: 0.2206\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 609us/step - loss: 0.2018 - huber_metric_10: 0.2018\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f845023daf0>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`model.metrics` contains the model's loss followed by the model's metric(s), so the `HuberMetric` is `model.metrics[-1]`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like it works fine! More simply, we could have created the class like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(tf.keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class handles shapes better, and it also supports sample weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=tf.keras.losses.Huber(2.0), optimizer=\"nadam\",\n",
    "              weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 645us/step - loss: 0.2505 - HuberMetric: 0.5049\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 635us/step - loss: 0.1416 - HuberMetric: 0.2854\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2,\n",
    "                    sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.2505398094654083, 0.2505398573110885)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(history.history[\"loss\"][0],\n",
    " history.history[\"HuberMetric\"][0] * sample_weight.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_metric_v2/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"my_model_with_a_custom_metric_v2\",\n",
    "                                   custom_objects={\"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 665us/step - loss: 0.2257 - HuberMetric: 0.2257\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 628us/step - loss: 0.2034 - HuberMetric: 0.2034\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8460c2fa00>"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "exponential_layer = tf.keras.layers.Lambda(lambda x: tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – like all layers, it can be used as a function:\n",
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an exponential layer at the output of a regression model can be useful if the values to predict are positive and with very different scales (e.g., 0.001, 10., 10000)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 0s 845us/step - loss: 1.0631 - val_loss: 0.4457\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 591us/step - loss: 0.4562 - val_loss: 0.3798\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 585us/step - loss: 0.4029 - val_loss: 0.3548\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 597us/step - loss: 0.3851 - val_loss: 0.3464\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 582us/step - loss: 0.3708 - val_loss: 0.3449\n",
      "162/162 [==============================] - 0s 427us/step - loss: 0.3586\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3586341440677643"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(1),\n",
    "    exponential_layer\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
    "model.fit(X_train_scaled, y_train, epochs=5,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, it's often preferable to replace the targets with the logarithm of the targets (and use no activation function in the output layer)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(tf.keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = tf.keras.activations.get(activation)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\", shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"he_normal\")\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\", shape=[self.units], initializer=\"zeros\")\n",
    "        super().build(batch_input_shape)  # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"units\": self.units,\n",
    "                \"activation\": tf.keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 836us/step - loss: 2.8036 - val_loss: 2.9430\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 671us/step - loss: 0.7903 - val_loss: 1.3091\n",
      "162/162 [==============================] - 0s 426us/step - loss: 0.6557\n",
      "INFO:tensorflow:Assets written to: my_model_with_a_custom_layer/assets\n"
     ]
    }
   ],
   "source": [
    "# extra code – shows that a custom layer can be used normally\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)\n",
    "model.save(\"my_model_with_a_custom_layer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 892us/step - loss: 0.5665 - val_loss: 0.4506\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 692us/step - loss: 0.4502 - val_loss: 0.5153\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f848168da30>"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows how to load a model with a custom layer\n",
    "model = tf.keras.models.load_model(\"my_model_with_a_custom_layer\",\n",
    "                                   custom_objects={\"MyDense\": MyDense})\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMultiLayer(tf.keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        print(\"X1.shape: \", X1.shape ,\" X2.shape: \", X2.shape)  # extra code\n",
    "        return X1 + X2, X1 * X2, X1 / X2\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape1, batch_input_shape1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our custom layer can be called using the functional API like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X1.shape:  (None, 2)  X2.shape:  (None, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'my_multi_layer_4')>,\n",
       " <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'my_multi_layer_4')>,\n",
       " <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'my_multi_layer_4')>)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – tests MyMultiLayer with symbolic inputs\n",
    "inputs1 = tf.keras.layers.Input(shape=[2])\n",
    "inputs2 = tf.keras.layers.Input(shape=[2])\n",
    "MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the `call()` method receives symbolic inputs, and it returns symbolic outputs. The shapes are only partially specified at this stage: we don't know the batch size, which is why the first dimension is `None`.\n",
    "\n",
    "We can also pass actual data to the custom layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X1.shape:  (2, 2)  X2.shape:  (2, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       " array([[ 9., 18.],\n",
       "        [ 6., 10.]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       " array([[18., 72.],\n",
       "        [ 8., 21.]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       " array([[0.5      , 0.5      ],\n",
       "        [0.5      , 2.3333333]], dtype=float32)>)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – tests MyMultiLayer with actual data \n",
    "X1, X2 = np.array([[3., 6.], [2., 7.]]), np.array([[6., 12.], [4., 3.]]) \n",
    "MyMultiLayer()((X1, X2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's create a layer with a different behavior during training and testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyGaussianNoise(tf.keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's a simple model that uses this custom layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 867us/step - loss: 2.1976 - val_loss: 26.5902\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 671us/step - loss: 1.4224 - val_loss: 19.3606\n",
      "162/162 [==============================] - 0s 423us/step - loss: 1.0180\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0180009603500366"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – tests MyGaussianNoise\n",
    "tf.random.set_seed(42)\n",
    "model = tf.keras.Sequential([\n",
    "    MyGaussianNoise(stddev=1.0, input_shape=input_shape),\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\",\n",
    "                          kernel_initializer=\"he_normal\"),\n",
    "    tf.keras.layers.Dense(1)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualBlock(tf.keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [tf.keras.layers.Dense(n_neurons, activation=\"relu\",\n",
    "                                             kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(n_layers)]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualRegressor(tf.keras.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = tf.keras.layers.Dense(30, activation=\"relu\",\n",
    "                                             kernel_initializer=\"he_normal\")\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = tf.keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 810us/step - loss: 5.2455\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 807us/step - loss: 0.8515\n",
      "162/162 [==============================] - 0s 512us/step - loss: 0.6072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as dense_19_layer_call_and_return_conditional_losses, dense_19_layer_call_fn, dense_20_layer_call_and_return_conditional_losses, dense_20_layer_call_fn, dense_21_layer_call_and_return_conditional_losses while saving (showing 5 of 20). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_custom_model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_custom_model/assets\n"
     ]
    }
   ],
   "source": [
    "# extra code – shows that the model can be used normally\n",
    "tf.random.set_seed(42)\n",
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "model.save(\"my_custom_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 879us/step - loss: 0.7176\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 816us/step - loss: 0.5186\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.62953055],\n",
       "       [1.2767944 ],\n",
       "       [4.634055  ]], dtype=float32)"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – the model can be loaded and you can continue training or use it\n",
    "#              to make predictions\n",
    "model = tf.keras.models.load_model(\"my_custom_model\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "model.predict(X_test_scaled[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could have defined the model using the sequential API instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "block1 = ResidualBlock(2, 30)\n",
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\",\n",
    "                          kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    tf.keras.layers.Dense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: due to an issue introduced in TF 2.2 ([#46858](https://github.com/tensorflow/tensorflow/issues/46858)), `super().build()` fails. We can work around this issue by setting `self.built = True` instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReconstructingRegressor(tf.keras.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [tf.keras.layers.Dense(30, activation=\"relu\",\n",
    "                                             kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = tf.keras.layers.Dense(output_dim)\n",
    "        self.reconstruction_mean = tf.keras.metrics.Mean(\n",
    "            name=\"reconstruction_error\")\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = tf.keras.layers.Dense(n_inputs)\n",
    "        self.built = True  # WORKAROUND for super().build(batch_input_shape)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss)\n",
    "        if training:\n",
    "            result = self.reconstruction_mean(recon_loss)\n",
    "            self.add_metric(result)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 820us/step - loss: 0.7640 - reconstruction_error: 1.2728\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 809us/step - loss: 0.4584 - reconstruction_error: 0.6340\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 786us/step - loss: 0.4211 - reconstruction_error: 0.4342\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 745us/step - loss: 0.3753 - reconstruction_error: 0.3597\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 772us/step - loss: 0.3618 - reconstruction_error: 0.2908\n"
     ]
    }
   ],
   "source": [
    "# extra code\n",
    "tf.random.set_seed(42)\n",
    "model = ReconstructingRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing Gradients Using Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2 = 5, 3\n",
    "eps = 1e-6\n",
    "(f(w1 + eps, w2) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A non-persistent GradientTape can only be used to compute one set of gradients (or jacobians)\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)  # returns tensor 36.0\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)  # raises a RuntimeError!\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)  # returns tensor 36.0\n",
    "dz_dw2 = tape.gradient(z, w2)  # returns tensor 10.0, works fine now!\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – if given a vector, tape.gradient() will compute the gradient of\n",
    "#              the vector's sum.\n",
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows that we get the same result as the previous cell\n",
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "    z = z1 + z2 + z3\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra code – shows how to compute the jacobians and the hessians\n",
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "hessians = [hessian_tape.gradient(jacobian, [w1, w2])\n",
    "            for jacobian in jacobians]\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)  # same result as without stop_gradient()\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=inf>]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable(1e-50)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = tf.sqrt(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=30.0>"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.log(tf.exp(tf.constant(30., dtype=tf.float32)) + 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1.], dtype=float32)>]"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1.0e30])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z):\n",
    "    return tf.math.log(1 + tf.exp(-tf.abs(z))) + tf.maximum(0., z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is the proof that this equation is equal to log(1 + exp(_z_)):\n",
    "* softplus(_z_) = log(1 + exp(_z_))\n",
    "* softplus(_z_) = log(1 + exp(_z_)) - log(exp(_z_)) + log(exp(_z_)) ; **just adding and subtracting the same value**\n",
    "* softplus(_z_) = log\\[(1 + exp(_z_)) / exp(_z_)\\] + log(exp(_z_)) ; **since log(_a_) - log(_b_) = log(_a_ / _b_)**\n",
    "* softplus(_z_) = log\\[(1 + exp(_z_)) / exp(_z_)\\] + _z_ ; **since log(exp(_z_)) = _z_**\n",
    "* softplus(_z_) = log\\[1 / exp(_z_) + exp(_z_) / exp(_z_)\\] + _z_ ; **since (1 + _a_) / _b_ = 1 / _b_ + _a_ / _b_**\n",
    "* softplus(_z_) = log\\[exp(–_z_) + 1\\] + _z_ ; **since 1 / exp(_z_) = exp(–z), and exp(_z_) / exp(_z_) = 1**\n",
    "* softplus(_z_) = softplus(–_z_) + _z_ ; **we recognize the definition at the top, but with –_z_**\n",
    "* softplus(_z_) = softplus(–|_z_|) + max(0, _z_) ; **if you consider both cases, _z_ < 0 or _z_ ≥ 0, you will see that this works**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_softplus(z):\n",
    "    def my_softplus_gradients(grads):  # grads = backprop'ed from upper layers\n",
    "        return grads * (1 - 1 / (1 + tf.exp(z)))  # stable grads of softplus\n",
    "\n",
    "    result = tf.math.log(1 + tf.exp(-tf.abs(z))) + tf.maximum(0., z)\n",
    "    return result, my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1.], dtype=float32)>])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extra code – shows that the function is now stable, as well as its gradients\n",
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Training Loops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)  # extra code – to ensure reproducibility\n",
    "l2_reg = tf.keras.regularizers.l2(0.05)\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
    "                          kernel_regularizer=l2_reg),\n",
    "    tf.keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(step, total, loss, metrics=None):\n",
    "    metrics = \" - \".join([f\"{m.name}: {m.result():.4f}\"\n",
    "                          for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if step < total else \"\\n\"\n",
    "    print(f\"\\r{step}/{total} - \" + metrics, end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n",
    "loss_fn = tf.keras.losses.mean_squared_error\n",
    "mean_loss = tf.keras.metrics.Mean()\n",
    "metrics = [tf.keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "362/362 - mean: 0.6219 - mean_absolute_error: 0.4975\n",
      "Epoch 2/5\n",
      "362/362 - mean: 0.6272 - mean_absolute_error: 0.5049\n",
      "Epoch 3/5\n",
      "362/362 - mean: 0.6019 - mean_absolute_error: 0.4951\n",
      "Epoch 4/5\n",
      "362/362 - mean: 0.6088 - mean_absolute_error: 0.4971\n",
      "Epoch 5/5\n",
      "362/362 - mean: 0.6159 - mean_absolute_error: 0.5032\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(f\"Epoch {epoch}/{n_epochs}\")\n",
    "    for step in range(1, n_steps + 1):\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch, training=True)\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "\n",
    "        # extra code – if your model has variable constraints\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "\n",
    "        mean_loss(loss)\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "\n",
    "        print_status_bar(step, n_steps, mean_loss, metrics)\n",
    "\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0425ac6b66024c7d83d98459be6f1811",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fac07a118bd649158e28c73591809f95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "162f6cd2e9b4491d9a3b1bc378990eef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "116af880df174758bf744dc1fe5fa81e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20afe56af0b54d12be3dd84e29e9f0d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "662afff06af24ded83a719eabc1a8a83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# extra code – shows how to use the tqdm package to display nice progress bars\n",
    "\n",
    "from tqdm.notebook import trange\n",
    "from collections import OrderedDict\n",
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=f\"Epoch {epoch}/{n_epochs}\") as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "                with tf.GradientTape() as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "\n",
    "                gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))\n",
    "\n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "\n",
    "                steps.set_postfix(status)\n",
    "\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x7f83d1c22a90>"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube = tf.function(cube)\n",
    "tf_cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:** the rest of the code in this section is in appendix D."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF Functions and Concrete Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ConcreteFunction tf_cube(x) at 0x7F843148D910>"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function = tf_cube.get_concrete_function(tf.constant(2.0))\n",
    "concrete_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function is tf_cube.get_concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring Function Definitions and Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7f8441219040>"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'x:0' shape=() dtype=float32>,\n",
       " <tf.Tensor 'pow/y:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op = ops[2]\n",
    "list(pow_op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'pow:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Identity:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_tensor_by_name('Identity:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"__inference_tf_cube_3515915\"\n",
       "input_arg {\n",
       "  name: \"x\"\n",
       "  type: DT_FLOAT\n",
       "}\n",
       "output_arg {\n",
       "  name: \"identity\"\n",
       "  type: DT_FLOAT\n",
       "}"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.function_def.signature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How TF Functions Trace Python Functions to Extract Their Computation Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    print(f\"x = {x}\")\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x = Tensor(\"x:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x = 2\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x = 3\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x = Tensor(\"x:0\", shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant([[1., 2.]]))  # New shape: trace!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x = Tensor(\"x:0\", shape=(2, 2), dtype=float32)\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7f84312f51f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7f84312f51f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant([[3., 4.], [5., 6.]]))  # New shape: trace!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = tf_cube(tf.constant([[7., 8.], [9., 10.]]))  # Same shape: no trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to specify a particular input signature:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function(input_signature=[tf.TensorSpec([None, 28, 28], tf.float32)])\n",
    "def shrink(images):\n",
    "    print(\"Tracing\", images)  # extra code to show when tracing happens\n",
    "    return images[:, ::2, ::2] # drop half the rows and columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing Tensor(\"images:0\", shape=(None, 28, 28), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "img_batch_1 = tf.random.uniform(shape=[100, 28, 28])\n",
    "img_batch_2 = tf.random.uniform(shape=[50, 28, 28])\n",
    "preprocessed_images = shrink(img_batch_1)  # Works fine, traces the function\n",
    "preprocessed_images = shrink(img_batch_2)  # Works fine, same concrete function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python inputs incompatible with input_signature:\n",
      "  inputs: (\n",
      "    tf.Tensor(\n",
      "[[[0.7413678  0.62854624]\n",
      "  [0.01738465 0.3431449 ]]\n",
      "\n",
      " [[0.51063764 0.3777541 ]\n",
      "  [0.07321596 0.02137029]]], shape=(2, 2, 2), dtype=float32))\n",
      "  input_signature: (\n",
      "    TensorSpec(shape=(None, 28, 28), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "img_batch_3 = tf.random.uniform(shape=[2, 2, 2])\n",
    "try:\n",
    "    preprocessed_images = shrink(img_batch_3)  # ValueError! Incompatible inputs\n",
    "except ValueError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Autograph To Capture Control Flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"static\" `for` loop using `range()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in range(10):\n",
    "        x += 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'add/y' type=Const>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'add_1/y' type=Const>,\n",
       " <tf.Operation 'add_1' type=AddV2>,\n",
       " <tf.Operation 'add_2/y' type=Const>,\n",
       " <tf.Operation 'add_2' type=AddV2>,\n",
       " <tf.Operation 'add_3/y' type=Const>,\n",
       " <tf.Operation 'add_3' type=AddV2>,\n",
       " <tf.Operation 'add_4/y' type=Const>,\n",
       " <tf.Operation 'add_4' type=AddV2>,\n",
       " <tf.Operation 'add_5/y' type=Const>,\n",
       " <tf.Operation 'add_5' type=AddV2>,\n",
       " <tf.Operation 'add_6/y' type=Const>,\n",
       " <tf.Operation 'add_6' type=AddV2>,\n",
       " <tf.Operation 'add_7/y' type=Const>,\n",
       " <tf.Operation 'add_7' type=AddV2>,\n",
       " <tf.Operation 'add_8/y' type=Const>,\n",
       " <tf.Operation 'add_8' type=AddV2>,\n",
       " <tf.Operation 'add_9/y' type=Const>,\n",
       " <tf.Operation 'add_9' type=AddV2>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" loop using `tf.while_loop()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra code – shows how to use tf.while_loop (usually @tf.function is simpler)\n",
    "@tf.function\n",
    "def add_10(x):\n",
    "    condition = lambda i, x: tf.less(i, 10)\n",
    "    body = lambda i, x: (tf.add(i, 1), tf.add(x, 1))\n",
    "    final_i, final_x = tf.while_loop(condition, body, [tf.constant(0), x])\n",
    "    return final_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" `for` loop using `tf.range()` (captured by autograph):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x = x + 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'range/start' type=Const>,\n",
       " <tf.Operation 'range/limit' type=Const>,\n",
       " <tf.Operation 'range/delta' type=Const>,\n",
       " <tf.Operation 'range' type=Range>,\n",
       " <tf.Operation 'sub' type=Sub>,\n",
       " <tf.Operation 'floordiv' type=FloorDiv>,\n",
       " <tf.Operation 'mod' type=FloorMod>,\n",
       " <tf.Operation 'zeros_like' type=Const>,\n",
       " <tf.Operation 'NotEqual' type=NotEqual>,\n",
       " <tf.Operation 'Cast' type=Cast>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'zeros_like_1' type=Const>,\n",
       " <tf.Operation 'Maximum' type=Maximum>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(0)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling Variables and Other Resources in TF Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(counter, c=1):\n",
    "    return counter.assign_add(c)\n",
    "\n",
    "increment(counter)  # counter is now equal to 1\n",
    "increment(counter)  # counter is now equal to 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"counter\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function(counter).function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment()\n",
    "increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"assignaddvariableop_resource\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function().function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Counter:\n",
    "    def __init__(self):\n",
    "        self.counter = tf.Variable(0)\n",
    "\n",
    "    @tf.function\n",
    "    def increment(self, c=1):\n",
    "        return self.counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = Counter()\n",
    "c.increment()\n",
    "c.increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def tf__add(x):\n",
      "    with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
      "        do_return = False\n",
      "        retval_ = ag__.UndefinedReturnValue()\n",
      "\n",
      "        def get_state():\n",
      "            return (x,)\n",
      "\n",
      "        def set_state(vars_):\n",
      "            nonlocal x\n",
      "            (x,) = vars_\n",
      "\n",
      "        def loop_body(itr):\n",
      "            nonlocal x\n",
      "            i = itr\n",
      "            x = ag__.ld(x)\n",
      "            x += 1\n",
      "        i = ag__.Undefined('i')\n",
      "        ag__.for_stmt(ag__.converted_call(ag__.ld(tf).range, (10,), None, fscope), None, loop_body, get_state, set_state, ('x',), {'iterate_names': 'i'})\n",
      "        try:\n",
      "            do_return = True\n",
      "            retval_ = ag__.ld(x)\n",
      "        except:\n",
      "            do_return = False\n",
      "            raise\n",
      "        return fscope.ret(retval_, do_return)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x += 1\n",
    "    return x\n",
    "\n",
    "print(tf.autograph.to_code(add_10.python_function))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra code – shows how to display the autograph code with syntax highlighting\n",
    "def display_tf_code(func):\n",
    "    from IPython.display import display, Markdown\n",
    "    if hasattr(func, \"python_function\"):\n",
    "        func = func.python_function\n",
    "    code = tf.autograph.to_code(func)\n",
    "    display(Markdown(f'```python\\n{code}\\n```'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__add(x):\n",
       "    with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
       "        do_return = False\n",
       "        retval_ = ag__.UndefinedReturnValue()\n",
       "\n",
       "        def get_state():\n",
       "            return (x,)\n",
       "\n",
       "        def set_state(vars_):\n",
       "            nonlocal x\n",
       "            (x,) = vars_\n",
       "\n",
       "        def loop_body(itr):\n",
       "            nonlocal x\n",
       "            i = itr\n",
       "            x = ag__.ld(x)\n",
       "            x += 1\n",
       "        i = ag__.Undefined('i')\n",
       "        ag__.for_stmt(ag__.converted_call(ag__.ld(tf).range, (10,), None, fscope), None, loop_body, get_state, set_state, ('x',), {'iterate_names': 'i'})\n",
       "        try:\n",
       "            do_return = True\n",
       "            retval_ = ag__.ld(x)\n",
       "        except:\n",
       "            do_return = False\n",
       "            raise\n",
       "        return fscope.ret(retval_, do_return)\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(add_10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using TF Functions with tf.keras (or Not)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, tf.keras will automatically convert your custom code into TF Functions, no need to use\n",
    "`tf.function()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom loss function\n",
    "def my_mse(y_true, y_pred):\n",
    "    print(\"Tracing loss my_mse()\")\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom metric function\n",
    "def my_mae(y_true, y_pred):\n",
    "    print(\"Tracing metric my_mae()\")\n",
    "    return tf.reduce_mean(tf.abs(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom layer\n",
    "class MyDense(tf.keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = tf.keras.activations.get(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[1], self.units),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.biases = self.add_weight(name='bias', \n",
    "                                      shape=(self.units,),\n",
    "                                      initializer='zeros',\n",
    "                                      trainable=True)\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, X):\n",
    "        print(\"Tracing MyDense.call()\")\n",
    "        return self.activation(X @ self.kernel + self.biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom model\n",
    "class MyModel(tf.keras.Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = MyDense(30, activation=\"relu\")\n",
    "        self.hidden2 = MyDense(30, activation=\"relu\")\n",
    "        self.output_ = MyDense(1)\n",
    "\n",
    "    def call(self, input):\n",
    "        print(\"Tracing MyModel.call()\")\n",
    "        hidden1 = self.hidden1(input)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = tf.keras.layers.concatenate([input, hidden2])\n",
    "        output = self.output_(concat)\n",
    "        return output\n",
    "\n",
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "296/363 [=======================>......] - ETA: 0s - loss: 1.5172 - my_mae: 0.8562Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.3255 - my_mae: 0.7900 - val_loss: 0.5569 - val_my_mae: 0.4819\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 792us/step - loss: 0.4419 - my_mae: 0.4767 - val_loss: 0.4664 - val_my_mae: 0.4576\n",
      "162/162 [==============================] - 0s 460us/step - loss: 0.4164 - my_mae: 0.4639\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.4163525104522705, 0.4639028012752533]"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can turn this off by creating the model with `dynamic=True` (or calling `super().__init__(dynamic=True, **kwargs)` in the model's constructor):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel(dynamic=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the custom code will be called at each iteration. Let's fit, validate and evaluate with tiny datasets to avoid getting too much output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.507260322570801, 2.0566811561584473]"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, you can compile a model with `run_eagerly=True`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae], run_eagerly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.507260322570801, 2.0566811561584473]"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extra Material – Custom Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining custom optimizers is not very common, but in case you are one of the happy few who gets to write one, here is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMomentumOptimizer(tf.keras.optimizers.Optimizer):\n",
    "    def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
    "        \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n",
    "        super().__init__(name, **kwargs)\n",
    "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate)) # handle lr=learning_rate\n",
    "        self._set_hyper(\"decay\", self._initial_decay) # \n",
    "        self._set_hyper(\"momentum\", momentum)\n",
    "    \n",
    "    def _create_slots(self, var_list):\n",
    "        \"\"\"For each model variable, create the optimizer variable associated with it.\n",
    "        TensorFlow calls these optimizer variables \"slots\".\n",
    "        For momentum optimization, we need one momentum slot per model variable.\n",
    "        \"\"\"\n",
    "        for var in var_list:\n",
    "            self.add_slot(var, \"momentum\")\n",
    "\n",
    "    @tf.function\n",
    "    def _resource_apply_dense(self, grad, var):\n",
    "        \"\"\"Update the slots and perform one optimization step for one model variable\n",
    "        \"\"\"\n",
    "        var_dtype = var.dtype.base_dtype\n",
    "        lr_t = self._decayed_lr(var_dtype) # handle learning rate decay\n",
    "        momentum_var = self.get_slot(var, \"momentum\")\n",
    "        momentum_hyper = self._get_hyper(\"momentum\", var_dtype)\n",
    "        momentum_var.assign(momentum_var * momentum_hyper - (1. - momentum_hyper)* grad)\n",
    "        var.assign_add(momentum_var * lr_t)\n",
    "\n",
    "    def _resource_apply_sparse(self, grad, var):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n",
    "            \"decay\": self._serialize_hyperparameter(\"decay\"),\n",
    "            \"momentum\": self._serialize_hyperparameter(\"momentum\"),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 0s 444us/step - loss: 4.9648\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 444us/step - loss: 1.7888\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 437us/step - loss: 1.0021\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 451us/step - loss: 0.7869\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 446us/step - loss: 0.7122\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfe0a94d50>"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=[8])])\n",
    "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n",
    "model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 11."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. TensorFlow is an open-source library for numerical computation, particularly well suited and fine-tuned for large-scale Machine Learning. Its core is similar to NumPy, but it also features GPU support, support for distributed computing, computation graph analysis and optimization capabilities (with a portable graph format that allows you to train a TensorFlow model in one environment and run it in another), an optimization API based on reverse-mode autodiff, and several powerful APIs such as tf.keras, tf.data, tf.image, tf.signal, and more. Other popular Deep Learning libraries include PyTorch, MXNet, Microsoft Cognitive Toolkit, Theano, Caffe2, and Chainer.\n",
    "2. Although TensorFlow offers most of the functionalities provided by NumPy, it is not a drop-in replacement, for a few reasons. First, the names of the functions are not always the same (for example, `tf.reduce_sum()` versus `np.sum()`). Second, some functions do not behave in exactly the same way (for example, `tf.transpose()` creates a transposed copy of a tensor, while NumPy's `T` attribute creates a transposed view, without actually copying any data). Lastly, NumPy arrays are mutable, while TensorFlow tensors are not (but you can use a `tf.Variable` if you need a mutable object).\n",
    "3. Both `tf.range(10)` and `tf.constant(np.arange(10))` return a one-dimensional tensor containing the integers 0 to 9. However, the former uses 32-bit integers while the latter uses 64-bit integers. Indeed, TensorFlow defaults to 32 bits, while NumPy defaults to 64 bits.\n",
    "4. Beyond regular tensors, TensorFlow offers several other data structures, including sparse tensors, tensor arrays, ragged tensors, queues, string tensors, and sets. The last two are actually represented as regular tensors, but TensorFlow provides special functions to manipulate them (in `tf.strings` and `tf.sets`).\n",
    "5. When you want to define a custom loss function, in general you can just implement it as a regular Python function. However, if your custom loss function must support some hyperparameters (or any other state), then you should subclass the `keras.losses.Loss` class and implement the `__init__()` and `call()` methods. If you want the loss function's hyperparameters to be saved along with the model, then you must also implement the `get_config()` method.\n",
    "6. Much like custom loss functions, most metrics can be defined as regular Python functions. But if you want your custom metric to support some hyperparameters (or any other state), then you should subclass the `keras.metrics.Metric` class. Moreover, if computing the metric over a whole epoch is not equivalent to computing the mean metric over all batches in that epoch (e.g., as for the precision and recall metrics), then you should subclass the `keras.metrics.Metric` class and implement the `__init__()`, `update_state()`, and `result()` methods to keep track of a running metric during each epoch. You should also implement the `reset_states()` method unless all it needs to do is reset all variables to 0.0. If you want the state to be saved along with the model, then you should implement the `get_config()` method as well.\n",
    "7. You should distinguish the internal components of your model (i.e., layers or reusable blocks of layers) from the model itself (i.e., the object you will train). The former should subclass the `keras.layers.Layer` class, while the latter should subclass the `keras.models.Model` class.\n",
    "8. Writing your own custom training loop is fairly advanced, so you should only do it if you really need to. Keras provides several tools to customize training without having to write a custom training loop: callbacks, custom regularizers, custom constraints, custom losses, and so on. You should use these instead of writing a custom training loop whenever possible: writing a custom training loop is more error-prone, and it will be harder to reuse the custom code you write. However, in some cases writing a custom training loop is necessary⁠—for example, if you want to use different optimizers for different parts of your neural network, like in the [Wide & Deep paper](https://homl.info/widedeep). A custom training loop can also be useful when debugging, or when trying to understand exactly how training works.\n",
    "9. Custom Keras components should be convertible to TF Functions, which means they should stick to TF operations as much as possible and respect all the rules listed in Chapter 12 (in the _TF Function Rules_ section). If you absolutely need to include arbitrary Python code in a custom component, you can either wrap it in a `tf.py_function()` operation (but this will reduce performance and limit your model's portability) or set `dynamic=True` when creating the custom layer or model (or set `run_eagerly=True` when calling the model's `compile()` method).\n",
    "10. Please refer to Chapter 12 for the list of rules to respect when creating a TF Function (in the _TF Function Rules_ section).\n",
    "11. Creating a dynamic Keras model can be useful for debugging, as it will not compile any custom component to a TF Function, and you can use any Python debugger to debug your code. It can also be useful if you want to include arbitrary Python code in your model (or in your training code), including calls to external libraries. To make a model dynamic, you must set `dynamic=True` when creating it. Alternatively, you can set `run_eagerly=True` when calling the model's `compile()` method. Making a model dynamic prevents Keras from using any of TensorFlow's graph features, so it will slow down training and inference, and you will not have the possibility to export the computation graph, which will limit your model's portability."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. Implement a custom layer that performs _Layer Normalization_\n",
    "_We will use this type of layer in Chapter 15 when using Recurrent Neural Networks._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: The `build()` method should define two trainable weights *α* and *β*, both of shape `input_shape[-1:]` and data type `tf.float32`. *α* should be initialized with 1s, and *β* with 0s._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Solution: see below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: The `call()` method should compute the mean_ μ _and standard deviation_ σ _of each instance's features. For this, you can use `tf.nn.moments(inputs, axes=-1, keepdims=True)`, which returns the mean μ and the variance σ<sup>2</sup> of all instances (compute the square root of the variance to get the standard deviation). Then the function should compute and return *α*⊗(*X* - μ)/(σ + ε) + *β*, where ⊗ represents itemwise multiplication (`*`) and ε is a smoothing term (small constant to avoid division by zero, e.g., 0.001)._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LayerNormalization(tf.keras.layers.Layer):\n",
    "    def __init__(self, eps=0.001, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.eps = eps\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.alpha = self.add_weight(\n",
    "            name=\"alpha\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"ones\")\n",
    "        self.beta = self.add_weight(\n",
    "            name=\"beta\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        mean, variance = tf.nn.moments(X, axes=-1, keepdims=True)\n",
    "        return self.alpha * (X - mean) / (tf.sqrt(variance + self.eps)) + self.beta\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"eps\": self.eps}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that making _ε_ a hyperparameter (`eps`) was not compulsory. Also note that it's preferable to compute `tf.sqrt(variance + self.eps)` rather than `tf.sqrt(variance) + self.eps`. Indeed, the derivative of sqrt(z) is undefined when z=0, so training will bomb whenever the variance vector has at least one component equal to 0. Adding _ε_ within the square root guarantees that this will never happen."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c.\n",
    "_Exercise: Ensure that your custom layer produces the same (or very nearly the same) output as the `tf.keras.layers.LayerNormalization` layer._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create one instance of each class, apply them to some data (e.g., the training set), and ensure that the difference is negligeable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=5.6045884e-08>"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = X_train.astype(np.float32)\n",
    "\n",
    "custom_layer_norm = LayerNormalization()\n",
    "keras_layer_norm = tf.keras.layers.LayerNormalization()\n",
    "\n",
    "tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yep, that's close enough. To be extra sure, let's make alpha and beta completely random and compare again:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=2.2921004e-08>"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_alpha = np.random.rand(X.shape[-1])\n",
    "random_beta = np.random.rand(X.shape[-1])\n",
    "\n",
    "custom_layer_norm.set_weights([random_alpha, random_beta])\n",
    "keras_layer_norm.set_weights([random_alpha, random_beta])\n",
    "\n",
    "tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Still a negligeable difference! Our custom layer works fine."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. Train a model using a custom training loop to tackle the Fashion MNIST dataset\n",
    "_The Fashion MNIST dataset was introduced in Chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: Display the epoch, iteration, mean training loss, and mean accuracy over each epoch (updated at each iteration), as well as the validation loss and accuracy at the end of each epoch._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "X_train_full = X_train_full.astype(np.float32) / 255.\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "X_test = X_test.astype(np.float32) / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = tf.keras.optimizers.Nadam(learning_rate=0.01)\n",
    "loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = tf.keras.metrics.Mean()\n",
    "metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "901e5649b50840538874aed5bab0d4ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f29690a5ade4bd8a6d164d106ba2d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2ea6579132c48c087c7f59e6309387a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94fa1527062c4cf7a277a548bbddc855",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbcfc9a0c4b64151a18cf27080872dd3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc3831d9b98e488c837ba9644bf4b94a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=f\"Epoch {epoch}/{n_epochs}\") as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape() as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(tf.keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: Try using a different optimizer with a different learning rate for the upper layers and the lower layers._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_layers = tf.keras.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "])\n",
    "upper_layers = tf.keras.Sequential([\n",
    "    tf.keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])\n",
    "model = tf.keras.Sequential([\n",
    "    lower_layers, upper_layers\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_optimizer = tf.keras.optimizers.SGD(learning_rate=1e-4)\n",
    "upper_optimizer = tf.keras.optimizers.Nadam(learning_rate=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = tf.keras.metrics.Mean()\n",
    "metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e81cf85cf7548748ec760afcbd71aa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "475109b927d044a7bba030f234c67838",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af0a22afae4f47359f8fdfbac96e38bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5519ec96e42f4281987a84a5434a0734",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed04f31b3a7d4b3b9a0cd63b644e2ed5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfc4b1b4d40f4003ab8a3140a68ec883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=f\"Epoch {epoch}/{n_epochs}\") as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape(persistent=True) as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                for layers, optimizer in ((lower_layers, lower_optimizer),\n",
    "                                          (upper_layers, upper_optimizer)):\n",
    "                    gradients = tape.gradient(loss, layers.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, layers.trainable_variables))\n",
    "                del tape\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(tf.keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}