diff --git a/Chap15-Autoencoders/Chap15-Autoencoders.ipynb b/Chap15-Autoencoders/Chap15-Autoencoders.ipynb
index d5ab85a..8a14307 100644
--- a/Chap15-Autoencoders/Chap15-Autoencoders.ipynb
+++ b/Chap15-Autoencoders/Chap15-Autoencoders.ipynb
@@ -54,7 +54,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
+    "import os, sys\n",
     "import numpy as np\n",
     "import tensorflow as tf\n",
     "\n",
@@ -261,7 +261,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -284,7 +284,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -340,7 +340,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -352,7 +352,7 @@
       "epoch : 2, Train MSE : 0.01044\n",
       "epoch : 3, Train MSE : 0.01036\n",
       "epoch : 4, Train MSE : 0.01054\n",
-      "Wall time: 37.2 s\n"
+      "Wall time: 28.9 s\n"
      ]
     }
    ],
@@ -371,7 +371,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -431,7 +431,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -497,7 +497,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -509,7 +509,7 @@
       "epoch : 2, Train MSE : 0.01716\n",
       "epoch : 3, Train MSE : 0.01761\n",
       "epoch : 4, Train MSE : 0.01720\n",
-      "Wall time: 41.2 s\n"
+      "Wall time: 31.6 s\n"
      ]
     }
    ],
@@ -529,7 +529,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -554,6 +554,1014 @@
     "show_reconstructed_digits(inputs, outputs, './model/stacked_ae_tying.ckpt')"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3.3 한 번에 한 층씩 학습하기\n",
+    "\n",
+    "3.1과 3.2에서 처럼 한 번에 전체 오토인코더를 학습시키는 것보다 아래의 그림처럼 한 번에 오토인코더 하나를 학습하고, 이를 쌓아올려서 한 개의 stacked-오토인코더를 만드는 것이 훨씬 빠르며 이러한 방식은 아주 깊은 오토인코더일 경우에 유용하다.\n",
+    "\n",
+    "![](./images/stacked-ae04.PNG)\n",
+    "\n",
+    "\n",
+    "\n",
+    "- [단계 1]에서 첫 번째 오토인코더는 입력을 재구성하도록 학습된다.\n",
+    "- [단계 2]에서는 두 번째 오토인코더가 첫 번째 히든 레이어(`Hidden 1`)의 출력을 재구성하도록 학습된다.\n",
+    "\n",
+    "- [단계 3]에서는 단계1 ~ 2의 오토인코더를 합쳐 최종적으로 하나의 stacked-오토인코더를 구현한다.\n",
+    "\n",
+    "\n",
+    "텐서플로에서 이렇게 여러 단계의 오토인코더를 학습시키는 방법으로는 다음과 같이 두 가지 방법이 있다.\n",
+    "\n",
+    "-  각 단계마다 다른 텐서플로 그래프(graph)를 사용하는 방법\n",
+    "- 하나의 그래프에 각 단계의 학습을 수행하는 방법"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 3.3.1 여러개의 그래프에서 오토인코더 학습하기\n",
+    "\n",
+    "각 오토인코더를 다른 그래프를 사용하여 학습한다. 그런 다음 이런 오토인코더의 가중치와 편향을 복사해 초깃값으로 지정해서 적층 오토인코더를 만든다."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from functools import partial\n",
+    "\n",
+    "reset_graph()\n",
+    "\n",
+    "# Mini-batch\n",
+    "def shuffle_batch(features, labels, batch_size):\n",
+    "    rnd_idx = np.random.permutation(len(features))\n",
+    "    n_batches = len(features) // batch_size\n",
+    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
+    "        batch_x, batch_y = features[batch_idx], labels[batch_idx]\n",
+    "        yield batch_x, batch_y\n",
+    "\n",
+    "def train_autoencoder(train_x, n_neurons, n_epochs, batch_size, \n",
+    "                      learning_rate=0.01, l2_reg=0.0005, seed=42,\n",
+    "                      hidden_activation=tf.nn.elu, \n",
+    "                      output_activation=tf.nn.elu):\n",
+    "\n",
+    "    graph = tf.Graph()\n",
+    "    with graph.as_default():\n",
+    "        tf.set_random_seed(seed)\n",
+    "        \n",
+    "        n_inputs = train_x.shape[1]\n",
+    "        \n",
+    "        inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "        \n",
+    "        dense_layer = partial(\n",
+    "            tf.layers.dense,\n",
+    "            kernel_initializer=tf.keras.initializers.he_normal(), \n",
+    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
+    "        \n",
+    "        hidden = dense_layer(inputs, n_neurons, activation=hidden_activation, name='hidden')\n",
+    "        outputs = dense_layer(hidden, n_inputs, activation=output_activation, name='outputs')\n",
+    "        \n",
+    "        reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)\n",
+    "        \n",
+    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
+    "        loss = tf.add_n([reconstruction_loss] + reg_losses)\n",
+    "\n",
+    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
+    "        train_op = optimizer.minimize(loss)\n",
+    "        \n",
+    "    with tf.Session(graph=graph) as sess:\n",
+    "        tf.global_variables_initializer().run()\n",
+    "        n_batches = len(train_x) // batch_size\n",
+    "        for epoch in range(n_epochs):\n",
+    "            batch_x = None\n",
+    "            for iteration in range(n_batches):\n",
+    "                batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))\n",
+    "                sess.run(train_op, feed_dict={inputs: batch_x})\n",
+    "            loss_train = reconstruction_loss.eval(feed_dict={inputs: batch_x})\n",
+    "            print('epoch : {}, Train MSE : {:.5f}'.format(epoch, loss_train))\n",
+    "            \n",
+    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
+    "        hidden_val = hidden.eval(feed_dict={inputs: train_x})\n",
+    "        \n",
+    "    return hidden_val, params[\"hidden/kernel:0\"], params[\"hidden/bias:0\"], params[\"outputs/kernel:0\"], params[\"outputs/bias:0\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 단계 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch : 0, Train MSE : 0.01839\n",
+      "epoch : 1, Train MSE : 0.01773\n",
+      "epoch : 2, Train MSE : 0.01953\n",
+      "epoch : 3, Train MSE : 0.01933\n"
+     ]
+    }
+   ],
+   "source": [
+    "hidden_output, W1, b1, W4, b4 = train_autoencoder(train_x, n_neurons=300, n_epochs=4, batch_size=150,\n",
+    "                                                  output_activation=None)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 단계 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch : 0, Train MSE : 0.00432\n",
+      "epoch : 1, Train MSE : 0.00468\n",
+      "epoch : 2, Train MSE : 0.00470\n",
+      "epoch : 3, Train MSE : 0.00448\n"
+     ]
+    }
+   ],
+   "source": [
+    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 단계 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reset_graph()\n",
+    "\n",
+    "n_inputs = 28*28\n",
+    "\n",
+    "inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "hidden1 = tf.nn.elu(tf.matmul(inputs, W1) + b1)\n",
+    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
+    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
+    "outputs = tf.matmul(hidden3, W4) + b4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAD+CAYAAACnfntIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFypJREFUeJzt3UtsVVUXwPFd3m2hLYVCKZRCeQkYCRAUFaMx+Rio0SghERlo1ERiookkajAxUSc6UyZGjQYUHZiYKHEgBh8oIKIhggjIQ6Q8ShHaeltoS8vjm5jtWgvusS333rar/99onezbe84l4ayctc7eO+/y5csBAACPBvT0BQAAkC0kOQCAWyQ5AIBbJDkAgFskOQCAWyQ5AIBbJDkAgFskOQCAWyQ5AIBbg3J8PpZX6T3yevoCgL4mlUpxD+sliouLO3UP40kOAOAWSQ4A4BZJDgDgVq57cgCAbsrL+7cNlakdZOz3yHN4wJMcAMAtkhwAwC3KlQDggCwzXrx4UY0NHDgw7d9dunSp0+cYMODf56Kkcqktefbk5tw8yQEA3CLJAQDcIskBANyiJwcAOZbUo5JjsgcWQvLr/efPn4/x0KFD036nHUvSld6e/GxHR4cak32/rvymTOBJDgDgFkkOAOAW5UoAyLKkVUUuXLigxpJKgvJ7bClx2LBhMbblQvvZpLGkcqb8Xvs5WYa0v0GWUrtSnszE1AOe5AAAbpHkAABukeQAAG7RkwOALEvqQ9lltQYNSn9bbmtri7Hsc4UQQmFhYYxtv0z2/YYMGZJ4vqQ+mP1bSfbr7Odk389+f9JYJqYX8CQHAHCLJAcAcItyJQBkgSxD2jKcPG5paVFjskTX2tqqxurr69N+Z3t7e4yLi4vVWH5+foztlAU7hUAe21KqLEna8yeVWeWUAnmd9nvsaijpPtcVPMkBANwiyQEA3CLJAQDc6vM9uR9//FEdr169Osbjx49XY7Iu/fDDD6ux0tLSq8YA0B2yt2aX2Wpubo6x7bvV1NTE+OTJk2rs6NGjMbY9sPLy8hiPGDFCjclenoxDCKGhoUEdnzt3LsZTp05VY6NGjYrx4MGD1Zi8b86YMUON2euR5O+wPUB53N3pBDzJAQDcIskBANzKy8Qqz12Q8ZPZx+KDBw9263vkK7cLFy68pmvqjkmTJsV41apVamzixInZOGV2dyoEHEqlUmnvYfZeKkuUtiQpy5AnTpxQY7JcaUuJu3fvjnFjY6MaS5pCIKcpyFJpCCE0NTWpY/m3sjwZQggFBQVXPV8IIUyfPj3G//vf/9TYggULYixXZglB/7slTbWwiouLO3UP40kOAOAWSQ4A4BZJDgDgVp+fQvDZZ5+p4507d8Z49uzZamzPnj0x3r59uxpbv359jL/88ks1Nnny5Bj/+eefnb42+4rvuHHjYnzs2LG0fyf7cyGE8Pzzz3f6nAB6B/mKfSqVUmOyR1ZbW6vG5GdtT06Sr/qHoHcGT1oqzC7rVVZWpo6rq6tjbPt18h5qe4LHjx+PsbzXhRDCnDlzYiyncuUCT3IAALdIcgAAt/p8uXLmzJmJx9INN9wQ42XLlqmx1157LcZHjhxRY7Jcefjw4U5fm904UD7Cy+8MIYTTp0/H+Lrrruv0OQD0nKRX3OUr9vZeIEuGdjUUueGpLfvJV/rlK/sh6FVF7Kv/cgqD3XXAKikpifGuXbvU2L59+2Jsy5UjR46MsZ3CIEupduWSzk4hYMUTAAAMkhwAwC2SHADArT7fk8sUWTNO6okl9fz+i5y2cObMGTV20003xXjx4sXdPgeA3kH24ezq+nJ5rPnz56sx2T+z/Tp5n7Kv4svzFRUVqTHZAzx79qwaGzNmjDo+f/58jO00BXndlZWVamzRokUxttO35LXZqVWy72Z/r9wpnJ3BAQAwSHIAALcoV2aRfdS///77Y2zLF2+88UaMc70iAIDuka+123Ka/D8+cOBANSY3dJYluRBCaGtri3HSZqN2FwBJTkMIQZcL7bXY8qGcJmXPkTS9QbZ5bAlUrv7SlZ0GMrFLDk9yAAC3SHIAALdIcgAAt+jJZdHatWvVcV1dXYztsjtVVVW5uCQAWdKVZadkX8z25+XO2XYXbdm/szsNyP6V7IGFoHtpcopACFe+OyB3Gjh06JAak7sSzJ07V43NmjUrxrZfJ6/bThOQv9/24Lq7lJc69zV/AwAAvRRJDgDgFuXKDPvjjz9ivHLlyrSf27ZtmzouLy/P2jUB6Fn2tX1ZPrQ7FMhX+u3r/XLlElt2lGU/++q//KzdNFW2UUIIYf/+/TH+4Ycf0p6joqJCjclpEbbsKEuUtjwrf6PdIYEpBAAAJCDJAQDcIskBANyiJ5dhn3/+eYztq7JLly6NcXV1dc6uCUDuydff5VJdIeg+lF3WK4nsZ9lpArIPZ3t5sgdoX8s/cuSIOt66dWvasbvuuivG8+bNU2OjR4+OcVJvTe6yEIKeJsEUAgAAuoAkBwBwiyQHAHCLntw1sn23Tz/9NMZ2u4tXX301xnbeDAC/7PZZsmdl543JvpTt5dlel1RcXBxj26+Tc+Nqa2vV2KZNm9SxnBtn5/DdeOONMZ4yZYoak/0z22eU57f3Pjlm/46eHAAACUhyAAC3KFdeo/fee08db968OcYPPfSQGmPaANB/yDKkXem/qKgoxnaZLVmutKU9+fq93TVclkTt38m2yk8//aTGvvnmG3Usr3XZsmVqTE4bkMt42eux55f/FnZ6gyxR2qXKbMunO3iSAwC4RZIDALhFkgMAuEVProt27typjp966il1XFJSEuNXXnklJ9cEoPeRfSn7Krwcs0tZyWkD9pX6YcOGxdi+3i/ZqU3Hjh2LsZ0ycODAAXU8bdq0GD/44INqTO7+bXctl+xvsn24dLIxtYonOQCAWyQ5AIBblCs7Qb62a1+ptSsQLF++PMZMGQAQwpWv+8sypCxBhqBLew0NDWn/zq6UUlBQEOPm5mY1tmHDhhhv3LhRjdnVWJYsWRLj+fPnq7GysrIY26kPskRqx+R90k4LSJpekAk8yQEA3CLJAQDcIskBANyiJ3cVttZ99913x3j//v1qbObMmer45Zdfzt6FAei17Gvzkn2lX/aebB9K9rPsNAE5FcH20uSSWHZH719//TXGtl932223qeN77rknxrIHF0LyLuby/LYnJ3dFsO8xyO9kZ3AAALqAJAcAcIty5VXY13btCgHSunXr1HFpaWk2LglAL2dLa7LtYVfykGU5W76TUwHs37W0tMTYlgQbGxtj/P3336uxLVu2XPW6Qriy5TJ58uS0n21vb4+xvU8mbZoqx+y/k/xs0qaw3cWTHADALZIcAMAtkhwAwC16cv9IpVIxXrhwYdrPffjhh+p47ty5WbsmAH2Hff09qddke12SnG5gPyePZQ8uhBBqa2tjfPLkSTUmpynY+9stt9yijuXUBNs/k9dmlyNL6kHK77Fj2ejDSTzJAQDcIskBANyiXPmPNWvWxPjw4cNpP7do0SJ1nIkZ+QD6PnsvkOVL+0q9HJPTAkLQUwNsuVKO1dfXq7H169fHePv27WpMlivlFIGrHctdAuRKJSHo32HLjrKUmbSqSVJ5Mhv3U57kAABukeQAAG6R5AAAbvXbntzBgwfV8UsvvdQzFwLApaT+kuy12X6dXM3ffsepU6diLKcMhBBCXV1d2r+TuwnMmjVLjQ0fPjzttVmylyiX+ApB9+TszgqyD5e0k0E28CQHAHCLJAcAcKvflis3b96sjpuamtJ+Vq7SbTcqBICukuVEu6GqXEnEvm5fVFQU41GjRqmx6urqtOeTu6NUVFSoMTtNQJ7TliTlWNIKL1ZPTrXiSQ4A4BZJDgDgFkkOAOBWv+3JJbGrcm/cuDHG9OQAZJLtZSWt5i/vP+Xl5Wrs9ttvj7Htz8menN0JXO5EHoLun9klx+TUANtns9faW/AkBwBwiyQHAHArz74GmmU5PRkSsX0C0EWpVKpH72Hyfp20Q4EtJcpjO2WgryouLu7UPYwnOQCAWyQ5AIBbJDkAgFu57skBAJAzPMkBANwiyQEA3CLJAQDcIskBANwiyQEA3CLJAQDcIskBANwiyQEA3CLJAQDcIskBANwiyQEA3CLJAQDcIskBANwiyQEA3CLJAQDcIskBANwiyQEA3CLJAQDcIskBANwalOPzXc7x+ZBeXk9fANDX1NTUcA/rJaqqqjp1D+NJDgDgFkkOAOAWSQ4A4Faue3IAgF4sL+/fVtfly32/BcmTHADALZIcAMAtypUA0EtdvHhRHQ8cODDGAwboZ5RLly5dNQ5Blx0HDUq+7SeVKOU5L1y4kPb8lrxuWQ7NBZ7kAABukeQAAG6R5AAAbvXbntxHH32kjs+dOxfjHTt2qLF33nkn7fe8+OKL6vjOO++M8R133HENVwigP7A9qqTeluyXdXR0qDF5D7Nj8jtHjBihxmxvT/ba7LXJfl5LS0va6xw8eLA6Hjp0aIxlfy4XeJIDALhFkgMAuJWX4xntPTp9/sknn4zx22+/nZVzzJo1K8ZbtmxRY8XFxVk5ZzexCwHQRd3dhcBOBZDH9lX89vb2GDc2Nqqx1tbWGNfW1qqxv//+O8apVCrt39l7vi1JVlZWxnj8+PFqrKCgIMb2fiZLmbYkKUukQ4YMUWPyemzpNAm7EAAA+j2SHADALZIcAMAt11MIZA8uhM734ebOnauOlyxZEuODBw+qsffff18d7927N8affPKJGnvsscc6dX4AvtienOyRNTc3q7Fjx47F2PbkTp48GePTp0+rMdmHs30+2SOzY7Z/J49LSkrUWGFhYYztNAHZT8vPz1djsl9n/y1kT872CzMx3YAnOQCAWyQ5AIBb7sqVR48ejfG7776b9nMLFixQxxs2bIixfE02BP3Kq33UPnTokDreunVrjM+cOdOJKwbQ38iSoS1X1tfXx1jez+yYLe1NmDAhxuXl5WnPbace2HuYnIpw6tQpNTZ16tQY25VT5PUklRntLghyyoT9TfK4u7sX8CQHAHCLJAcAcIskBwBwy11PTvbBbH1X9uG++uorNTZ8+PBOff/atWvV8c8//5z2s/fdd1+nvhOAb7YPlfS6vbwXlZWVqbEZM2bEeNKkSWps9OjRV/2OEPQOBd99950aq6urU8eyJ9fW1qbGZF/s/Pnzac+ftJOClbSUFz05AAASkOQAAG65K1fOmzcvxvYVfjkVwJYIOstOS5CvvwLov5Jef7dTj+TKIbYMN23atBjbkqQsVxYVFakxOfWpqalJjcnVUewqKvY+WVpaGuOksqPcCNV+1q6GIn+//U45ncJOPejKrgTp8CQHAHCLJAcAcIskBwBwy11PTsrUTtzr1q2L8a5duxI/u3jx4hhPmTIlI+cH0PvZ/pHsNdkelexL2WUEZf9q7Nixakz28mz/Sr4fYJfjkssNyp1SQgiho6NDHctzVldXqzH5G+3vlb/J9t2SxuS/jf1Nsl9pe56dxZMcAMAtkhwAwC3X5cru+uWXX9TxE088EWM7y3/cuHHqePXq1TG2JQoACEGvgGLLcHJqgCxPhqBLffZe1NDQEOPdu3ersX379sVYrmgSQghVVVXqWLZZZs+ercZkidKeX5YWbSlT3gtteTQTJckkPMkBANwiyQEA3CLJAQDcoid3Fdu2bVPHtvYsrVixQh1Pnz49K9cEoHezr8bLXpNd1ktOL7DLY8m+lP3Os2fPxrilpUWN7d+/P8Z2l5W//vorxnZHb7l7QAghVFZWxlgu8WXPL5dJDEH3Ge11y99k31WQPbqkpdG6iyc5AIBbJDkAgFuUK//x6KOPxvjjjz9O+7lnnnlGHT/33HNZuyYAfYfdTSDplfqklfeTdi+QDh8+rI5/++23GJ84cUKNyZLgmDFj1Nj111+vjmW50pLfY8ussq1jN4mVkkqSSf+G3cWTHADALZIcAMAtkhwAwK1+25OTr8KGEMIXX3wR47a2NjUmV+V+4YUX1Jh9jRYArKSV95Nek29tbVXHqVQqxrYnJ3cesDuwyCW/hg8frsZsb+3cuXMxtj2xpN0EkqZayZ6kXdZLjtleHj05AAASkOQAAG7123Ll0qVL1bFcEcB6+umnY2xXAACA/2LLbnLagC3RyWkDdgrB6dOnY1xbW6vG5Aoow4YNU2OTJk2KcUVFhRqzUwpkC6a5uVmNyTKrLaXK8qWcImHH7G+Svz9p6kF38SQHAHCLJAcAcIskBwBwq1/15Hbs2BHjTZs2pf3cAw88oI5XrlyZrUsC0A90Zeku+Sq+fS2/vr4+xnaHb9kjGzVqlBqTOw+UlJSoMdsvlNdjr1uO2d0E5NSrpN0T7Pny8/NjbKczZAJPcgAAt0hyAAC3SHIAALdc9+TsPI5Vq1bFuL29Pe3fzZ8/Xx2zdBeA/2KX50raaidpWS9537LLD8olsWy/TO7wbXf/Lisri/GcOXPUWGFhoTqWy3rZnqCcf2fvoXJunN21XH7W9uTsMmOZxpMcAMAtkhwAwC3X5cq33npLHX/99ddpPyt3BmfKAICusmU4WaK0r9vLV/GbmprUmJwmYHca2Lt3b4xtKVNOG7DTBKqqqmJsdyhImgpgdwxI2hVAXrdcfiwEPTWgvLw87Xfasm4m8CQHAHCLJAcAcIskBwBwy3VPzu7ineT111+PMVMGAFwrubSV7TXJnpx8ZT+EEI4cORLjAwcOqDHZ67LfKfteEydOVGNyOx3bZ7O9PXk9dpkt+bf2e44fP37V6wxBT2GwPTm7BFim8SQHAHCLJAcAcMt1ubIr5CP7tbzGKh/vk1bwtisJSHalltWrV3fq3PZ8slxrXxMGkF3yPmJLcnKVE3u/ka/U250G5H3KtlWS7i+NjY1px1KplDqWq5PYnbplGfLMmTNqTE4hsN8pv8f+W8jfkTRFobt4kgMAuEWSAwC4RZIDALhFT+4f48ePz8j3rFixIsYVFRVqrK6uLsZvvvlmRs6XRP6mxx9/POvnA/Av2WuzK/bL/pl9TV/2qOwOBXKl/4aGBjUmV/630wJkr8teiz2H3GnA9v3k9AL7DoAcKygoUGOVlZUxtkuO2e/JNJ7kAABukeQAAG65LlcuX75cHa9Zsybr57Q7H3SWfMU26fH9kUceUcc333xz2s/eeuut3boWAF1nX3+XZUD72rz8rN3gVLYZRo4cqcZqampiXFtbq8Z27doV42+//VaNNTc3x1iWI0O4cnUUuSKJHZMbnMqSawghFBUVpf072bqxvzdpekEmdiXgSQ4A4BZJDgDgFkkOAOBWnn19NMtyejLrgw8+iLF9jTaJrHV35dX/Z599Vh1PnTo17WfvvffeGMsVw7Mo8+vnAM7V1NR06x5me0uyD2XvwXJnbtmDCyGEPXv2xHj37t1q7Pfff4+x3W1cnsPeXyZMmKCO5Y4BticolwQrLCxUY2PHjo1xdXW1GistLY2xfedALjlo+5pJy3xVVVV16h7GkxwAwC2SHADArX5VroRCuRLoou6WK5PKbrZ8l5+fH2P7mr48tquayF0A7KalclUVuWpKCFdu2pqUE2RJUpYgQ9CrnCTtwGK/3+500FmUKwEA/R5JDgDgFkkOAOCW62W9AKA3sMtVSbZH1draGmO7C4Ds18lltEIIYfLkyTG2PcCOjo4YyykKIVw5nUp+VsaW7bvJaRK2l9iTeJIDALhFkgMAuEW5EgByTJb2knYvsKVFWT60K44klRblOex32nKpLDXasmPShq7ys0nl2VzjSQ4A4BZJDgDgFkkOAOAWPTkAyLKkZb2S2N6WPJa7fYegV/O3S3d1pUcme3vyO0NI3u28t+JJDgDgFkkOAOAW5UoAyDFZ9uvuTjC2XChXLklaccSez5ZS5fSGvlKSTMKTHADALZIcAMAtkhwAwK1c7wwOAEDO8CQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHCLJAcAcIskBwBwiyQHAHDr/5oAA58zxm9bAAAAAElFTkSuQmCC\n",
+      "text/plain": [
+       "<Figure size 720x288 with 4 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "show_reconstructed_digits(inputs, outputs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 3.3.2 하나의 그래프에서 오토인코더를 각각 학습하기\n",
+    "\n",
+    "전체 적층 오토인코더를 위한 그래프를 만들지만 각 오토인코더를 독립적으로 학습하기 위한 연산도 추가합니다. 단계 1은 맨 아래층과 맨 윗층을 훈련하고(즉, 첫 번째 오토인코더), 단계 2는 두 개의 가운데 층을 훈련한다(즉, 두 번째 오토인코더).\n",
+    "\n",
+    "![](./images/stacked-ae05.PNG)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reset_graph()\n",
+    "\n",
+    "n_inputs = 28 * 28\n",
+    "n_hidden1 = 300\n",
+    "n_hidden2 = 150  # 코딩 유닛\n",
+    "n_hidden3 = n_hidden1\n",
+    "n_outputs = n_inputs\n",
+    "\n",
+    "learning_rate = 0.01\n",
+    "l2_reg = 0.0001\n",
+    "\n",
+    "activation = tf.nn.elu\n",
+    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
+    "initializer = tf.variance_scaling_initializer()\n",
+    "\n",
+    "W1_init = initializer([n_inputs, n_hidden1])\n",
+    "W2_init = initializer([n_hidden1, n_hidden2])\n",
+    "W3_init = initializer([n_hidden2, n_hidden3])\n",
+    "W4_init = initializer([n_hidden3, n_outputs])\n",
+    "\n",
+    "W1 = tf.Variable(W1_init, dtype=tf.float32, name=\"weights1\")\n",
+    "W2 = tf.Variable(W2_init, dtype=tf.float32, name=\"weights2\")\n",
+    "W3 = tf.Variable(W3_init, dtype=tf.float32, name=\"weights3\")\n",
+    "W4 = tf.Variable(W4_init, dtype=tf.float32, name=\"weights4\")\n",
+    "\n",
+    "b1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
+    "b2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
+    "b3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
+    "b4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
+    "\n",
+    "inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "hidden1 = activation(tf.matmul(inputs, W1) + b1)\n",
+    "hidden2 = activation(tf.matmul(hidden1, W2) + b2)\n",
+    "hidden3 = activation(tf.matmul(hidden2, W3) + b3)\n",
+    "outputs = tf.matmul(hidden3, W4) + b4\n",
+    "\n",
+    "reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
+    "\n",
+    "with tf.name_scope(\"phase1\"):\n",
+    "    phase1_outputs = tf.matmul(hidden1, W4) + b4  # hidden2와 hidden3 통과합니다\n",
+    "    phase1_reconstruction_loss = tf.reduce_mean(tf.square(phase1_outputs - inputs))\n",
+    "    phase1_reg_loss = regularizer(W1) + regularizer(W4)\n",
+    "    phase1_loss = phase1_reconstruction_loss + phase1_reg_loss\n",
+    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
+    "\n",
+    "with tf.name_scope(\"phase2\"):\n",
+    "    phase2_reconstruction_loss = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
+    "    phase2_reg_loss = regularizer(W2) + regularizer(W3)\n",
+    "    phase2_loss = phase2_reconstruction_loss + phase2_reg_loss\n",
+    "    train_vars = [W2, b2, W3, b3]\n",
+    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=train_vars) # hidden1 동결"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "init = tf.global_variables_initializer()\n",
+    "saver = tf.train.Saver()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "훈련 단계 #1\n",
+      "0 훈련 MSE: 0.0077777165\n",
+      "1 훈련 MSE: 0.0073502758\n",
+      "2 훈련 MSE: 0.0077700005\n",
+      "3 훈련 MSE: 0.0078010904\n",
+      "훈련 단계 #2\n",
+      "0 훈련 MSE: 0.39963287\n",
+      "1 훈련 MSE: 0.009410476\n",
+      "2 훈련 MSE: 0.004594677\n",
+      "3 훈련 MSE: 0.00306658\n",
+      "테스트 MSE: 0.010246018\n"
+     ]
+    }
+   ],
+   "source": [
+    "training_ops = [phase1_training_op, phase2_training_op]\n",
+    "reconstruction_losses = [phase1_reconstruction_loss, phase2_reconstruction_loss]\n",
+    "n_epochs = [4, 4]\n",
+    "batch_sizes = [150, 150]\n",
+    "\n",
+    "with tf.Session() as sess:\n",
+    "    init.run()\n",
+    "    for phase in range(2):\n",
+    "        print(\"훈련 단계 #{}\".format(phase + 1))\n",
+    "        for epoch in range(n_epochs[phase]):\n",
+    "            n_batches = len(train_x) // batch_sizes[phase]\n",
+    "            for iteration in range(n_batches):\n",
+    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
+    "                sys.stdout.flush()\n",
+    "                batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_sizes[phase]))\n",
+    "                sess.run(training_ops[phase], feed_dict={inputs: batch_x})\n",
+    "            loss_train = reconstruction_losses[phase].eval(feed_dict={inputs: batch_x})\n",
+    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
+    "            saver.save(sess, \"./model/my_model_one_at_a_time.ckpt\")\n",
+    "    loss_test = reconstruction_loss.eval(feed_dict={inputs: test_x})\n",
+    "    print(\"테스트 MSE:\", loss_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from ./model/my_model_one_at_a_time.ckpt\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAD+CAYAAACnfntIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF3NJREFUeJzt3VmoldX/x/Flaurx+FOP83HKeazQkhyKHEACJTHxorwoKkiCgqQBg6C66bK8iYpCy4LCwMIibQBFQ80cSc3UY+U8dByO8/i/6b/6rG/up32Oe2/P+Z736+r7sPbZz7MFny/P9/ustZpcu3YtAADg0S03+wIAACgWkhwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwC2SHADArWYlPh/Lq9QfTW72BQANzYEDB7iH1ROVlZV53cN4kgMAuEWSAwC4RZIDALhV6p4cAKAe051pmjSpW+ve7m5T1+8pBJ7kAABukeQAAG5RrgSABiirJJg1dunSpWTs6tWryfEtt/zz7NO0adNkrHnz5jn/rhBlzmLgSQ4A4BZJDgDgFkkOAOAWPTkAuIls/8r205T2yC5evJiMaY+sWbNmOcfs92d91o6pK1euJMf5Xnep8SQHAHCLJAcAcItyJQCUmJYos163t2U+LQnaUqIe6zSAENLS5n+VGfXYlkT1HPb8dmqC0nPa36vXWpvSbb54kgMAuEWSAwC4RZIDALhFTw4ASkx7bbbvpEtnXbhwIRnTnpXtrenfZb2yb/t1tg+m32vP36pVq7zOr3EIIVy+fDnn9ej3FKIHZ/EkBwBwiyQHAHCLciUAFIGW3mwZ7ty5czG2r95rae/MmTPJmJYPy8rKkrF27drFuE2bNslY69atY2xLmToWQjptIGsKgS1l6pgtZeqY3b1AvyerXFnXUiZPcgAAt0hyAAC3SHIAALcafE9uzZo1yfG8efNi3L1792RMX3999NFHk7GKiorrxgBwo2xv6+zZszHev39/MrZjx44Y79u3LxnTflrPnj2TMb1vtW/fPhnTHlmLFi2SMdu/O3XqVIztVAA9f9b0gq5du2aeI9d32r5b1tSDfPEkBwBwiyQHAHCrSTFmmGco+MkGDRqUHO/cubNO39O2bdsYjx49+oauqS5uu+22GM+dOzcZ69WrVzFOmXvpcwDXdeDAgbzvYTo14PTp0/Z7Yrxr165kbNOmTTHW0mEIIRw/fjzGWh4MIV3JxJZA//e//8U4a5NUe606LSGEEDp27BhjO72hS5cuMR4zZkwyNnbs2BiXl5cnY5qD7LXYY1VZWZnXPYwnOQCAWyQ5AIBbJDkAgFsNfgrBF198kRxrPXvYsGHJ2NatW2O8du3aZOzLL7+M8bJly5KxPn36xHjPnj15X5utfXfr1i3Ge/fuzfl32p8LIYSXXnop73MCuDls/0hff7evwt96660xtktgde7cOcZ2yS19NV+nIYSQ9uHsuxY1NTUxtn0+e93ah7PXpve/Q4cOJWPar7NTCGyPTtlzFBpPcgAAt0hyAAC3GvwUgkI5f/58jH///fdkTMuVVVVVeX+nliRCSMuV+p0hhHD06NEYL168OBmbNm1a3uesBaYQALVUmykEyk4hOHnyZIwPHz6c8+/sNAFtgdhypX6n3WlAj/VeF8K/y5dZOxZs3Lgxxl9//XUypiupPP/888nYhAkTYmw3bc3arUGP7eauTCEAADR6JDkAgFskOQCAWw1+CkGhtGzZMsaDBw/O+bkhQ4bU+Rw6beHYsWPJ2D333BPjyZMn1/kcAOofuwq/9qU6deqUjNk+mNKenJ2WoMe6jFcI6RJjtgdn+376t0eOHEnGtmzZEmO7C8HQoUNjbHdI0Ou2PTm7Q0Mu7AwOAIBBkgMAuEW5sojsKt3Tp0+PsV1l4K233oqxLR8AaHjsK+9KNzXNKsPZaUhadrT3EC0DZk0hsH+nrRrrr7/+So43bNgQYzston///jG2G0/b36G07Gn/zbL+DfPFkxwAwC2SHADALZIcAMAtenJFtGDBguRYV+3u0KFDMta7d+9SXBKAEtHeV/PmzZMx7TXZHpn23ew0AV2x347pa/p26S7t19kxe226Q4pdukunEIwaNSoZmzJlSox79OiR8/x2ykBWv7AQeJIDALhFkgMAuEW5ssB2794d4zlz5uT83OrVq5Nju8kggIZNS2+2tKjlSrtpqO4uUFZWloxlrXiiZUC7YbNei51ecO7cueR427ZtMf7222+TMS2l2o1Q+/XrF2O7qomeP6skaadTMIUAAIAMJDkAgFskOQCAW/TkCmzJkiUx1vp1CCHMnDkzxn379i3ZNQEoPe0n2d6S9uHsmPaz7Er/bdu2ve7nQkj7WVlLftkltqqrq5PjzZs3x9juljJ16tQY65SBEEIoLy+Pse0zar/Q3hf1s7ZfSE8OAIAMJDkAgFskOQCAW/TkbpCtLy9evDjGLVq0SMbeeOONGGft/gvAF9sj0///dqdu/azdBkfH7PJYulyXnSen9yLb59Ptc0IIYdWqVdf9uxBCmDZtWozt3F69F9pr096anQunPUI7908/W9f+HE9yAAC3SHIAALcoV96gDz74IDleuXJljB955JFkjGkDQOOh5TVbvtOV/23rQkuEtux45syZGNvX9LWUacuM+nr/wYMHk7GvvvoqOd6xY0eMp0+fnowNHTr0ut8ZQghHjhyJsf1NWb83qyTJFAIAADKQ5AAAbpHkAABu0ZOrpU2bNiXHzzzzTHLcrl27GL/++usluSYA9Y/tmSntQ9mdufWV+qzlwOyyXtqHs1MPTp8+HeNly5YlY+vXr0+OdemwiRMnJmOdO3eOsd2ix55TaW/R/ib9t7C9y0LgSQ4A4BZJDgDgFuXKPOhj+cMPP5yM2ZLErFmzYsyUAaDxsKuaZO1CoJ+1u3/rPSVrVRNLy4z2WpYvXx7jjz/+OBmzOw3MmDEjxgMHDkzG9PV/ew473UHp77e7IOhvLMSUAYsnOQCAWyQ5AIBbJDkAgFv05K7D1pp1B1xd8iaEEIYMGZIcv/baa8W7MAANku01af/K7mSiUwPsmL5ur9OVQkinEOzbty8ZW7p0aYx37tyZjI0fPz451qW8evbsmfP8dscA3d3ATifQa7P3V/0eenIAANQCSQ4A4Bblyuuorq5OjvX1W2vhwoXJcUVFRTEuCUA9lzVNwK5OomO2JKmlPvu6vb7Cb8eOHz8eY3vP0p0GbJlz0qRJyfGwYcNibFdj0Q1es6ZM2OkEdqPUXH+X9bm64kkOAOAWSQ4A4BZJDgDgFj25v508eTLGo0ePzvk5uyTOiBEjinZNABou7UvZ5f+yek/62datW+f8O91ZIIT0XQK7W4q+wn///fcnY/fee29y3KZNmxifPXs2GcvaJUCv1fbksv5Oe5J21/BC4EkOAOAWSQ4A4Bblyr/Nnz8/xlVVVTk/Zx/tizFDH4AvWdML7FQAnW5g/05Le3b3gE8//TTGixYtSsZ02sDw4cOTMd0INYS0JGqnPpSXl8fYlhb17+xuCVkrpdhzFBpPcgAAt0hyAAC3SHIAALcabU/OrsT96quv3pwLAeBCVn/e9p302E4n0DE79UB7XXangRUrVsRYp0SFEEKfPn1ibJf10t0DQkiXFcv6TXZ6gfbosvpsxe7B/et8JT0bAAAlRJIDALjVaMuVK1euTI51dW1LN0Zt1apV0a4JgE+2JJm14olOL7Cv4tfU1MTYlhn79u2bc2zs2LEx7tWrVzKmq6GEkE5TsOVSZXcoyHfz01JPu+JJDgDgFkkOAOAWSQ4A4Faj7cll0fp1CCF89913MaYnB6C2bB9Ke3K2P5fVE9PX9O1UAH13QPtzIYQwePDgGHfv3j3n+ULIf0dze91254H6gic5AIBbJDkAgFtNsl5lLYKSngyZ2D4BqKUDBw7c1HuY3q+1dGiP7Ur/+rr/f21M2lB2VqmsrMzrQnmSAwC4RZIDALhFkgMAuFXqnhwAACXDkxwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwC2SHADALZIcAMAtkhwAwK1mJT7ftRKfD7k1udkXADQ01dXV3MPqiYqKirzuYTzJAQDcIskBANwiyQEA3Cp1Tw4AcBPdckv2s821a9euGzdUPMkBANwiyQEA3KJcCQD1SJMm/7wZb8uFWaXGK1euxPjChQvJWLNm/9zq9ftDCKFp06Y5v8fSz169ejUZu3z5coztdev57fmKjSc5AIBbJDkAgFskOQCAW422J/fJJ58kx2fOnInx+vXrk7H33nsv5/e88soryfHEiRNjPH78+Bu4QgCNkfazbP9M+2WnT59Oxo4dOxbj6urqnH9XXl6ejLVo0SI5bt68ec4x+7dKe3S276Y9Oduvs7+x0HiSAwC4RZIDALjVpMQz2m/q9Pmnn346xu+++25RzjF06NAYr1q1Khlr27ZtUc5ZR+xCANRSoXYhyHrd/9y5czE+ceJEMvbHH3/E+JdffknGNm/eHOOjR48mY/p6v5YOQwihe/fuybHewyZMmJCMDRo0KMZt2rRJxjSX2N+kUx/0WkJI/y3stWWVMtmFAADQ6JHkAABukeQAAG65nkKgPbgQ8u/DjRgxIjmeMWNGjHfu3JmMffjhh8nxtm3bYvz5558nY0888URe5wfgi+1DXbx4McZ2KsCBAwdivH379mRMpzft2LEjGdNpUL17907GBgwYEOOzZ88mY/aepvcw7cGFEEJFRUWM7fSCW2+9NcZ2+THtrdl/C516YHtyhcCTHADALZIcAMAtd+XKP//8M8bvv/9+zs+NGjUqOV66dGmMy8rKkjF9DLcrdO/atSs5/vHHH2OsKxAAaLzsCiBasjt58mQyVlVVFWO9n4UQwqlTp2JsX/0fPXp0jKdOnZqMaRlw7dq1yZhOWbDHR44cyfk9dpqAsr+pVatWMba7F+g9Vcu4IaSrr/zXZq+58CQHAHCLJAcAcIskBwBwy11PTvtgdsky7cN9//33yVjW6tpqwYIFyfG6detyfnbatGl5fScAf7KWpNI+f8uWLZOx9u3bx7hdu3bJ2AMPPBDjkSNHJmN9+vSJsV1yS6cp2J5YTU1NcqxLgtnfoNet/cEQQujQoUOMbf9Me5C1mV6g56srnuQAAG6R5AAAbrkrV+ojvH2FXx999ZXW2rDTEuwrrwAaJ1va02M7hUDbI7Z8pyuJdOrUKRnTaQNdunRJxlq3bh1ju6rJvn37YrxmzZpkzG4SPXz48BhXVlYmY507d47x+fPnkzH9HbYEq9MS7L330qVLMbarqOh31nXHHJ7kAABukeQAAG6R5AAAbrnryalC7cS9cOHCGOvuu9czefLkGPfr168g5wdQ/9menPaTbN9Nj+3r/voave1f6TJXtkelPTK7VNeGDRti/PPPPydj9r0C7cnpLuEhpEtw2fPrse1B6nJgdgpD1m/Sf1N6cgAAGCQ5AIBbrsuVdbVx48bk+KmnnoqxXXm7W7duyfG8efNirI/hABovW2rTcqUt3+n0Anu/0Vfs7Xdq2VGnDIQQwurVq2NsN2kdN25ccnznnXfGuGvXrsmYXo/daUDLrjqdwV6rXdUka6PUupYoFU9yAAC3SHIAALdIcgAAt+jJXYfWr0PI3gF39uzZyfHAgQOLck0A6jfbP9Jem51eoEtZ2b/TsaxlA/VzIaR9uEWLFiVjq1atirHdUfz2229PjocNG5bz2nQKQdb0BnttWUt+6W/M2qGgrniSAwC4RZIDALhFufJvjz/+eIw/++yznJ977rnnkuMXX3yxaNcEoOGozWodWa/U68ol9lV8nUKwZ8+eZGz58uUxXrFiRTKm0xLuvvvuZGzKlCnJse48YKc36LV17Ngx55jdBaGsrCzGdmqVPYeyK6fUBU9yAAC3SHIAALdIcgAAtxptT84ubfPNN9/E2O54qzvwvvzyy8mY7jYOAP9Pe0126Srt19nX5vWeYvtXOp3JLt21a9euGGsPLIQQevToEWNdtiuEf+8+rq/065SBENKdXexUAO3D2R6k/ib7ndp3y+rP1RVPcgAAt0hyAAC3Gm25cubMmcnxkSNHcn722WefjXFFRUXRrglA46AlO/uavK4kYst+R48ejfH69euTsaqqqhjbEuSkSZNiPHjw4Jzns9dmV3vSlUzOnDmTjGmbJ6s8a8uV+p1ZOxLUFU9yAAC3SHIAALdIcgAAtxpVT05r2LoEjvXQQw8lx3PmzCnWJQFwSvtpWTtc2z6Ufvb48ePJ2NatW2O8cuXKZEynFAwfPjwZGzBgQIzttAQ7ZUrPb6dIaY/OLkemUw86dOiQjGnf0e5QoH9nr60QeJIDALhFkgMAuEWSAwC45bond+7cueR47ty5Mc7acfeuu+5Kjlm6C8CNsDtcax/OLuul9y2dFxdC2pPTZbxCCGHgwIExHjVqVDKmy3rZJb/sXDjdzsfSPpz9u3bt2sXYLvmlf2eXVNTfb+cMan+wrruE8yQHAHCLJAcAcMt1ufKdd95Jjn/44Yecn9WdwZkyAOBGaRnOliR1tX1b9qupqYmxlidDCOGnn36KsS0r3nfffTEeP358MqbLfJ04cSIZy1pWzO4KoDsN2J3BtQRrpxfob7JTFnTXcns+++9WFzzJAQDcIskBANwiyQEA3HLdk7O7eGd58803Y8yUAQA3SreUydrO5tSpU8nYwYMHY7xu3bpkbPv27THu27dvMjZy5MgYt2/fPhnTPpjdIse+mq/Tq2y/TqcG2L/Tft2xY8eSMTttQNl/m0LjSQ4A4BZJDgDglutyZW3o4/SNvLaqr/XaR/2sHXeVXall3rx5eZ3bnk/LtcVY3RvAP+xOA1mv1Get5KGr9NtdCPQ+ZVdD2bJlS4xtuVL/zu56YNszev+x9yItdWp5MoQQTp48GWM79UHvP2PHjk3GKioqQjHxJAcAcIskBwBwiyQHAHCLntzfunfvXpDvmT17dowrKyuTsUOHDsX47bffLsj5suhvevLJJ4t+PqAxsz25rNX1tT9vl+fSHp1d5krPcfjw4WRsyZIlMf7tt9+SMZ1CYHtwbdq0CfnS6Q62l/jrr7/m/M4JEybE2P5evZ667jSQhSc5AIBbJDkAgFuuy5WzZs1KjufPn1/0c9qdD/Klr/Xa0oZ67LHHkuMxY8bk/Oy4cePqdC0Aas+W2rS0qOXJENL/47bMqTsG2FVN+vfvH2Nbrty9e3eM9+/fn4xVVVXF2E6RsqVF/R133HFHzs/aKQRq6NChybF+j23jZG3SWgg8yQEA3CLJAQDcIskBANxqYuvBRVbSk1kfffRRjHWl7f+yefPmGNfm1f8XXnghOdZ6uvXggw/GuHPnznmf4wYU/l1dwLnq6uqc9zB7L9Xell1KK6vvrv27vXv3JmM7duyIsfbgQkiXDrO7f+sSYLaXZncI0PuU7Z/ptdk+Y79+/WI8cODAZEz7bm3btk3G9N/CLj+Y9e9UUVGR1z2MJzkAgFskOQCAW42qXIkE5UqglrLKlVbWyiV6bFcgKS8vj7G9P+sOBZaO6Y4AIYRQU1MTY1uu1LEQ0nKinV6g16rXGUL6m7J2XbCyypVZKFcCABo9khwAwC2SHADALXpyjRc9OaCWatOTy6L9K7vMlval7Gv6OhUha6kwO2WhZcuWOa/F7v6t12Zf4dd8YfuMej1ZPchCoScHAGj0SHIAALdc70IAAPWBLddpidLuXqCftX+npcWs1/TtWOvWra/7HSH8u7SpZUe7Ooleqy2X6nGJ22CZeJIDALhFkgMAuEWSAwC4RU8OAIrMThPQ3pYdy+pnaY/O9vJ0pX/bk9PjsrKyZOzChQt5f0/WddanPpziSQ4A4BZJDgDgFuVKACgxLe3ZV/Gz6GooWauI2FJm1uajWp4MIS2fFmOlklLjSQ4A4BZJDgDgFkkOAOBWqXchAACgZHiSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC4RZIDALhFkgMAuEWSAwC49X/kgxt5o9BfFQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 720x288 with 4 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "show_reconstructed_digits(inputs, outputs, './model/my_model_one_at_a_time.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3.4 특성 시각화"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from ./model/my_model_one_at_a_time.ckpt\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABeCAYAAAAzI++3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG9NJREFUeJzt3VePbEfVBuDXgMk5mpxzMiZHYSQThGVhggAhHa64gD/CTzgX3CBLCBASYMkImSQQOeecTTQ5m2Sbi0/Pqeo13fbMdM/M+bbXe9Mz3TtUrVp717tCrTrn+uuvT6PRaDT+/+NmJ92ARqPRaOwG/UJvNBqNhaBf6I1Go7EQ9Au90Wg0FoJ+oTcajcZC0C/0RqPRWAj6hd5oNBoLQb/QG41GYyHoF3qj0WgsBLc4zptddtllN4llqadOnTrnIMe/5S1vuUnI5Q1veMO+5fK2t73t+iS57rrrkiQ3u9kyucfrXve6fcvk9OnTa/XEau9zzjmQ2p21eOMb33igjlxxxRU3iefn4osvvlG5LPMpaTQajZsgjpWhHxT//e9/kwyWliS3uMX/Nfn2t799kuTvf/97kuTcc89Nkvz73/9eORfufOc7Jxls5pprrkmyP1ZzttS70Y6b3/zme37Tj9o/zNbv5HfttdeunE9ut7zlLVeOI0fXmZnyccjlxph5ZfDzuN/udrdLkvzjH/9IMuTmHO2nO2R2q1vdauV4xzmP7Pw+69BJ6MomHda+f/7zn2e+Iyfj6399Iova1033qtaB/+fz6FS91klhndVHVvSnPg8VjncNMvZ91TXHze+yo7CsmqE3Go3GQnBWMnQzllnt1re+9Znfbnvb2yYZDONvf/tbkuT3v/99kuQ///lPkjGz3ulOd0oymEdlmo73aWZ13WTMqu5d23lcrOyG7oeFYqX3uMc9koy2659rkCm5/Otf/0oyWKrr+Z68/Z8MGZJLZazHyeC1q47R3A59xsKcQzaYufaTDV3yPasQ4ySz+djK7o8T2ulztuhuc5vbJBl64bNacDOTnP+fn8V199Tf2VKq96pWwHHLyP2MdzLaqx88AOShzf435j/+8Y+TjL5439zhDndIslfnZl2p4+Me9OowzL0ZeqPRaCwEZxVDrzMS9jXP4NjUH/7wh5XP6sMy+97tbndLktzxjndMMtjVn/70pyTJX//61yRjhq2+xfr3jJPyrZvtZxZUmbd+6HdlmZuyR377298mSX7zm98kSa6++uqV62Iu899VDschl+rHxjxZFslos77/+c9/XvmkS6ya+973vkmSu971rivnkTNdm1lWMmSdbPazw1Fm62xiyXMbtMsxfMXapW+brBd9rUzeeevGnhxrLOKknp918RZy8R22TKa+/+Mf/5gkueqqq5IMnSAfulQt47/85S9JVvvs2Oq/30ZHmqE3Go3GQnBWMPQa7cW21s3gmACmbcY0U5oRK2O76KKLkiSPecxjkiQ///nPkyQf/ehHV87nAzN7Jnsj1uD/ddkOu4Tr8utXK2Q+xqf2Y0fkxpKR9XPPe94zyWAQfIKf//znkwx53u9+90syfIPJZt+5c+AomBiGWLOaZrYM5MbqIAvWC2Yu7oAhuTZdIRv9u/vd775y3nx/fSZXDJDVsEumXpm5NhgXbZi/q1kadNnzQmZk9OhHPzpJct555yUZlu0PfvCDJHutHZZxMvSE7mkP+VZ92RXIw7tCm7RntubIzG8+Z31PhrzI5WEPe1iS8V6hE/rEwv3Rj36UZOjgDPe+y13ukmTI5TBZQc3QG41GYyE4Kxi6Wa+ySgzUDDsfi4E7xkyKOWBT3/zmN5OMGfcBD3jAyvmYOZ8h/+nMMPyGCWEaZtKjyurAkirrNnPP+cX6g3nX2f6HP/xhksEcnvvc5yZJ7nWve63cE3O7//3vv3KdBz7wgUkGA0mGTCsDxjiOgnlVZlv9sbOVxA+KFZMNRk4XnPulL30pyWDwZEDOvnfd6ldO9mbZsBzo8FH40I1xvTfr1Rgmg1nqs+cFa/bp+/qc0UEycL3f/e53K8dh8vM1xC7I8agtOXJhQbG+fT+PW40d+PzFL36RZDDtX/7yl0mGHB760IcmGeMq2849yMkY/OQnPzlzz5qZVvV1XQzkxtAMvdFoNBaCE2XoldlWfx5WM2dWmOnMZpiB2R5L+exnP5sk+cAHPpBksBXs1j1EqzET7O0+97nPmXvWXG0zavVb74qRVkZes31qznwyGBFfJzby05/+dOUclodPLOGLX/xiksHkWTwYCNYw+x1rJkfNL64ZA7sA2WCQlVk9+MEPPnOsPrIgMEWy8v+HPvShJENXHvvYx64cR6dYb5i+/n3ve987c09yoj81C6euB9gGrlFX92KH+i9OkoyxwDzJr2ZF+aSDX/3qV5Mkn/vc55LstXL0y/M4+4pdo8a/6mrtOSa0C+gT2Vd9nbNcPP9k6dO7h16Ri+cLvv3tbydJTp8+nWRYe5deemmS5MILL1w5P9lr9dR1N9Wy2Q+aoTcajcZCcKIMHeOsvjMztxl1ng3NZj/72c+SJB/+8IeTDFb1vOc9L8mIKmOUrsGfZqZ81KMelST5yle+kmSvny8ZbLXmr2qfT8xoW2xiLO7v+9n//fCHPzxJcu9733uljeT061//OsmQn/6+613vSpK8853vTDLYHDnWvPYZjnUMmbn3pvE9CGq+PCbomvzb2MycleBcffebOICce9d88YtfnCR5wQtekGQwcQycNaefzv/GN75x5p7YvHsbjxtavXlY1MyeutqTNTuv7iSvmq1RM0AweZatbLBPf/rTSZKnP/3pSUacgWw9d2JYyWDrnhOWDv2pdZYOi1qbp2b7+PSc8nfP57I8tIlekW21aOkWi4cuyP7hM5cpNfeR7Mmjrg04jDyaoTcajcZC0C/0RqPRWAhOxOVSg4hMwlrEZ116kQCXANYXvvCFJMNcdi1B0U9+8pNJkic96UlJ9gZBpS1yxXBJzO6Oupy3pi3WBUeHRV2WX4NeNVVsdi9wFXEDCF5xSTGpn/a0pyUZQUVB0yc/+clJkhe+8IVJRlCvpgXOqZKCWwKSNWhrLHYZAKxL0rXPeM1j8fWvfz3JCFgJ4Bn7973vfUmGS+BlL3tZkuFmYDbTP2Y4dwWdmVP0oBZn2pRuuQ3oBT2sRcPqcvZk72KqWqpAUPNXv/pVkuFC4HLhfuK+IIP6DHuukuTKK69MMnSOu+aCCy5YaVMteX1QGCc6yuVUXVHcJPqYjAVC+kMHpD2TrefFe+SJT3xikhH8fNaznpUk+f73v59kb/LBnLaonZ4b7q51z9p+0Qy90Wg0FoITDYrWwFBNXxSQmFN9Kju2QMZsX0sCmDGf8pSnJBkz62WXXZZkzKTPf/7zk4w0vXmRRl0KXotj1SDgtqjpnLXPWKrA5twPLAjI7pJLLkky2NG3vvWtJMkjHvGIJMkjH/nIld8FjHwKfs1ykeKIjVU5YJCHYaP1nMrIMZ663H0ue1wXedCJj3/840kGc2SdCFyx+lgpL3/5y5MMK4A15HdlEeb710VytfDSNgydfmDVZEMv6IB7Yp/JCNKSm6A5a+S73/1ukpF2R2bG3RJ3bfjYxz6WZFgkGKsgfTKCftpVS1nXctmHRS2LW5+juuhwTmLA5o3tFVdckWTItloTmPz555+fZKQ1Sr6oi+t8zmm/dMU41ZRiOIiF2wy90Wg0FoITYeh19qrMtvpF16V48Vs+7nGPSzLYCr8YVvLa1742yWAWZmd+U6l/0h6ltM3+K37FWkyINbArhlFLmVbGVYtFYZrJYMkslmc84xlJkoc85CEr/cNAPvKRjyQZPj1ywMz4GS2swLzmQlRkZDm0ccTI4FCF+gt7q1sM+n+TVZcM9iR+YsylmL3mNa9JMlI0a5GxyrrIRJwCK50ZurLMmO2mTR22Yeg1XZEM3JsueCbm9tGpWgbW/65FdtJ6LVgTfxCfsODIuNC3udzAM5/5zCSD5dcSDLveoq6W6a0bk5DBgx70oDPnsEC//OUvJxnPyalTp5KM0hdvf/vbkwx50X3PhfcPi5kV5/z5XVefMXpGhz1fB4ktNENvNBqNheBEfehmZn4lMypGbGadC0LVTWj5/OqyZ6wMy8JaMAw+dTMnFmMWX1dEpxZj4k+rzPqw2LShLLaATcncEXFPBjN/5StfmWRYKPx01VeKSbBwWDB1K79qHc3L+DELzIscyGkXMQXXqEvnsZa6lH5moxiYNn/nO99JMnQAU2O9aDedwZBkOswxi2T0e7YKavvqQqJdFixzrbqwCDxfc3G7z3zmM0mGLtE1fcamyc7iH2PruaAnZIiRs/TmZ0FcSvv4iHe5sUOyNzuuLqUnL/79xz/+8WfO9Sxj3HSCpUJnZP3oNyutLuaqJZn9P8cDHVvLlrhXtUL3g2bojUajsRCcCEOvPiGzHSaBPZrB58iwWUtuMfaKjTz1qU9NMhg8nzDfOV/Zc57znCSDZVrOXf2k8/0xsTqz1gJHhwWGiFnwnWOKllVrG1adJK9//euTjGwd/lQ+TjImv2c/+9lJBkMnB9kvtRwtv/O88UctflRZM5/gvMT6oKi5/8Zbf/RT+2ffLV8t+E17sDHWDF86vP/9708ymL170EtW3+wPr0WbqqVT/d6HQd2A2qexqdsozvK3PkM+OT2Q6YNJulbdts+9qwVct72bLREy4U+uPvNtZVJ11fXq84Qtszrm9QNk5BljoWDLLFtwbs0lx8Bdz7ujbsCT7C0Tof3eJzWOtB80Q280Go2F4EQY+qZNhbEAEWT+qZlpVR+3WZ4/DGvFqhTRMcOaQc1+7jX7GZNVJmqW9YmpVUa+6wL9Nd8YGxVTeOlLX3rmWCtAMSiyrLM/Zou9YOSYCbnwRbsnOWJXychmwTSq/1i7tymfqz2u5f+63gD7nrOTWFtQN+9wDcyNb5eFKEaBWcrgwL6wrjn3nbVSc6232VZsE1xbvzwnNf+alZaMOEDd6FrcSrzAs4eJijWRFb2ScSWWRSaznniWPHvaWxn6YbGfzeWT9VsGAmuNztAjWzGC94v3yXve854k4/nQR1a0tlnnMZflroXvPD/V6jwImqE3Go3GQnBWbHBRNzLmO8aYZr8oZmnFG1bCD4pZYImu5RrOx77MzGZWvrOZNfCjYb21XO6mLIPDAvupW4rxRfJf8nsmg6nWmAK5YBDyh2ufrPrEQNwLW8BI5hxzx5IpBoSxsgZ2USoWjC8GZJzJTDZPMlYx0hW59HL0MaRakvm9733vSr/IgoWGfVc/6gzssPrOyWQXK0Wr9VK3Y3TvOUvsCU94QpLBUuWZuyZ9IFfjzfKo2yDqV10tPG+qscmnvUtrZR3qhjQYr/vOcTJjLwNGv703vIu8b2rc72tf+1qSEWsiT8/fujhBLR9ctzqsG3LsB83QG41GYyE40ZWiZqBaZbBG7ecZClvik8I0ZbNgoNiZa6siKKvD75inmdS9Z1bDp2oGrYx8V8y8+pwry8Me6urYJLn88suTDOYgewHLxyAwdUwNy5dvi3H4ZMFg8HOdiZoRUfuhLdsAi5791PM96ubYM+uiC1aIGlN+30996lNJht+Yf9N4s9boJV+0TCpxhnk9QM1QqNvz7UImWLRrVr89VmlcjG0y8sR9qkCJWYupyLemL9Y+uBbZaAuGjtnOm2qQSd3AgWw2+bwPCv2vz6nPWmFURdJkvEdkv3nWyNA1vX9U5iS3d7zjHUmGrog5sKbrtpXztWr+Pev6UCusD3xGo9FoNM5KnGgeep2BzOrVbzfn0dbsAdF17EmkGGN7yUtekiS5+OKLk4yZ2DVrrRAz8cx+a51i/1dmsS1Td73qb8MssSrsYV7NqsYENoT9VBZJplbCYRoYOTnKeiEvvvR5LMgIa3es73eR9VNXgtb/+cfJZLbmyIteaR9GjlXSFcdZRcyKwfT1izVHlqyfZLDPefVosn2t7xn6ZUzqVmvVZ01Gyd7t2aru+iSTmnP/wQ9+MMlg6HT26quvXrnXnKGBxW9aJburTcRrnR99YbXV+jMzQ9cflq3nwQbiKnCqRvqiF70oyd7KluJK1oWIVZDf/EyQbd3ovmbiHQTN0BuNRmMhONEdi2pVNFFfUXqzvFVtyZi1fNYVbXyC8tLNuGZts6HZkc9MtB8DxeaS4euqtVxqtsG2jLSyJPeRN401mfVn3y0Wz3dno2Os0oo+MpXpAe7Ff4dRyJ7BGuY1ASwp59Ro/C5qf9c4S61wiVmpFzLXuCaLuuMQNk+ONROBL921VeBzvn7ztc/6iZm7NtnUeu7b1C1hhdVdkSrjk5UzM2N51uIK2sHKck07gNE9usanTtfFoDx/ZDs/C+Ifxs5zNOeq7xL6ZHysSWHRsqjmNopLkZXNsDF015QpRdZiV675pje9Kclg6vru/HWbmMuwMa7eaXXNyL76vu8jG41Go3FW46xYKYqFYQOYkllfVkIyZiur9jBHrASbxlj5uOSH8vVhbxdeeOHKebW+czJYba0rMrP4XcLMTA78knz3ZDDnhGNUqi6KrvMXs1xkaJA1plYr39WVpjJa7HCUDFannXVV6i5RM6NqHZlabXNGzZ5yDtZU8+Zlifier5UuseLEb+YVqazMupvVunZtC/2qmTPabUxrffpkPC9Yc409ea5YtpglvfIMYOjV2p73FK33ghqL2jYvnd7RaW0wfiwpz5NdiJJh0TvXc6P/r3jFK1aOw6prhhQfu76waLzDxK6S0W+xkJoZ1VkujUajcRPGsTL0OuNgVdgxZmTmxjzmaot+M7tiQHxYZkL+M2wMw6iV17DvWrluzqOtVdHqCshNWTuHhdmdj1GbsYVazTAZPnO+cpaIXGt+OXnntYqiPjpO/8nPqsh1GRPYJx+g743bYWp1VF9zjS8YJz5+/tE5m0Q7yBET0l6M0TmYJGsGC1N1kWWElerfHFdwTK3AidGS6y4ygNyDDtcdrmqcKRmy0Gbjal2H50Qf7bxDZmIVrlPr+GjTHN9hyZIzndpl3fwZVf9UZmVJyWKa66GDY1k5fOvOoZfy9L2zLrroopV7y9t/85vfnGQ8N/NYeH94Br2T6Aor6SCWSzP0RqPRWAiOlaHXnNdaq6QeZ7affW8YRM1ddSzfL1+mTAUzI+ZutpbTi+Ws2+cUO6m72t/Y3qiHRc1ZxhYq+5sr5pnNsTPMXB14OxlhFBgYFlrrXfCP13o181iQJQbh3nWF3mFQs0Cwz7qKTvv5S+cdYbBPesZ64yeViaDPjucPrYwSHC+uMPezZibsMv+8ggxqDRBtMHaz/75WDmWN6At5ygwxtp4n/niWUa2zT1fnZwGrrTtc1WqiuwJLzHNJt+mU+8+64m+xt5oR5Xc+cyvTZUixjD1vb33rW5OM/PVLLrkkyaq3geVSd7Uiy8NY/s3QG41GYyE40WqLZiIztNnL/5jHzDDMvmZCjFzGgWvKgjE7W/nIt1UZiuyNdbnT2EjFLldEzsBksGF9xvqsfJvrllS/unNZKCwR/rp3v/vdSYb85OFjqfx5arjo61zNECPHmmsu+y6BgWNQdKLmXM9jgXUaS30xnnyorBbX4Od2HhaGQVUZzQyT5edcjLXq+rb7Z87tdI/63NRa7MnQJePOR65mD993XWns+E984hNJhn7VOEJdJ5IMhkmva12bXchihraSj1286v6e3hHz3zVzS//ojkqc2mwPVqwfgyePU6dOJRnvlzmnvNbg8ds22WLN0BuNRmMh6Bd6o9FoLAQnurCIaVSDO1BN+mSYUcxIJiSzk4nIdBIosqiA6b4pFXFdoMYx2qGdtUznroI77uezbloBs6laU/IEcciBfBQoIg8LRcitbsAg/fGGAtncPNpXF/Jsk87pWsxTrgDmM3PVcYJTyXAjAfeCYJTl3VxX5PmqV71qpT+CU/rJvbVuC7XqrqHDtUzELlCDZmRhjMhqdi1wvRlX+iA1k8vI9oZcXFdddVWSoWe1PEJdrDUHz8mCa6um/db0321h0ZyxIBeulvr7/DedpRMWGgpu1oVGZEunyPfVr351ktH3dQvM6FctDU2Wju2gaKPRaNwEcaJBUbMXplNLo5rV5mR8jEFgymyLKWAn0qukqNXAIrZVN0/Gtubyn1iWYyoz2qb41DqYmbWVJYD1kMfMvOq2VWZ5rMhnXSik2BD2IrVPn7VFuujMRjEr7IZcavnSbVBTzer3GJEg1LwJLxavfVI39Q0LU3oYO2W91BIP2oDpY3OzZVm/qymcu9SVmj5bLQm6MBeEojMsMwte9F2g+NJLL02yN+XP2Er51AYyFaSeZVKDoPX52XXaIusUA/ZuEOA0jnNSQf2tbmFJl7x/jGNNznBPlrHrCMTPJRFqOe4bKl+xXzRDbzQajYXgRBn6jaX9YZOzrwtTwCQx6cq6LJwxI2K32EvdWqr66+dZko/LOdWS2HXaIubruhhHXSgyL/3HILRVeiHLRL/4yi2YqIzD8aCIGQY/F3pyrbrJ91Ggsn33qvGXOcWUnOqWeBgR5kgWLCJszbXItm7msG5bMXpW0/d2bcXN9/V81I2LPRvamww910c+Ynpjq0ZFpOh6XVynP+7luXOdub819lMt212nLXp+WFa1NPC6TaKVjiAzqaqV3V955ZVJktOnT69cW2kAxbk2We+ztUT/vKMcW8fzIGiG3mg0GgvBiTL0WnCpzuR1E9n5O8wBi6qZKLUAkFlP6dNa9AmzWLdxK/+zGX/XjHwTahv1DcPgr0z2ygPjMMtj3j5lg2BsrsmXiokr2lSLdyVDHiyIoyibC7XEqvGt2UmzlYC1k1NlbDWugJXyx89Ft5K9flbXn/Wh+oWPUiZg7OvGEdo3s0J9sNCOvohFWOJPT7DHugmD8djEImc9qdkrR5HxM6NmRtWyvXzVrM5kyIWs6AjGbjy9PxxHRzB5ulDfFXWD7PnvWiZim43Em6E3Go3GQnCiDH0T0+Vb8vscBTermd3r8tlaHL6Ww8U8N20bZ6ael+hisceNOqvXJcIz+6s5/BgSJuEcs78YBPbqHhhI3UqN/GYryjlH4R/ehOqTruM4b7dWy/nWvgDZ6HPNuRdHION63HxPf9fNzI8D7q19dGBmhVgg5slSc2xdt4FpzkWlkvGcVau6liZO9j7Px43qz15XTK9aUnXbPP52x8mmki1Gp2S7wA3lku8yGwyaoTcajcZCcKIMfRNmxpOszmCV8dScTQwE66o+101+esetW9F1HH7Q/WBTGd9ktJEFwk9YN3moebF1DUDNN8Y81kXtj2qT38NgXS5zLQlcfZV1qzZ9q1ae/2tcZt09D7Kh71Ghllee9dfffOaYd91gum6EXv+vm15XWc0bhp8UM6+4oc3ca8luTJuFa+z5yl2LHOhU3Si9lvk+aoutGXqj0WgsBOecLbNno9FoNLZDM/RGo9FYCPqF3mg0GgtBv9AbjUZjIegXeqPRaCwE/UJvNBqNhaBf6I1Go7EQ9Au90Wg0FoJ+oTcajcZC0C/0RqPRWAj6hd5oNBoLQb/QG41GYyHoF3qj0WgsBP1CbzQajYWgX+iNRqOxEPQLvdFoNBaCfqE3Go3GQtAv9Eaj0VgI+oXeaDQaC0G/0BuNRmMh6Bd6o9FoLAT9Qm80Go2FoF/ojUajsRD0C73RaDQWgn6hNxqNxkLQL/RGo9FYCP4HT+tO+JHhiMQAAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 432x288 with 5 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "with tf.Session() as sess:\n",
+    "    saver.restore(sess, \"./model/my_model_one_at_a_time.ckpt\")\n",
+    "    W1_val = W1.eval()\n",
+    "\n",
+    "for i in range(5):\n",
+    "    plt.subplot(1, 5, i + 1)\n",
+    "    plot_image(W1_val.T[i])\n",
+    "\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Stacked-오토인코더를 이용한 비지도 사전학습\n",
+    "\n",
+    "대부분이 레이블되어 있지 않는 데이터셋이 있을 때, 먼저 전체 데이터를 사용해 stacked-오토인코더를 학습시킨다. 그런 다음 오토인코더의 하위 레이어를 재사용해 분류와 같은 실제 문제를 해결하기 위한 신경망을 만들고 레이블된 데이터를 사용해 학습시킬 수 있다.\n",
+    "\n",
+    "\n",
+    "\n",
+    "![](./images/unsupervised.PNG)\n",
+    "\n",
+    "\n",
+    "\n",
+    "위와 같은 방법을 텐서플로에서 구현할 때는 [Transfer Learning](http://excelsior-cjh.tistory.com/179?category=940399)포스팅에서 살펴본 방법과 같이 구현하면 된다. 이러한 비지도 사전학습 방법에 대한 소스코드는 [여기](https://github.com/rickiepark/handson-ml/blob/master/15_autoencoders.ipynb)에서 확인할 수 있다."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Denoising 오토인코더\n",
+    "\n",
+    "오토인코더가 의미있는 특성(feature)을 학습하도록 제약을 주는 다른 방법은 입력에 노이즈(noise, 잡음)를 추가하고, 노이즈가 없는 원본 입력을 재구성하도록 학습시키는 것이다. 노이즈는 아래의 그림처럼 입력에 [가우시안(Gaussian) 노이즈](https://en.wikipedia.org/wiki/Gaussian_noise)를 추가하거나, 드롭아웃(dropout)처럼 랜덤하게 입력 유닛(노드)를 꺼서 발생 시킬 수 있다.\n",
+    "\n",
+    "\n",
+    "\n",
+    "![denoising-autoencoder](./images/denoising.PNG)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 5.1 텐서플로로 구현하기\n",
+    "\n",
+    "이번에는 텐서플로를 이용해 가우시안 노이즈와 드롭아웃을 이용한 denoising-오토인코더를 구현해보도록 하자. 오토인코더 학습에 사용한 데이터셋은 위에서 부터 다뤘던 MNIST 데이터셋이다."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 5.1.1 Gaussian noise"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reset_graph()\n",
+    "\n",
+    "################\n",
+    "# layer params #\n",
+    "################\n",
+    "noise_level = 1.0\n",
+    "n_inputs = 28 * 28\n",
+    "n_hidden1 = 300\n",
+    "n_hidden2 = 150  # coding units\n",
+    "n_hidden3 = n_hidden1\n",
+    "n_outputs = n_inputs\n",
+    "\n",
+    "################\n",
+    "# train params #\n",
+    "################\n",
+    "learning_rate = 0.01\n",
+    "n_epochs = 5\n",
+    "batch_size = 150\n",
+    "\n",
+    "# denoising autoencoder\n",
+    "inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "# add gaussian noise\n",
+    "inputs_noisy = inputs + noise_level * tf.random_normal(tf.shape(inputs))\n",
+    "\n",
+    "hidden1 = tf.layers.dense(inputs_noisy, n_hidden1, activation=tf.nn.relu, name='hidden1')\n",
+    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name='hidden2')\n",
+    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name='hidden3')\n",
+    "outputs = tf.layers.dense(hidden3, n_outputs, name='outputs')\n",
+    "\n",
+    "# loss \n",
+    "reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)\n",
+    "# optimizer\n",
+    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(reconstruction_loss)\n",
+    "\n",
+    "# saver\n",
+    "saver = tf.train.Saver()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch : 0, Train MSE : 0.04450\n",
+      "epoch : 1, Train MSE : 0.04073\n",
+      "epoch : 2, Train MSE : 0.04273\n",
+      "epoch : 3, Train MSE : 0.04194\n",
+      "epoch : 4, Train MSE : 0.04084\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Train\n",
+    "with tf.Session() as sess:\n",
+    "    tf.global_variables_initializer().run()\n",
+    "    n_batches = len(train_x) // batch_size\n",
+    "    for epoch in range(n_epochs):\n",
+    "        for iteration in range(n_batches):\n",
+    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
+    "            sys.stdout.flush()\n",
+    "            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))\n",
+    "            sess.run(train_op, feed_dict={inputs: batch_x})\n",
+    "        loss_train = reconstruction_loss.eval(feed_dict={inputs: batch_x})\n",
+    "        print('\\repoch : {}, Train MSE : {:.5f}'.format(epoch, loss_train))\n",
+    "    saver.save(sess, './model/my_model_stacked_denoising_gaussian.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from ./model/my_model_stacked_denoising_gaussian.ckpt\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAD+CAYAAACnfntIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF75JREFUeJzt3UmMVcUXx/FCxoZGhkbmoZlkEgwoYVKjLthoNGpYIAuNmkhMNJFEDSYm6kZ3ysao0YCiCxMTp0RRIZIgAQciEEAJUzOICDRNN3Qzw39j6v+rA694/ejX0IfvZ1WVen3vfS/pe3LPuVXV7sKFCwEAAI9uuNoXAABAuRDkAABuEeQAAG4R5AAAbhHkAABuEeQAAG4R5AAAbhHkAABuEeQAAG51aOXzsbzKtaPd1b4AoK2pra3lHlZG7dqlt6XcilxVVVVF3cN4kgMAuEWQAwC4RZADALjV2jU5AEALyNWr2rdvn/TPnj1b8LO2DpYbO3/+fMGxG274/zNTqbvblGNXHJ7kAABuEeQAAG6RrgSAa0guZadjNiWp6UN7DB3LpSft3547dy4Z03SlPb/S1KWlx2gNPMkBANwiyAEA3CLIAQDcoiYHAK1M61K2fqVjHTqkt2jbV7laWq6WZ50+fbrgWMeOHWPbXndumsLlzllOPMkBANwiyAEA3CJdCQBlZl/b17SjHdOU4JkzZ5KxU6dOxbZNAWqa8cSJE8mYphL1+Jc6jqYhKyoqCo6dPHmy4Dm6du2ajOWmReh3tNfWEniSAwC4RZADALhFkAMAuEVNDgBamda27Cv7xS7rZV/Z1+PY1/u7dOkS2507d07GunXrlvQ7deoU27ZGptMb9HMhpDU6O4VBj2OvW4+TW46s1B0KeJIDALhFkAMAuEW6EgDKQFN7NtWmUwFyK4zY1J72c+k7m4LUaQKX24VAU4027ZgbK/Q5e06bStXP2vSo/jalrprCkxwAwC2CHADALYIcAMCtNl+TW7t2bdJftGhRbA8aNCgZ0yVqHnvssWSsd+/el2wDQCm0ZmZ3w9aaXFNTUzJ28ODB2D527Fgydvz48di2S37p1AA7TcAu86XsElxa+zp8+HAy1tjYGNs9e/ZMxqqqqmLb3kMrKytju3v37gWvuxx4kgMAuEWQAwC41a7UWeQlavGTjRkzJulv27atpOP06NEjtqdPn35F11SK6urq2F64cGEyNnTo0HKcMv8eMYCL1NbWFryH5V7NtynJAwcOxHZNTU0ytnXr1tg+cuRIMqav1Nt7t45pWtEeR1OlIVycrtTVUez5GxoaYvvGG29MxkaNGhXbU6ZMScbGjx8f2/3790/GNF1pj6lTJuzUg6qqqqLuYTzJAQDcIsgBANwiyAEA3GrzUwi+/PLLpL9+/frYnjBhQjK2efPm2P7ll1+Ssa+++iq2v//++2Rs+PDhsb1r166ir013/w0hhAEDBsT23r17C/6d1udCCOGll14q+pwAyif3DoNdgiu300BdXV1s25rc/v37Y1trYCGkr/fbpbN0FwCdhhBCWoezuwfY+p3WDw8dOpSM1dbWxra9h+n3HzZsWDKm0x3sdAr9TS+35FgpeJIDALhFkAMAuNXm05Xjxo3L9tWkSZNie+7cucnYm2++Gds2faDpyp07dxZ9bTYtoOlKPWYIaVpg7NixRZ8DQOux6Urt517pt+lKTQnaV+P1lX5b8tBVRuz59JhDhgxJxnSKVK9evZIxXQkqhHTawKZNm5Kx1atXX/J89lrtbgK5jVj17+z0BvvblIInOQCAWwQ5AIBbBDkAgFttvibXUnQpm1xNLFfzuxydtmBX9542bVpsz549u+RzACif3Cvutkamr8rbv9PaWp8+fZIxXebK1sv69u1bcEzPYWt5ukOAfVfAflanBthlvbSepvfMEELo16/fJdshpHVGW5PTvv0NW2JKAU9yAAC3CHIAALdIV5aRXUngoYceim076//tt9+ObZuGAHBtsOkz/T+2aTh9/d3+v2uKMpfm7NatWzKmG47aNKOuhmLvIZoutN/Bvravq5Po6ishpCup2E1TR44cGdujR49OxnTaQu667e+kv2GpO+bwJAcAcIsgBwBwiyAHAHCLmlwZLVmyJOnrbsD6Sm8IF6/aDeDap/UkW+vSGpJdnkrrd5WVlcmY1qzsa/r6Wbujtx7TXktu94L6+vqk/+uvv8b2mjVrkrGjR4/G9sSJE5MxnV5ld//W2qLdrUGvx/5OpdbhFE9yAAC3CHIAALdIV7awHTt2xPaCBQsKfs6mAezjPYC2TVOGNn2oq5rYaQL6WU0zhpBODbBTFuxnCzl27FjSX7t2bdL/+uuvY9vuuqK7GUyZMiUZ0ykEOtUhhPxuDTplQX+XlsKTHADALYIcAMAtghwAwC1qci3sm2++iW3NNYcQwpw5c2J7xIgRrXZNAMrP1pq0Rparl9k6lC5tZZfA0nPYJbB0zNYAdWfyrVu3JmM//PBD0tfdwG3dT+twEyZMSMZ0mS/7fXPTKXRKQW6Js1J3CedJDgDgFkEOAOAWQQ4A4BY1uStk625ffPFFbNtc+xtvvBHbxc5pAdA25Wpr2s/VqHJLhdmanN5TtAYXQgj79u2L7W+//TYZW7FiRdLXZbamTZuWjM2cOTO27XsFdgmyQtdmv29uR3OW9QIAIIMgBwBwi3TlFfrwww+T/qpVq2L70UcfTcaYNgD4ZdOHmoazKbpOnTrFtn1NX3fqtmnHkydPXvIYIaSpvYaGhmTsp59+iu1ly5YlY8ePH0/6mqK86667krFbb701tnv37p2M5VKSud0aclMt7I4JpeBJDgDgFkEOAOAWQQ4A4BY1uWZav3590n/22WeTvi5t8/rrr7fKNQG49pRah9Lanv07rdc1NTUlY1pb27hxYzK2fPny2N6+fXsyNmrUqKR/zz33xPasWbOSMb2/2R3NtbZop1YpW7tUuSkDpU4n4EkOAOAWQQ4A4BbpyiKcOHEitufOnZuM2Vdc582bF9tMGQB8Kza9VmoaLrfyvn31f/PmzbGtUwZCCOH333+PbbsT+YwZM5L+1KlTY7uqqioZ01VNclMYNK1q2e+rq5zoai8hXJyuLQVPcgAAtwhyAAC3CHIAALeoyV2CfcX1vvvui227q+64ceOS/muvvVa+CwNwTdGaUa5+1JxdvHVKga356zJfBw8eTMZ0etOGDRuSscbGxtjWmlsIF08T0CkFdmcBrRHa2prW03LLceV2Vsh9likEAAAYBDkAgFukKy/hyJEjSX/lypUFP7t06dKkb1fmBnB9sGk4fcVepyGFkKbz7Ion2rev1B89ejS2//zzz2Rs9+7dsa3pyRDS3QPuv//+ZMxujNq3b9/YtilC3QXBXpuucpJLLdqNUXO7F+jvlJtOkcOTHADALYIcAMAtghwAwC1qcv+pr6+P7enTpxf83CeffJL0J0+eXLZrAtB22GkCWqOy9TqdCmCXx9LP2nqdHrOuri4Z0xqZndp0++23x/add96ZjPXr1y/pd+7cObbt8ly56Q3Kfl+tw9namvbtb1hqHS45xhUfAQCAaxRBDgDgFunK/yxevDi2d+7cWfBzd9xxR9JviVWyAbR9uZU8bEpSU332dX9NSeqUgRBC2LdvX2xryjOEEHr06BHbgwYNSsZ0p4GBAwcmY7rZqb1umz7Uc+ZSmbkUrJWbTlHqKieKJzkAgFsEOQCAWwQ5AIBb121Nbtu2bUn/1VdfvToXAsAlrSfZJbD01Xi7zNXhw4dju6amJhnbu3dvbDc1NSVjPXv2jG07LaCysjK2bX0sNxXAjul3srU87dtz5KZFtMROAzk8yQEA3CLIAQDcum7TlatWrUr6DQ0NBT+rqwdUVFSU7ZoA+KFpuNymqTZ9p+lLO01A0542zdm9e/eC16J/pzsJ2OsMId0oNbezgk0t5qYC2GtVuRRlbnPZYvEkBwBwiyAHAHCLIAcAcOu6rcnlzJw5M+n/+OOPsU1NDkAxiq0n6TJeIaRTAexuAtXV1QX/TtkaoNbS7HJcuuuAPW6uXmanF+gUgtzyXM2prbXEsok8yQEA3CLIAQDcaleOGeYZrXoyZLF9AtBMtbW1LXIPy63yUWyKzq6ikjumphZz0wKuRCvHklBVVVXUD8WTHADALYIcAMAtghwAwK3WrskBANBqeJIDALhFkAMAuEWQAwC4RZADALhFkAMAuEWQAwC4RZADALhFkAMAuEWQAwC4RZADALhFkAMAuEWQAwC4RZADALhFkAMAuEWQAwC4RZADALhFkAMAuEWQAwC4RZADALjVoZXPd6GVz4fC2l3tCwDampqaGu5hZdSuXXpbunCh8M9dXV1d1D2MJzkAgFsEOQCAWwQ5AIBbrV2TAwCUma1tnT9/PrZzdS77t/Y42r/ccUpRjmPyJAcAcIsgBwBwi3QlAFxDbIpQaTrv3LlzydipU6cK/t2ZM2di+/Tp08mYpjJDCKFz586x3aVLl2SsY8eORV1nzg03tO6zFU9yAAC3CHIAALcIcgAAt67bmtynn36a9BsbG2N73bp1ydj7779f8DivvPJK0r/33ntj++67776CKwTgldbBmrOUldbdjh8/nozV19fHdl1dXTLW0NBwyXYIIVRWVib9vn37xnZVVVUy1qNHj9iuqKgoeN22XtipU6dQiP5dqXW+HJ7kAABuEeQAAG61K8cM84yruoL3M888E9vvvfdeWc4xfvz42P7555+TMX3UvwawCwHQTKXuQmBf01c2taf9I0eOJGN///13bO/cuTMZ0/7GjRuTsZqamoLXMmTIkKQ/ceLE2J48eXIydsstt8R27969kzGdXmDTk+3bt49tO4VApzfoMS6HXQgAANc9ghwAwC2CHADALddTCLQGF0LxdTibh37kkUdie9u2bcnYRx99lPS3bNkS259//nky9uSTTxZ1fgC+2HcftA518uTJZEynBhw4cCAZ++uvv2Lb3ot06pOt1+k5bL2sZ8+eSV9rZlpLCyFd8uvGG28sOGaXGMu9+5GbXlDsMXJ4kgMAuEWQAwC45S5duWfPntj+4IMPCn5u6tSpSX/ZsmWx3bVr12RMH6ft677bt29P+qtXr47tw4cPF3HFADzSV/Vt+i43pn07haC2tja2NeUZQpo+nDRpUjI2YMCA2O7Tp08yZqcC6JSCsWPHJmMDBw6MbbsaiqY2m5qakjG91twUAju9IbcyTLF4kgMAuEWQAwC4RZADALjlriandTD7yqnW4ZYvX56M2ZW4C1myZEnS/+233wp+9sEHHyzqmADaPlsz0nqSXa7q7NmzBce0371792RszJgxsW1rYqNHj47tYcOGJWODBw+ObX3V315LCOn3sOfQ+6Q9jtbabG1N78X2fPpZW69riWUneZIDALhFkAMAuOUuXTllypTYtq/w61QAu+Ffsey0hNOnT5d0HAC+5HYasKlM7dsUnU4F6NAhvUUPHTo0tu2mqV26dIntm266KRnTe59uEB1CurNBCOk0BUvTp/YequewK6Xo1Cs79UHHbHpSfydWPAEAwCDIAQDcIsgBANxyV5NTLbUT99KlS2N7w4YN2c/Onj07tkeOHNki5wdwbdKaka276avyuXqSrclpbcu+pq/1LFv30vqdPebRo0dj++DBg8nYpk2bkr7W5GzdT3cssLsQ6Pnt99V+ru6Wq+WViic5AIBbBDkAgFuu05Wl+uOPP5L+008/Hdt2xXBd3TuEEBYtWhTbdiUDAG1bLtWWW+Ujl67UV/9DSNN+DQ0NyZi+/n/ixIlkTNOc9fX1yZhOE9CdWi7VV/379y94bXZ6g7Krmui90E4hUDY9yRQCAAAyCHIAALcIcgAAt6jJXcKaNWuSvq3Dqfnz5yf9m2++uSzXBODaZmtUuXqdvipva/dal7I1Kq3D2d23tQ63d+/eZOzff/+N7d27dydjtj98+PDY7tWrVzKmUxrskob6fU+ePBmKlfudSt0NXPEkBwBwiyAHAHCLdOV/nnjiidj+7LPPCn7u+eefT/ovvvhi2a4JwLVNX4e3q5NoStKuQKIr+NtVPnSagC2VaBrQrkZy5MiR2LY7CeiOLDY9WVdXl/RHjBgR2/Z1f52KYKc36GooNu2o0xtsWtf+NoXGSl39hCc5AIBbBDkAgFsEOQCAW9dtTc7ms7/77rvYtq+/9uvXL7ZffvnlZExzzQB8s6+0a33J1qF0aoBdkkrrcLkV+3P1Krt0lh4zd1/S3b1DuHgXcV3Ky9bd1q1bF9tdu3ZNxqqrq2Pb7gCj57S7F+jvZH/f3G7rxeJJDgDgFkEOAODWdZuunDNnTtK3Gwmq5557LrZ79+5dtmsC0HblVuvIpd1yuxfY9J1OPdC2/az9O3393qYLtRwTQrriid3pQKcQ6CoqIaTTHfr06ZOMDR48OLbtFAKdemFXf8n9FsXiSQ4A4BZBDgDgFkEOAODWdVWT09dfV65cWfBzDz/8cNJfsGBBuS4JgBO519/t0l3FstMEdOV/uwuA1s9snU9rZBMmTEjGhg0blvR1p3I7nUqvZ//+/cnYsWPHYtt+X6272eXPctML2IUAAIAMghwAwC2CHADALdc1OTvHY+HChbFt89nqtttuS/os3QWgubQulqst2S1ktG/vYfv27Ytt3VonhLReZpcK0x2+ddmuEEIYOHBg0telxOyyXkOHDo3to0ePJmNao8stf2aXEdOanL1uxTw5AAAMghwAwC3X6cp333036a9YsaLgZ3VncKYMAGgum2rTdKXdTUB33La7b+txmpqakrF//vkntnW3b8suP1hZWRnbuoN3CBenSzVFatOluou43clFdyO3Uw80XWn/rm/fvhd/gf+wrBcAABkEOQCAWwQ5AIBbrmtydhfvnLfeeiu2mTIA4EppPckuz6X1OlsT07qXrbvpa/u6jFYI6VJaujSXvRY7LcBOBdB6mh3buXNnbG/fvr3g31n6/e39Vb+//S3stjyl4EkOAOAWQQ4A4JbrdGVz6Gut9nXf5tAVtu1K3PoorrvoWva13UWLFhV1bns+TdfaHXcBtCx739CUpE1Xat+uvqSv4mvqMoQQGhsbY9uueKL3MPuavp5vx44dyZid+qBTGuwUBl1xxe5CkJum0KNHj9i2KUj93exvyC4EAABkEOQAAG4R5AAAblGT+8+gQYNa5Djz58+Pbbu694EDB2L7nXfeaZHz5eh3euqpp8p+PgD/p/Uk+2q81r3q6+uTMa3D2R2+dcV+O73g0KFDsa21sxBC2LJlS8FrsbV8uzu30nqh/dyAAQNie/DgwclYv379Cv6dvsdga3Is6wUAQAZBDgDglut05bx585L+4sWLy35Ou/NBsfS1Wps+UI8//njSnzFjRsHPzpo1q6RrAXDlNA1n0465KUS6IojdYFTTd3aqkaYra2pqkjGdbmBXPNFUYghpSnTIkCHJ2MiRI2Pb7h6gx7Hln2HDhsW2TjUIIZ3eZH+n3L2wWDzJAQDcIsgBANwiyAEA3Gpnl3Qps1Y9mfXxxx/Htl1KJ2fDhg2x3ZxX/1944YWkP2rUqIKffeCBB2I7t1NuC7ry9XKA60xNTU3R9zCts9vX9rVv70W6JJdd2V/raXv27EnGdu3adcnP2fPZa+nfv3/S13par169kjGtF+amAnTr1i0Z075dYlB/J1uTy00bqK6uLuoexpMcAMAtghwAwK3rKl2JBOlKoJmak67UVFtuhwKbktN0Xm5HFJva001U7S4EusKKldtgtaKiouC12evWa82lHXOrmli5MdKVAIDrHkEOAOAWQQ4A4JbrZb0A4GrRupStUWldyu6UrXUwO6Z9O6bLatm6l04bsMuB2SkM2rfLaukO43a381zdrdDn7LWVA09yAAC3CHIAALdIVwJAGeReqc+lMjWdaNOFeszmTEvQlVNsmtFOL9A0qH2FX/u5HQLsd9KpBzY9We5pbDzJAQDcIsgBANwiyAEA3KImBwBloLWm3Gr6uekFdiz3ur2eLzct4HLn15pdbgpDjq0X2jpga+JJDgDgFkEOAOAW6UoAuIpsKq/YlKRl045K06U25ZhLJdppAnqcVt7BpmQ8yQEA3CLIAQDcIsgBANxq7Z3BAQBoNTzJAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDcIsgBANwiyAEA3CLIAQDc+h9L8mbPj/UfPgAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 720x288 with 4 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "show_reconstructed_digits(inputs, outputs, './model/my_model_stacked_denoising_gaussian.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 5.1.2 Dropout"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reset_graph()\n",
+    "\n",
+    "################\n",
+    "# layer params #\n",
+    "################\n",
+    "noise_level = 1.0\n",
+    "n_inputs = 28 * 28\n",
+    "n_hidden1 = 300\n",
+    "n_hidden2 = 150  # coding units\n",
+    "n_hidden3 = n_hidden1\n",
+    "n_outputs = n_inputs\n",
+    "\n",
+    "################\n",
+    "# train params #\n",
+    "################\n",
+    "dropout_rate = 0.3\n",
+    "learning_rate = 0.01\n",
+    "n_epochs = 5\n",
+    "batch_size = 150\n",
+    "\n",
+    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
+    "\n",
+    "# denoising autoencoder\n",
+    "inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "# add dropout\n",
+    "inputs_drop = tf.layers.dropout(inputs, dropout_rate, training=training)\n",
+    "\n",
+    "hidden1 = tf.layers.dense(inputs_drop, n_hidden1, activation=tf.nn.relu, name='hidden1')\n",
+    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name='hidden2')\n",
+    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name='hidden3')\n",
+    "outputs = tf.layers.dense(hidden3, n_outputs, name='outputs')\n",
+    "\n",
+    "# loss \n",
+    "reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)\n",
+    "# optimizer\n",
+    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(reconstruction_loss)\n",
+    "\n",
+    "# saver\n",
+    "saver = tf.train.Saver()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch : 0, Train MSE : 0.03552\n",
+      "epoch : 1, Train MSE : 0.02802\n",
+      "epoch : 2, Train MSE : 0.03202\n",
+      "epoch : 3, Train MSE : 0.03036\n",
+      "epoch : 4, Train MSE : 0.02835\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Train\n",
+    "with tf.Session() as sess:\n",
+    "    tf.global_variables_initializer().run()\n",
+    "    n_batches = len(train_x) // batch_size\n",
+    "    for epoch in range(n_epochs):\n",
+    "        for iteration in range(n_batches):\n",
+    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
+    "            sys.stdout.flush()\n",
+    "            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))\n",
+    "            sess.run(train_op, feed_dict={inputs: batch_x})\n",
+    "        loss_train = reconstruction_loss.eval(feed_dict={inputs: batch_x})\n",
+    "        print('\\repoch : {}, Train MSE : {:.5f}'.format(epoch, loss_train))\n",
+    "    saver.save(sess, './model/my_model_stacked_denoising_dropout.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from ./model/my_model_stacked_denoising_dropout.ckpt\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAD+CAYAAACnfntIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF6xJREFUeJzt3VtsVUUXwPFBbi0tbYEWKHfkJkiUu3hLUBNeTDRqeEAeNGoiMdFEEjWYmKgv+qa8GDUaUPTBxESND2pQAwEFYgg3uba0QLmUttBS2kJpge/FzLdmyR73OZxT6Or/97Qmc87e+zTpnuy19sz0u3btmgMAwKLbbvYFAACQLwxyAACzGOQAAGYxyAEAzGKQAwCYxSAHADCLQQ4AYBaDHADALAY5AIBZA3r4fCyvcuvod7MvAOhtGhsbE+9h/fol/0uxslQ6+m8Y+7tVVFSkuofxJAcAMItBDgBgFoMcAMCsnq7JAYBJ2dbdYrW82Dli9St9LbJ9223hs00mx5Gf1d+T7dj5Y/JRu+RJDgBgFoMcAMAs0pUAkAO5mkKQNrWoz9fd3e3jrq6uxO/1798/6NPnkP2678qVK9e9Tuecu3z5cuK1xf42sTRn2mPE8CQHADCLQQ4AYBaDHADALGpyAJBSJq/bp6W/p2tmkqyJxWpUgwYNSjzHf73eL2trsZpgrE9ep+6L/Q1jvz3bvy9PcgAAsxjkAABmka4EgJRylZKMHUem+nRKULZ1SlC3pViaUX9PpiuvXr0a9A0ePDjxOPI3yekMzoXp04EDB6a+7lzgSQ4AYBaDHADALAY5AIBZ1OQAIM9ir9Tr1+ZlzWrAgORbdGdnZ9CWNTJd95J9upam62e6DpdE1xXlteopDPL3xn6TPmasPpkWT3IAALMY5AAAZpGuBIA8iK3koVOE0sWLF32sdxOQbZ2+Kyoq8rFOOcr05X+9si+nCejUovyuPk4sXSrpv4W81lhaN1s8yQEAzGKQAwCYxSAHADCr19fktm3bFrTXrFnj47FjxwZ9hYWFPn7mmWeCvuHDh183BoA0Yqv56xqcrK01NDQEfceOHfNxS0tL0Hf27NnE88v6VUFBQeL5Ojo6gj59bfK7JSUlQZ9sT5o0KeibPHmyj8vLyxOvLXbdmqzzsQsBAAAKgxwAwKx+2T4CZinnJ5sxY0bQrqqqyuo4paWlPl68ePENXVM25KP/6tWrg74JEybk45Q3/m4u0Mc0NjYG9zB5/9Rpt0uXLvm4ubk56Dt9+rSPt2/fHvRVV1f7+OjRo/r8PpblF30+Geu2nl4Q201ATwUYNWqUjxctWhT0PfHEEz6+++67gz45vSGW1o3t1qCvu6KiItU9jCc5AIBZDHIAALMY5AAAZvX6KQTff/990N61a5eP77zzzqBv3759PtZ58B9++MHHv/zyS9AnX42tra1NfW16SZzKykof19XVJX5Pv5r7xhtvpD4ngJ4j63CxV+Hb2tqC9pkzZ3x87ty5oE8u66Vf4Y+t9C+nCbS3twd9su6la3ByGS/nwikFegrDyZMnfXz48OGg7+DBgz7W7xHIaQn67yRrbXoZM70EWDZ4kgMAmMUgBwAwq9enK2fOnBltS3fddZePly9fHvS9//77Ptav7cp0ZU1NTepr0+kEma6Ux3QufDX4jjvuSH0OALcmmWrTr7/LFKFemWncuHE+1q/UDxkyxMf69X69kolUXFx83di5f6cP5fQGnZLcunVr4jliUxjk749NE8gHnuQAAGYxyAEAzGKQAwCY1etrcrkiX3GN1cRiNb//IqctNDU1BX333HOPj5cuXZr1OQDcHHo1f1nrkssGOhfW0/SUoWHDhvlYv+4vpxB0dnYmnl9PC4jt1K1rcnJKg5zO4JxzO3fuTDyHXGZM74Ig625yOoNz4W/SUwbk79d1zbR4kgMAmMUgBwAwi3RlHulVB+Qq3frR+8MPP/SxXl0cwK1PpxZlW776r9v6/12+4q/TjDLVp9N+MiWo037yfqP79MpMcmNWucKJc861trb6WK9qIqdC6OuWK5no8+ViY9QYnuQAAGYxyAEAzGKQAwCYRU0uj9atWxe06+vrfTxixIigb+LEiT1xSQDyRL+KL9u6DiVrcnqZLflZfcwrV674WL/CL2tt8nPOhbUuvdygru0dOXLEx3pZL1k/1DU5uWu4rk/q60m6bi0XNTqe5AAAZjHIAQDMIl2ZY/JRf9WqVYmf06t5jx49Om/XBKDnydf2dWpRtmOrfOh0nU41Jp1Pk6/068/JHVCcc27Lli0+/vvvv4O+GTNm+Pj2228P+oqKinysf6+87lgqNbaharZ4kgMAmMUgBwAwi0EOAGAWNbkc+/HHH30sl7Jxzrlly5b5WOezAfRuup6kX6OPfVaSNStdk5Pf08eP9cm23rV7z549Qfv333/3cVtbW9A3efJkH0+fPj3okzst6Dqj/B362mJ1t1i9Li2e5AAAZjHIAQDMYpADAJhFTe4G6brbd99952M9V+S9997zcWwpGwC9TyZzvGTdTd9DJL0cmDymnm8ma116Pp38ntxKxznnfvvtt6BdW1vr4ylTpgR9M2fO9HF5eXnitcbub7pPLiumf2+2dTiJJzkAgFkMcgAAs0hX3qDPP/88aG/evNnHTz/9dNDHtAHAllg6LbaCftpV+bu7u7O6Fn1dHR0dPv7zzz+DPjllwLlwea45c+YEffIeVlBQEPTFpkzE0qyynY8yDk9yAACzGOQAAGYxyAEAzKIml6Fdu3YF7Zdffjlol5WV+fjdd9/tkWsC0DNiu39rsSWp5LSB2BQCPRVAvmIfW/JLL921f/9+H8ulB51zrrm5OWjLOtyiRYuCvoqKiuuez7mwJherrWWy1U4u8CQHADCLQQ4AYBbpyhQuXrzo4+XLlwd9+tF7xYoVPmbKAGBLbFqAfoVepuz06icyRan70q7KL3f7di5M9TU0NAR9mzZt8rHc+du5f08FWLhwoY/nz58f9A0dOtTHekWnWCpV3if174vthC5/U+xvH8OTHADALAY5AIBZDHIAALOoyV2Hzhk/+uijPj506FDQJ1flds65d955J38XBuCWIutE+r4Rex1eTg3Qr/vL78kV+vU59Ir9nZ2dPj5w4EDQt2HDBh+fPHky6FuyZEnQXrBggY/HjBkT9MXqZ7FaYtIxnAtrl9nW3aLny/kRAQC4RTDIAQDMIl15HefOnQvaGzduTPzs+vXrg/bw4cPzcUkAbnGxqQA6Radfv5dk2lGTqUz9uaamJh9v3bo16Nu7d6+P9WanixcvDtqzZs3ycWlpadAnU6ux3QRim5/G0po6xZuL1VB4kgMAmMUgBwAwi0EOAGAWNbl/nD9/3sc6Ry199dVXQXvu3Ll5uyYAvYeuu8VqcrJP1+fkZ/XO4PJ7bW1tQZ+cNnD48OGgT55j3rx5Qd8jjzwStOV7BXrJL3k9urYmpwLoel3a5bnSLmmWCZ7kAABmMcgBAMwiXfmPtWvX+rimpibxcw888EDQzscmfwB6v9iq/LG0XGwFkPb2dh+fOnUq6NuzZ4+P6+vrg77x48f7WJdYKisrg7bc3UCnHSV9bXJnBb1pqkzBdnR0BH0ylaq/l4v7K09yAACzGOQAAGYxyAEAzOqzNbmqqqqg/fbbb9+cCwHQa6WtGekpBHJ3AT2FQNbr9A4Fzc3NPq6urg76WlpafCx38HbOuVGjRvl42rRpQZ+ug8k6nL7uWF9smoBs69+rz5/0PZb1AgBAYZADAJjVZ9OVmzdvDtqtra2Jn5UboxYWFubtmgD0LrFVOGIr78vX9HWfTAPqnQYaGxt9fOLEiaDvzJkzPtYpQZmu1OfT9z6Z6iwqKkq8tthv16lM+Vndl288yQEAzGKQAwCYxSAHADCrz9bkYu67776gvWHDBh9TkwP6rkx2rk5br4spKSkJ2pMnT/axXnJrxIgRPh4yZEjQN3XqVB8XFxcHfbota3axemFsmoAW+71pdyhgFwIAABQGOQCAWf2yfQTMUo+eDFFsnwBkqLGxMSf3sLQpupjYDgE6PRh7bT9tKlHLR2oxExUVFanuYTzJAQDMYpADAJjFIAcAMKuna3IAAPQYnuQAAGYxyAEAzGKQAwCYxSAHADCLQQ4AYBaDHADALAY5AIBZDHIAALMY5AAAZjHIAQDMYpADAJjFIAcAMItBDgBgFoMcAMAsBjkAgFkMcgAAsxjkAABmMcgBAMxikAMAmDWgh893rYfPh2T9bvYFAL1NS0tL6ntYv37//xe7di3/t76beb4bOWe2111WVpbqHsaTHADALAY5AIBZDHIAALN6uiYHAH1C2vpSrmpb2R5TfzYX15LJ+dNeW7bXwpMcAMAsBjkAgFmkKwHgJsokDXfbbf9/LtEpwStXriR+7+rVq4mf08cZOHDgdc+nDRgQDh/d3d0+zlV6VLZj34vhSQ4AYBaDHADALAY5AIBZfbYm9/XXXwft9vZ2H+/YsSPo+/TTTxOP89ZbbwXthx9+2MdLliy5gSsEcKvLx1JasmYma2nOhXWvzs7OoK+jo8PH8n6m25cuXQr6hgwZErRLSkoS+4qLi308aNCg6/8A9+/6WdpaYv/+/YM++TdlCgEAAAqDHADArH49sVq1cFN3IXjppZd8/Mknn+TlHLNmzfLxli1bgr7S0tK8nDNL7EIAZCiTXQjSunz5ctCWacdTp04FfXV1dT6urq4O+o4dO5bYd/jwYR/rNOP48eODtryHPfjgg0HfnDlzfDxy5MigTx43Np0h26kAGrsQAAD6PAY5AIBZDHIAALNMTyGQNTjn0tfh5s6dG7SfeuopH1dVVQV9X3zxRdDev3+/j7/99tug7/nnn091fgC9TyYr78vX+C9cuBD0yTqcrKU5F9baampqgr5Dhw5d9xjOOXf+/Hkfy2kIzoV1Pn3dEydODPpmz57t44KCgqBPLgempz7ElhWTfbG/Ict6AQCgMMgBAMwyl648fvy4jz/77LPEzy1cuDBo//zzzz7Ws/xjr8bqV3X/+OMPHzc1NaW4YgAW6PSkTK/p+0ZXV5eP5ZQB58JU5sWLF4O+wYMH+7iysjLoGzNmTOK1yRVPGhsbg762tragXV5e7mOdIpRtvUOBvDY9LSJt2jGWrmTFEwAAFAY5AIBZDHIAALPM1eRkHUzncGUd7tdffw365OraMevWrQvaf/31V+JnH3/88VTHBNA7Zftau1xtX++wLfsmTJgQ9I0dO9bH+t2BoqIiH8v6mHNh3a22tjboO3HiRNA+ffq0j/VuBi0tLT7WuxnEdiWITROQv19PPcgFnuQAAGYxyAEAzDKXrpw3b56P9Sv88nG6sLAwq+PraQn6VVkAfVPsdXtNpuh0mm/UqFE+HjduXOL3ZHpSn09OUXAuTC0OGzYs6NNlnebmZh+3trYGfQ0NDT7WqVSddo2dQ5LXHfsbMoUAAACFQQ4AYBaDHADALHM1OSlXO3GvX7/ex7t3745+dunSpT6eMmVKTs4P4Nana0ax1+FjUwhKSkp8rOt1cmqA7pPTBPQyYvJ8+j0CvSuBrIPJ7zkX/ia9HJjclUAv+SWPEzumrsnF+tLiSQ4AYBaDHADALNPpymzt3LkzaL/44os+7uzsDPr0SuBr1qzxsdxEEEDvF0uZZZKulJ/Vm4/KlGQmK/3Ltk4lypVL6uvrgz65oaq+br0SlJyKoFc8STtNQP8meT59DPlZphAAAKAwyAEAzGKQAwCYRU3uOrZu3Rq0dR1OWrlyZdCePn16Xq4JwM0X2/1b1+BkPUm/0i/bsV0IdF1f17MkuYv40aNHg77q6mof61qartHF6mdlZWU+llMdnAtri5ns8B37O+UCT3IAALMY5AAAZpGu/Mdzzz3n42+++Sbxc6+++mrQfv311/N2TQB6D53ak229K4Dsi6Ur9aomMiWq0451dXU+3rZtW9AnN0KNTQtwLtyMVV/b0KFDfax3coltfhrbISHfU614kgMAmMUgBwAwi0EOAGBWn63J6WVvfvrpJx/rHLXcqffNN98M+nTOHEDfIV+Hj702r8n7hl6VX9a2dJ1P7higd+0+cOCAj0+dOhX0yRqZrrPJ+5tz4f1Pn7+lpcXH+nX/2O4FSdeizxE7JrsQAACgMMgBAMzqs+nKZcuWBe2GhobEz77yyis+Hj58eN6uCUDvFVsNRafaZIoutoqJ3mlArmpSVVUV9O3Zs8fH+n4mU5JyGoBz/54KINOJOiUqV1KZNGlS0Cc3qY6VcWJpx0x2eUiLJzkAgFkMcgAAsxjkAABm9ama3I4dO3y8cePGxM89+eSTQXvVqlX5uiQAvZisIemakWzr1/ZjZI1O74Ail+7atGlT0Cd3GqioqAj6Zs6c6eM5c+Ykns+5cCrCvn37gr7jx4/7+NChQ0FfeXm5j/W7C/JvEatBZjINIy2e5AAAZjHIAQDMYpADAJhluiYn55Q459zq1at9rOefSPPnzw/aLN0F4Hpiy3rJ+WZ6mStZl9J98pi6JifrbrW1tYnHnDt3btD30EMP+XjcuHFBn54L197e7uNjx44FfXL+nawPOudcU1OTj+V2Pc6F2+nov5NcyisXNTiNJzkAgFkMcgAAs0ynKz/++OOg/dtvvyV+Vu4MzpQBAJmK7Yat03Dys7p0IncBkK/sOxcu3dXR0RH0zZ4928eLFi0K+saOHetjPZ1Br/wvU4t6CTCZrjxx4kTQV1NT4+OioqKgTy75FUtX6mtLu/xZDE9yAACzGOQAAGYxyAEAzDJdk9O7eMd88MEHPmbKAIBc0nUvWaOTNTjnnDt58qSPt2/fHvTJrW70a/pyKS1d25I7end1dQV9urZ37tw5H7e1tQV9si4mj+lcuMyXnhYxceJEH+s6n6zR6VpetnW44Bg3fAQAAG5RDHIAALNMpyszIR/Lb+QRefDgwT7Wj+wyZaFXMpD0Si1r1qxJdW59Ppmula8FA8iN2O7fkr6nyM9euHAh6JMrmeiV/mUqU/9Pnz9/3sc7d+4M+kaPHu3j4uLioE+ff+/evT7W6Uo9TUKSK57I63TOuYKCgsRjyLSrTsHmAk9yAACzGOQAAGYxyAEAzKIm9w+57M2NWLlypY/HjBkT9NXX1/v4o48+ysn5YuRveuGFF/J+PqCvie1CEKvty+/pKUuyr7u7O+iT9Sxdu5fLap0+fTroKysr87FcYss555qbm4O2rAnK3b6dc66wsDDxumXdTU9hkFMD9N9Ffla/VxCrc6bFkxwAwCwGOQCAWabTlStWrAjaa9euzfs59c4HacUe2aVnn302aN97772Jn73//vuzuhYANy62Aaj8H9crgEybNs3HOu0oVyc5cuRI0Cd3LJCrljgX3l9kytG5f6cr5copeoNVORVBTpdyLizPyN/gXDhtQaY1nQunQuhpEbGdHNLiSQ4AYBaDHADALAY5AIBZ/bLNc2apR0+mffnllz7Wu/HG7N6928eZvPr/2muvBe2pU6cmfvaxxx7z8ciRI1Of4wbc+Lu5QB/T0tKSeA/Tr7vLe6t+pV7W5PT35K4EevdtueSW3CXcOecOHjzo47q6uqBP1v30+fRUAHmfWrBgQdAna3LDhg1LfQ75+/VOA/Jvoa8lNoWgrKws1T2MJzkAgFkMcgAAs/pUuhIB0pVAhjJJV8b6ZIpOTxmSr9HrFftbW1t9fPbs2aBP7gKgNzSVx9Q7oOj0YWVlpY9HjBgR9MlpA3rskLus6OuW6UrdJ9t6NZTYdKrS0lLSlQCAvo1BDgBgFoMcAMAsanJ9FzU5IEO5qsnJ+66uQ8m2nnoQI78X2xEhtrOBvjY9PsjPxnYJj4mdL5NdB5hCAADo8xjkAABmmd6FAAB6ik7tpV1BP5YSbG9vD/rkRqn69Xq5u4BOgcpj6vPpz8qpALHfpFOLsq1Tkmn78oEnOQCAWQxyAACzGOQAAGZRkwOAHIhNE4h9Vn8utnuBXKVf76TS1dXlY73DtqRrcLHz/9d3k8TqbPoYudj9O4YnOQCAWQxyAACzSFcCwC0klrKTaUidkoyl/bJNH2ZyndmmGvO96hZPcgAAsxjkAABmMcgBAMzq6V0IAADoMTzJAQDMYpADAJjFIAcAMItBDgBgFoMcAMAsBjkAgFkMcgAAsxjkAABmMcgBAMxikAMAmMUgBwAwi0EOAGAWgxwAwCwGOQCAWQxyAACzGOQAAGYxyAEAzGKQAwCYxSAHADCLQQ4AYBaDHADALAY5AIBZDHIAALMY5AAAZjHIAQDM+h//dkxVIZ1jcQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 720x288 with 4 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "show_reconstructed_digits(inputs, outputs, './model/my_model_stacked_denoising_dropout.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Sparse 오토인코더\n",
+    "\n",
+    "오토인코더가 좋은 특성을 추출하도록 만드는 다른 제약 방법은 **희소성**(sparsity)를 이용하는 것인데, 이러한 오토인코더를 Sparse Autoencoder라고 한다. 이 방법은 손실함수에 적절한 항을 추가하여 오토인코더가 코딩층(coding layer, 가운데 층)에서 활성화되는 뉴런 수를 감소시키는 것이다. 예를들어 코딩층에서 평균적으로 5% 뉴런만 활성화되도록 만들어 주게 되면, 오토인코더는 5%의 뉴런을 조합하여 입력을 재구성해야하기 때문에 유용한 특성을 표현하게 된다.\n",
+    "\n",
+    "이러한 Sparse-오토인코더를 만들기 위해서는 먼저 학습 단계에서 코딩층의 실제 sparse(희소) 정도를 측정해야 하는데, 전체 학습 배치(batch)에 대해 코딩층의 평균적인 활성화를 계산한다. 배치의 크기는 너무 작지 않게 설정 해준다. \n",
+    "\n",
+    "위에서 각 뉴런에 대한 평균 활성화 정도를 계산하여 구하고, 손실함수에 **희소 손실**(sparsity loss)를 추가하여 뉴런이 크게 활성화 되지 않도록 규제할 수 있다.  예를들어 한 뉴런의 평균 활성화가 `0.3`이고 목표 희소 정도가 `0.1`이라면, 이 뉴런은 **덜** 활성화 되도록 해야한다. 희소 손실을 구하는 간단한 방법으로는 제곱 오차 $(0.3 - 0.1)^{2}$를 추가하는 방법이 있다. 하지만, Sparse-오토인코더에서는 아래의 그래프 처럼 MSE보다 더 경사가 급한 쿨백 라이블러 발산(KL-divergense, Kullback-Leibler divergense)을 사용한다."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 88,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEPCAYAAABV6CMBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcjXX7wPHPZd+VXSapbIlSDaVUlEqb4lFJSaW0UUqr9pLlV2mhTQqFKGtSkjURzWCeh0jJFlkGWWaYzPL9/XHNjJkxzHbOuc9yvV8vL+6z3PflZs51vtv1FeccxhhjTH4V8zoAY4wxocUShzHGmAKxxGGMMaZALHEYY4wpEEscxhhjCsQShzHGmAKxxGGMMaZALHEYY4wpEEscxhhjCqSE1wEUVbVq1Vy9evW8DsMY40erV+vvTZp4G0c4WbZs2S7nXPXCvDfkE0e9evWIjY31OgxjjJ8sWABt2sCIEdCjh9fRhA8R2VTY91pXlTEmqA0dClWqQNeuXkdiMljiMMYErb/+gqlT4Z57oGxZr6MxGSxxGGOC1gcfgHPw4INeR2KyCvkxjtwkJyezZcsWkpKSvA7F+FGZMmWIioqiZMmSXodi/CApCT7+GDp0gFNO8Toak1VYJo4tW7ZQsWJF6tWrh4h4HY7xA+ccu3fvZsuWLZx66qleh2P8YPx42LULevf2OhKTU1h2VSUlJVG1alVLGmFMRKhataq1KsOUczBkCDRtCm3beh2NySksWxyAJY0IYP/G4Wv2bFi5EkaOBPtnDj5h2eIIB4cPH2bTpoJNs966dWu+17RMmzatMGH59HwPP/xwvl7XvXv3Ap/bhLY33oBateDWW72OxOTGEocf3XrrrbRr1y7br6ioKDZs2JD5mvbt2x/15/bt2/P333/z6quv5nreVatWZTtn9eq6+POPP/5g5syZ2V777bffcuGFF3LRRRcxatSozMeHDh2a7XWjRo2iSZMmtGnTJvPXmWeeyYgRIzJfs3///mzXrVq16jHPl1XWv+P8+fMZNGgQAL///nuur7/uuuuyHW/duvWY5zbhZ+VKmDULHn4YSpf2OhqTm7DtqvLa6tWrSUxMRERwzlGiRAlKlixJ8eLFKV68eObrEhMTGTNmDADbtm3L17mbNm3K7NmzM4/btWuX6+sSExMZMGAAP/zwA6VLl+baa6/lsssuo27durm+/p577uGaa67JPJ41a1a25ytVqpSv6+a0Z88e+vTpA+jEhejoaEAHuH/66ScaNGhAzZo1AUhISCAhISFf5zXhacgQKFcO7rvP60jMsVji8JOGDRsycuRIKlasSKlSpTIf79q1K+XLl888FhEqVKgAkJlQ4uLi6NKlC02bNs313CkpKezduzfzODk5OdfXLV++nGuuuSbzet27d2fevHm5dv20atWKefPmMX/+/MzHSpUqRevWrTOPM2Yy5XXdnKpUqcIbb7wBwIIFC4iJick835YtWzjppJMyXztv3jx+/fVXdu7cSY0aNTJfN378eFq0aMHpp5+er2ua0PT33zB2rCaNKlW8jsYcS9gnjj59IC7Ot+ds3hzefvvYz8+ZM4eBAwdmHm/evJny5ctndu3ccsstPPnkk1x55ZWULl068xt45cqV08/fnA8//JD+/fvnev4lS5bw2GOP0bJly8zXZxg3bhyxsbF88sknHDhwIFt3UpUqVXLtHrrtttuO2R00fvx4atWqxfjx49mxYwdt27bl8ssvB6BZs2ZMmzaNkSNHsmrVqmPej8suu4ynn3468/jaa68FoFixYnTp0iXz8f379zNs2DCmTJlC7969GTNmTOYajRIlSthgeAQYNgxSUvTn1gSvsE8cXrj88su5/PLLWbduHSkpKYwcOZIGDRrQunVrihUrRsOGDTNfe9FFF2UmiEaNGuX7Gtdccw0vvfTSUY937dqV5557DoBTTz01W9fSb7/9Rv369Y96z9ixY0lISMjWislwwgknZLaIAM4//3yGDRuW7TU33HDDcbutnnzyScaNG8fmzZsBWLp0KUuXLs32ni1bttCjRw8GDRrEOeecQ0pKCrfffjvjx49HROjcufMxz2/CQ0KCrhTv1AmsYRncwj5xHK9l4G9z5szh0KFD1K5dm4SEBGbOnMmIESOyfTt/6aWXGDZsGBMnTgSgTZs2bNq0idtuuy2zRZEf//7771GPnXHGGWzatIk5c+ZQuXJlvv76a77//vtc3//7778zd+7cbI+tWbOGZs2aZY5PFEXLli1pkqMmdq9evXj88ccBqFmzJp9++mlm91SbNm249NJLrZURQUaOhL17oW9fryMxeQn7xOGlWbNmsW/fvmyPZR0jyNCrVy969eqV7bGNGzces6uqUqVKLF26lNtuuw3QcZLKlStz0003HfXa0aNH8/7773Po0CHGjRt3zPIc69ev55tvvjkq1mbNmmUelyxZkvXr13PLLbdQqlQpypQpQ8WKFRkyZEiu58xq2LBhxOXoM1y3bl22c9epU4d27dpltpIykkbWVpMJT6mp+iWvVSv9ZYKbJQ4/SkxM9MuH3llnncV333131ONZB7YzVKhQgSeffDLPc27ZsoWnn34629TZnKpWrcqPP/5YoFgz/Pbbb7nGZwzAxImwfj383/95HYnJD0scfvT777/n2vc/dOhQzjjjjOO+t0SJElSrVs1foeXqySefzFxjkaF169bHbPkURHJycq734u233z5q9libNm2Oet3AgQNpZV9Fw5JzMHAgNG4MHTt6HY3JD3HOeR1DkURHR7ucq6XXrFmT5wezCQ/2bx36vv0Wrr0WRo0CKxIQOCKyzDkXXZj32spxY4xnnIPXXoO6dW2Hv1BiXVXGGM8sXAiLF+v2sLatSuiwFocxxjMDB0KNGtCjh9eRmIKwxGGM8cTy5TBzJjz6qO0nHmoscRhjPDFwIFSqBA884HUkpqAscfiJiBy11uLQoUPUrFkzcz3DqFGjaNu2LRdffHFm/aZRo0bRsGHDzNLmK1euDHToxvjd2rUwaRL06gXpJdpMCLHBcT9p0KABw4YN4+qrr858bOTIkZnlw/fv38/HH3/MwoULKVasWLaSIY899hj3339/wGM2JlAGD4YyZeCRR7yOxBRGZCSOXBaUcfPN8OCDcPAgZNmDItOdd+qvXbsgZ4G9fKyArlKlCnXq1CEuLo7mzZuTmprKpEmTuOqqqwAoW7YsxYoVY9WqVZx11lmUth1rTIRYvx4+/1x//NJLk5kQY11VftS3b1/eeustACZOnMgNN9yQuedGyZIlmTJlCh999BF33XVXtrLmQ4YMyeyqOnz4sCexG+MvAwZA8eLw1FNeR2IKKzJaHMdrIZQrd/znq1XLVwsjN40aNSIpKYktW7YwYsQIpk6dmm072GrVqvHee++xfv16br755szqtNZVZcLV+vW6QvyhhyDL/l0mxFiLw8/69OnDHXfcwQUXXJBt57/ExMTMSrmnnXYaZcqUYf/+/V6FaUxAvPaaLvTLsq+XCUGR0eLwUKtWrShRogS9e/fO9nhiYiLXX3995l7kHTp0oHr16oB2VY0fPx7Q1keHDh0CHrcxvrZ+PYwerTOpatf2OhpTFAEtcigibYHBQAqwE7jbObcny/NRwPtAOaA80N85N+N457Qih5HN/q1DR48eMG6cJhBLHB5avhy+/hp5+eXgL3IoImWAD4HOzrkLgQXAgBwvexMY75xrB9wAfCIixQMVozHGP/78U1sb991nScNzZ54JyclFOkUgxziuAhY55zanH48AcvbBbAVqpf+5KrALSAtMeMYYf8kY27CZVB5KTIQ9e6B0af0HKYJAJo56wPqMA+fcAaCEiGStifkq8ICIrAZigF4ul740EekpIrEiEhsfH5/rxUJ9nxGTN/s3Dg1//gmffQb332+tDc+kpcEdd+i+vElJRT5dIBNHaXRsI6sUIOtP/0TgUedcE+AM4G0RqZvzRM654c65aOdcdMaAclZlypRh9+7d9sESxpxz7N69mzJlyngdisnDK69oayMfOxgbf3npJZg8WbO3D35mAjmragvQOuNARMoDSc65lPTjasApzrlvAJxzf4nIHOBKtFsr36KiotiyZQvHao2Y8FCmTBmioqK8DsMcx6+/6irxxx+31oZnJkyAV1+Fu++GPn18cspAJo7vgP4iUtM5twPoCYzL8vxuABFp5pxbKSIVgMuBCQW9UMmSJTn11FN9EbMxpgiefx4qVrSxDc8sW6alk1q3hvffBxGfnDZgicM594+I9Aami0gaOt5xr4jcDexwzs0Qkc7AWyJSAp2SO8w590ugYjTG+M4vv8CUKdpVVbWq19FEqKgouOEGePddHRT3kYCu4/CH3NZxGGO8164d/O9/OjhesaLX0USYpCQoUUJ/HYOIBP86DmNM5JgzR3/162dJI+Cc0/GMa6+F1FS/XMIShzHGp5zThHHyyTqJxwTYwIHwxRe6nURx/6yftlpVxhifmjZNxzc++cQnMz9NQUyZAs8+C127+rWSpI1xGGN8JjUVzjpLf1+16rhd7MbXVqyAiy/WkiLz50PZssd9eVHGOOyf1RjjM599BqtXw5dfWtIIOBFo1kwX+uWRNIp8KWtxGGN84eBBaNBAxzZ+/tlnSwZMXpKTdWk+6ABTPm+8zaoyxnhuyBD4+2944w1LGgGTmgr/+Y8uzYeA3XhLHMaYItuxAwYPho4ddZGyCZC+fWH6dAhwpQxLHMaYInvpJV1zNmiQ15FEkPfeg3fegUce0U3cA8gShzGmSH77DT7+WNdsNGzodTQR4ttv4eGH4frr4c03A355SxzGmCJ56ikoXx5eeMHrSCJIUpLurTFunN8W+R2PJQ5jTKEtWABffw3PPAO5bI1jfC2jhEinTrBwIVSo4EkYljiMMYWSlqaTeU4+WbvZjZ8lJurMg88+02MPp65Z4jDGFMro0RAbCwMG+H29mUlNhdtu01ouVap4HY2tHDfGFNz+/do91aqVfp4ZP3v8cS0C9u67cN11XkdjicMYU3D9++vajW++scV+fvfWW/D229of2Lu319EA1lVljCmg33/Xz7G77oLoQhWsMAWyf7+uDvdg2u2xWIvDGFMgfftqufQBA7yOJMylpupU2xdf1JkIxYLne37wRGKMCXozZ2r31PPPQ61aXkcTxtasgSZNICZGj4MoaYC1OIwx+XT4MPTpoxVwbfqtH/39N7RvD//+C9WqeR1NrixxGGPy5b33YO1abXGUKuV1NGHqwAHdK3z3bl1dGeDihfllicMYk6edO7WQYfv2cM01XkcTppKToXNnWLlSs/N553kd0TEFV8eZMSYoPfEEHDqkM0Nt+q2fpKRAuXIwfLhm6CBmLQ5jzHEtWKBVLp55Bho39jqaMHX4sC6/nzw5JDKztTiMMceUnAwPPginnALPPed1NGHqo4+gRQuIjw+JpAGWOIwxx/HWW7B6NQwdqr0oxscmToQHHtBKkSec4HU0+WaJwxiTq82b4eWXoUMH3S/I+NjcuVroq1Ur+PJLKFnS64jyzRKHMSZXffqAc7o7qfGxFSvgxht1Ucz06SHXnLPEYYw5yowZMGWK7upXr57X0YSh6tXh4ovh+++Dokx6QYlzzusYiiQ6OtrFxsZ6HYYxYePgQWjaFEqXhv/+1xb7+dSePVC5sifbveYkIsucc4UqU2ktDmNMNgMGwIYN8P77ljR8au9eaNsWevTwOpIis8RhjMm0ciUMHgzduulnnPGRQ4fghhu0eGEY7HxlCwCNMYBW8b7nHjjxRJ2Ga3wkJQW6doWFC2HcOLjiCq8jKjJLHMYYQNdq/PKLfrZVrep1NGHk0Udh6lTd9rVLF6+j8QlLHMYYNm6EZ5/VAoZh8tkWPLp0gZNOCpptX33BEocxEc45uP9+rXbxwQchU/Ui+C1bphVuL7pIf4WRgA6Oi0hbEflFRBaLyFQROWoCs4jcKyIrRGShiHwWyPiMiURjx+pygoEDoW5dr6MJE++8oxuyz5jhdSR+EbAWh4iUAT4ErnDObRaRR4EBwP1ZXnM9cBNwkXPuoIh99zHGn+LjdYV4q1ZazND4wKef6k3t1AmuusrraPwikC2Oq4BFzrnN6ccjgA45XvMs8JBz7iCAC/XVicYEuT59YP9++PjjoFiTFvq++gruvReuvFJnGZQIz9GAQCaOesD6jAPn3AGghIiUBBCREsBJwHUiMk9E5orI1bmdSER6ikisiMTGx8cHIHRjws+0afrZ1q8fnHmm19GEgY0bjxQtnDxZl96HqUAmjtJASo7HUoCMVkU1oAbwp3OuLdAd+EBETsp5IufccOdctHMuunr16v6M2ZiwtHs33HcfnH22Jg7jA/Xq6YDRjBlQvrzX0fhVINtRW4DWGQciUh5Ics5lJJNdQIJz7msA59xfIhIDnAH8HcA4jQl7vXtr8vj+eysrUmSxsboy/OKL4aabvI4mIALZ4vgOaC8iNdOPewLjMp5MTyA/icg1ACJSFWgGrAxgjMaEvUmT4IsvtPLt2Wd7HU2I+/VXHQB/4AFdeh8hAtbicM79IyK9gekikoaOd9wrIncDO5xzM4AHgOEi8mR6bH2cczsDFaMx4S4+Xj/jzj0Xnn7a62hC3Nq1cPnlOpYxbVpEzS4I6JB/enLIObH50yzPbwNsrzFj/MA5nXK7d69uPhdCG84Fn3Xr4LLL9KbOmQOnn+51RAEVnnPFjDFH+fJL3eJ6wADdb8MUwQcfwL//wrx5cMYZXkcTcLaRkzERYPt2TRannQaLF4ft8oLASU2FTZv0hoYo28jJGHNMzuneQQkJMGqUJY1C+/tvuPZa2LJFxzNCOGkUlf0XMibMffghfPutlk9q0sTraELUjh06EL5lC2zdClFRXkfkKUscxoSx336Dvn11xmivXl5HE6Li4zVpbN4MM2fC+ed7HZHnLHEYE6YOH9YKGOXKad29YtYxXXB79uiOfX/+qSvCL77Y64iCgiUOY8LUSy/B8uVaNumkowr3mHxJTdV1GlOn6vRbA1jiMCYsLVwIgwbpoHjHjl5HE4L27YMyZaB6dfj5Z2uu5WB3w5gws28fdOumk37eftvraELQnj06ptGtmx5b0jiKz+6IiLQTkfm5PF5JRB4Wke/Sd/aLS//zIyJS0VfXN8bo1NuHHtLJP2PGQIUKXkcUYnbvhnbtYOVKuPNOr6MJWvlKHCLSWEQ25vL4RhFpfJz3FQN+BE4D+gAXAq3S/3wKWtQwcgq8GONno0ZpZe8XXoALLvA6mhCza5e2NFav1tpT11zjdURBy99jHKcCDYBzcuzmtxZ4TET2A/XTj40xRbBmjU65bdsWnn3W62hCjHPQubMWLvz6a93BzxyTvxPHX0A80FdEPnTOJQCISAXgPmAfsNHPMRgT9g4dgptv1v2DxoyJqEKtviECb7yhFSDbtfM6mqBXkMRRTURG5XzseG9wzh0WkUuApUAvEUkFSqFdZCWBls65fwsQgzEmF48+CqtWwXff2dTbAtmxQ7ulevaE6EKVbYpIBUkcicD4HI/luid4Vs65zSKyG7jdORcHICLNgfHOuY0FuL4xJhdffQUffQRPPAHt23sdTQjZvl3XZmzapDeubl2vIwoZBUkch5xzM7M+ICKHcnuhiJwIDM3yUBTwmoj8k358IlBbRMZkeU0f59yuAsRjTMRbvx7uuUerYLz2mtfRhJCtW3Ug/K+/tJCXJY0C8dcYx0FgVJbjUbm85q0cxwl+isWYsHT4MHTpot3z48fbxkz5tmGDJo34eO3bu+QSryMKOQVJHOVE5Macj+X2wvRxi9mg6ziAO9FurVqAANuA74GRzrl9BYzZGAP06wcxMbo5U716XkcTQpYs0VWSc+ZAy5ZeRxOS8ps4/gEmAzl7UCenP5crESkJLAbmAX2BDWjiqAfcD/wsIs2dc4cLFrYxkW3yZHjzTV3s95//eB1NiEhK0jIit96qYxonnuh1RCErX4nDObcD/aAvqCZATedc7xyPrwYeFpHtQDNgWSHObUxEWrtWFzWff74mD5MPMTFw4406V7ltW0saReTvIizrgBQReUhEymY8KCJlRSQjEdniP2PyKTFRWxilSulsqtKlvY4oBCxcqGMapUtbn56P+HUBoHMuUURaA48Dy9NLkAiQCiwAWmcsCjTGHJ9zutxg9Wr4/ns4+WSvIwoBP/wAN9ygs6Zmz474nft8xZeJYwlwT84HnXN/Ag/48DrGRKT334dx46B/f91byOQhLg6uuw4aN4ZZs6BmTa8jChs+66pyziU459b56nzGmCOWLNHV4ddeC88843U0IeKss7Ta47x5ljR8zArNGxPk4uO1/l5UFHz+uW0PkadPP4WNG/VGPfssVKnidURhx/4LGhPEUlJ0kd+uXTBpkk0GOi7n4OWXddvDIUO8jias2daxxgSxxx+HuXN1n41zzvE6miCWlgaPPALDhulcZUscfmUtDmOC1MiR8M470KcPdO/udTRB7PBhuP12TRqPP65dVSXsO7E/WeIwJgj9/DPcf79uDfH6615HE+T+/Rf++AMGD9abJeJ1RGHP0rIxQWbrVujUSQfDJ0ywL8/HtGePLuqrWBEWLdJVkSYgrMVhTBBJSoKOHSEhQXcwtQlBx7B1K1x8Mdx2mx5b0ggo+y5jTJDIWBkeEwNTpsCZZ3odUZD6/XfdE3zPHh3XMAFnicOYIPHWW7pO4+WXtR6fycXixdChg67RmD8fzj3X64giknVVGRMEpk/XCUGdOsFzz3kdTZBKTtbZUyeeqLMHLGl4xlocxngsLk63iDjvPFsZfkzO6RaH06ZBrVpQvbrXEUU0SxzGeOjvv7UO34kn6mB4uVz31IxgaWnw1FOaTQcPhmbNvI7IYF1VxngmMRGuv153Mf3mG6hd2+uIgsy//+qsqTfe0GlmznkdkUkX0MQhIm1F5BcRWSwiU0Uk18mGIlJFRHblsse5MWEhNVU/E+PiYPx4OPtsryMKMnv3wlVX6c0ZNEhnT9nCvqARsMQhImWAD4HOzrkL0Y2cBhzj5a8DOwMVmzGB9vTT2l3/9ttaKt1kkZqqO/YtXgxjx2pXlSWNoBLIMY6rgEXOuc3pxyPQbWOz7WUuIt2AeOCXAMZmTMAMH669Lw89BL17ex1NECpeXDNrtWq6P7gJOoFMHPWA9RkHzrkDIlJCREo655JBu7KArsD1aGLJlYj0BHoC1K1b158xG+NT33wDDz4IV1+trQ2Txddf68DPrbfCTTd5HY05jkCOcZQGUnI8lgI4ABFpBgwEujrncr4uG+fccOdctHMuurpNyzMhYulSuPlmaN4cvvzSalBlcg7efFNXPX7wgc6kMkEtkP91twCtMw5EpDyQlCVJ9AGqAPNE+zPrApeKSCXn3GcBjNMYn/vjD512W7s2zJgBFSp4HVGQSE7WPruPP9ZWxujRtpAlBAQycXwH9BeRms65HWhX07iMJ51zPbK+WERGAVOdc1MDGKMxPrdjB7Rvr3+eOdO2v86UnKx9dnPm6Bavr7xiSSNEBCxxOOf+EZHewHQRSUPHO+4VkbuBHc65GYGKxZhASUjQWVPbt8O8edCggdcRBZGSJeHCC+GOO/SXCRniQnxRTXR0tIuNjfU6DGOOkpys9fh++EGn3tq023QLF+o+Gi1beh1JRBORZc656MK8N+TbhWlp+gNqTDBJS4N779WuqQ8/tKSR6bPPdI3GU0/ZSvAQFvKJY8UK7SINR9WqVWPfvn3ZHtu4cSNRUVEeRWTywzl47DEd5335ZbjnHq8jCgIpKVr+t3t33YBp8mRb1BfCwmJC4L//+uY8CQkJVKpUiRo1auT6/M6dO9mzZw8nnHBCtsfbtWvHqlWrjnvuxo0bM3/+/OO+Jj4+nj///DPz+ODBg8TExFAhfQpOQRLGLbfcQps2bXjggQfy/R7jG6+8Au+8A336wPPPex1NEEhIgP/8B2bNgl69YMgQHd8wIcsSRw7lypVj+/btuT6XM2FkmD17NgCffvop3bp1o2T6D0VKSgqjR4+mR48eub4vp5UrV/Luu+9mHl955ZUMy7LDWZcuXbjgggvyfa4TTzwxX681vvPOO/DSS3Dnnbo0wb5UA2XLQvnyOuXWml9hIeQHx0Wi3WefxdKtW9HPlZCQQK1atUhISMj1+RNOOIGNGzceM4HUqFGDlStXUjN9vuWuXbto1KgRu3fvLnAcX375JRs3bqR27drcdNNNVKtWDdCuqtatW7Nly5Zjvv/999/PfP+kSZM477zzCnR9UzijRsFdd+lmTBMm2AI/ZszQ1Y516mj/nWXRoBLRg+PguxaHiCAiREVFERUVRY0aNahRo0bmcWpqKnKc//zly5fPlnQSEhIyu5nya9OmTZx99tmsWbOGhg0bsn37dqKjo4mLi8vzvc45hgwZwtChQxk7diyff/45N998M999912BYjAFN3ky9OgB7drBuHERnjScg9de05rxL72kj1nSCCth8d/78GHfnKd8+fIcOHAg8/i5554jJSWFQYMG5ev9FSpUyPb+AwcOUL58+QLF8Oabb9KjRw/69euX+VjDhg155ZVXmDx5cq7vOXToENOnT+e9996jQoUK/Pjjj1SvXp06deowdepUunfvztixY+nRoweXXnopxWyRlU/Nnq3llVq2hClTdKZpxEpM1GbXV19p3fgsXa8mfIRF4ihqi2Pbtm2cc845Rz2ekJBAWloaI0YcXW9xxYoVnHzyyQDUr1+fpKQkdu7cyZVXXkmpUqUASE5O5p9//iEqKooSJUqwcePGPGNJS0ujRI6vq8WLFz/ue6ZMmcLYsWN54oknuO666wD4559/KFWqFM2aNSMmJobPPvuMV199lVq1anHGGWfkGYfJn59/1hJLjRrBt99GeCmRv/7SVsbKlfD669C3r7U0wpVzLqR/wXlu4EAXFM4880wXExOTebxixQrXqFGjAp1j3bp17pRTTnEvvPCC++qrr9ygQYPcqaee6pYtW+acc27Dhg2uTp06eZ7ntttucx988EHB/gKmQJYuda5SJefq13du2zavowkC+/Y516qVc99953UkJh+AWFfIz92w6LPw5awqr51++unExcVRq1aAQd4MAAAc4UlEQVQtli1bRvny5VmyZAnnnnuu16GZLJYv1w3qqlaFuXOhVi2vI/JIaqruznfoEFSqBIsWHSnMZcKWdVVlMWHCBB555JFjPh8fH8+mTZv8vgDvhBNOsPUXQSwuTgfBK1fW+lPpPZaRZ/duHcf4/nudbnvXXdY1FSFCfjpu8eLRrk+fWN580//XqlWrFrGxsZmJI+fYyO7du6lcuXLmGEVKSgp79+7NnEoLEBMTkzk2klVaWhr169fncPpIf8a/S8ZMLxGhWLFiTJw4kRtvvDFzOu4vv/xChw4djjrfvn37KFWqFGXLlj3quWOtUzF5W7lSN6UrVw7mz4fTTvM6Io8sW6aL+rZtg6FDtb6KJY2QUpTpuCHf4hDxrquqdu3aPvsQLlasGH/88QfOucwkkdvU35wD7C1btrREECCrV2uZpTJltHsqYpPG5MnQtSvUqAE//QQtWngdkQmwkB/jKFYsfMY4ihcvTokSJShevPhx14uYwPvtN7jsMl2fMXcu1K/vdUQeOuss3ZVq+XJLGhEq5BNHIFscPXv2pGLFioG52DGcdNJJzJo1y9MYIs3atZo0QJNGw4bexuOJtWuhXz9d3Fe/PkycCFm6YE1kscRRAK+88gqVK1cOzMWOoVSpUjRp0sTTGCLJr7/CpZfq5KE5c6BxY68j8sDYsXDeeTB8OGze7HU0JgiEfOIIp64qE1zi4qBNG/0/tmABnHmm1xEF2MGDWpTw9tvhnHP0hpxyitdRmSAQ8olDxHclRwqrTZs2tGnTxtsgjE/FxOjsqbJl4ccfI7Cl4ZyOY3zyiXZRzZsHtg+MSWezqnzg4Ycf9jYA41OLFsHVV2sX/ty5UK+e1xEFWEYl22ee0Z36rrrK64hMkAn5xBEMXVWdOnXyNgDjM/Pn6xftk07SpBFRX7ITEuDhh+H00+HZZ+GKK7yOyASpsOiq8jpx7Nq1i127dnkbhCmyWbO0pXHKKTqmEVFJIzYWzj1XNxVJTvY6GhPkQj5xBEOLo3PnznTu3NnbIEyRTJ6shV0bNdJWR+3aXkcUIKmpMGgQtGql9abmzTuyh4YxxxDyXVXB0OLo27evtwGYIhkxAu67Dy64AL75BiJqx91ff4XnntNtCz/6KML+8qawQr5WVbVq0a5ChVjysdWFMUcZPBiefloLuk6cqLX6IsL//qcrwDP+3KyZ1ZqKMBG9dWwwtDi2b99u9aJCjHPwxBOaNG69FaZNi5CkkZCgazPOPlv75EATiCUNUwDWVeUDXbp0AWB+xg+iCWopKdCzJ4wcCQ89pLubRsRuukuWwB13wLp1OtX2oou8jsiEqJBPHMWK6Ziel55++mlvAzD5lpSkLYypU+HFF/VXRHzZfu01eOEFnSo2d64uiTemkEI+cRQvrh8GyclQsqQ3MbS3Hc9Cwj//6P7gP/6orYzevb2OKICqV4c774S33tKd+owpgpAfHK9bN9r99Vcs8fHeFev866+/AHLdoMkEh02bdI3Gn3/qUoVbb/U6Ij9LSYH/+z9tYdxxh9fRmCAU0YPjxYvr7/v2eRdDt27d6Natm3cBmONavlyn2m7bpov8wj5prF0LrVvr6u+FC72OxoShsOiqAm8Tx3PPPefdxc1xffcd3HQTVK2qZdHDuiJ9WhoMG6ZTxcqUgS++gPSJG8b4kiUOH2jXrp13FzfHNGIE3H+/zjadMSMCVoMvXgyPPKJ9ciNGaMEtY/zAuqp8YP369axfv967AEw2zukEonvv1Tp9CxaEcdJITj7SHdW6tf5lZ8ywpGH8KmxaHHv3ehfD3XffDdg6jmBw6BD06KG9ND16wAcfeDfbzu9WrNC/5MqV8McfWv/9kku8jspEgLBJHF62OF5++WXvLm4ybdum021jYmDgQN1KIizXaCQlwSuv6Kyp6tVhwoQI3DTEeMkShw9ceuml3l3cAPrlu0MH2LNHK93eeKPXEfnJ4cPQogWsWgV33QVvvmmFCU3AhXziENEaQ14mjrVr1wLQqFEj74KIYFOm6LbYVaro7n3Nm3sdkR8kJup/9FKltNZUkya20ZLxTEAHx0WkrYj8IiKLRWSqiFTJ8XwnEZkrIvNF5CcROSs/561c2dsxjvvuu4/77rvPuwAilHPaJdWpkxZ3jYkJw6ThHIwerV1Rs2bpY488YknDeCpgLQ4RKQN8CFzhnNssIo8CA4D7s74MaO+cOywilwMfA+fnde7q1WHnTn9EnT8DBgzw7uIR6tAhLVQ4Zgx07QqffKJLF8LKmjXwwAM6U6pVqzCeGmZCTSC7qq4CFjnnNqcfjwDWkiVxOOcmZXl9DJCvn5RatWDHDl+FWXAXXnihdxePQJs2aStj+XJ49VVdIB12g+CDB8Pzz0OFCjB8uM6eiogSviYUBPJ/Yj0gc7GDc+4AUEJEjjVZ8nHgy9yeEJGeIhIrIrHx8fHUrAleboexatUqVq1a5V0AEWTuXDjvPK0MPn26bl4XVkkjo3bciSdqbZTfftMFKZY0TBAJ5P/G0kBKjsdSgGxVFkWkooh8irY2cq1X7pwb7pyLds5FV69ePbPF4VW9xl69etGrVy9vLh4hnIM33tCu/Zo1ITYWrrvO66h8aN06nRY2fLge9+ypYxs1angblzG5CGRX1RagdcaBiJQHkpxzKVkeawCMAQY456bl98Q1a+osxb17vZmZ+Prrrwf+ohEkMVF7aiZMgM6ddQOmChW8jspHEhJ0r4whQ3TG1NVXex2RMXkKZOL4DugvIjWdczuAnsC4HK/5ArjfORdbkBPXqqW/79jhTeJo0aJF4C8aIf78Ezp2hF9/hUGD4Mknw6hr6ptvtGWxbRt0765TxGwA3ISAgCUO59w/ItIbmC4iaeh4x70icjewA5gLNAfekOyfDHdkGVDPVc2a+vv27dC4sR+Cz0NcXBwAzcNuLqi3Jk2Cu++GEiVg5swwmoHqnGa/UqWgTh1dsXjBBV5HZUy+BXQBoHNuBjAjx8OfZvlzoeKJitLft2wpVFhF1qdPH8BqVfnKv/9qy+Ldd6FlyzCqqLFtm86UqlEDBgyAK6/UbBg2TSgTKUJ+5ThA3br6+8aN3lz/7bff9ubCYWjDBrjlFl3M16ePzkotVcrrqIooIUFH9t94QwfjHn/8yHOWNEwICovEUbasjnNs2ODN9a2LyjemTtXyS85p703Hjl5H5AOzZun4xfbtuqPUwIFw+uleR2VMkYTN5PB69bxrccTExBATE+PNxcPA4cPw2GOaKOrX14KFIZ00nIODB/XPUVHQsKFusvTll5Y0TFgIixYHaOL45Rdvrv3EE08ANsZRGGvXasmQ5cuhd294/XUoXdrrqIpg2TJ44gmoVk0TRZMmWjLEmDASNi2OU0/VUhTJyYG/9rBhwxg2bFjgLxzCnIOPP4Zzz9WW4uTJOhgeskljzRrtioqO1o2VLr3UuxWpxvhZ2LQ4GjeG1FRdgHvGGYG9dtOmTQN7wRC3e7dW0ZgyBS6/HD77LMR3Ov3iC63rXq6c1kB5/HEt2WxMmAqbFkdGslizJvDXXrx4MYsXLw78hUPQ7Nlw1lm69u2NN3TsOCSTxpYtupkSQLt2mizWr9eqi5Y0TJgLm8SRsfDPi8TRr18/+vXrF/gLh5CkJO36v+IKqFQJli6Fvn1DsHbfzp06kl+/Pjz4oD5WvbrOG65e3dvYjAmQsOmqqlhR13N4UaT2o48+CvxFQ0hsrM5IXb0a7rtPyzKVK+d1VAW0fbtu0/rBB7oZSPfu8MILXkdljCfCJnGAltv2YlasbRmbu8OHtedm4EBdZ/Pdd9C+vddRFdJXX2nGu/VWXf1t/+YmgoVaR8FxnX++FsXbtSuw112wYAELbMplNnFx0KIF9O+v48arVoVY0li/XgsQjhqlx/fco3tjjBljScNEvLBLHBD4VseLL77Iiy++GNiLBqnkZHjlFU0aO3fCtGn62XvCCV5Hlk+//abdUA0b6nSvrVv18bJloUEDb2MzJkiEVVdVdLQOti5dGthtDT799NO8XxQBYmN1mm1cnC7qe/ddqFrV66gK4LnntPhgmTLw8MM6Uyokp3wZ419h1eKoUAHOPFMTRyCddtppnHbaaYG9aBBJSNCJRuefr3uiTJoEY8eGQNJITdVgd+/W4/PPh379dCXpkCGWNIw5hrBKHKA/+0uXQlpa4K45e/ZsZs+eHbgLBpFvv9Vk/dZbOiSwejV06uR1VHlITIT33tPuqM6dtUsK4PrrdVDGptUac1xhlzjatoV//gnsOEf//v3p379/4C4YBLZvhy5d4NproXx5WLhQZ6oG9ViGczqFtm5d6NVL98WYNEm7pYwx+RZWYxygM3eKFYMZM44Mlvvb559/HpgLBYHUVK0x9cwzWgD2lVd006WgrTHlnO4727Sp7n0RF6d1pB5/HC680OvojAlJ4kK8EFt0dLSLjc2+RXnr1rpGa9kyj4IKU4sWaQXbFSugTRv48MMgnpmamKgDLcOG6Vzg9eu1hHJKiu5Fa0yEE5Flzrnowrw37LqqQLtPli/XnToDYebMmcycOTMwF/PAtm1wxx2akOPjYfx4mDs3SJPGzp3w6KO6l/d990Hx4tpEqlFDn7ekYUyRhWXiuP56/X3SpMBcb9CgQQwaNCgwFwugw4e1EGGjRrrvd79+uszhlluCbMfTgwePbP/oHAwfrvOxf/pJv0H06BGCNU6MCV5h2VUFcM45+mUzl6d8bvv27QDUqlXL/xcLAOd0jOiJJzRRXHstvP221vULKitWwIgR2iXVvDlkbKS1f79WUjTGHJN1VeXirrt0jGPlSv9fq1atWmGTNJYtg8su01ZbaipMn64l0IMqaUyZooXJzj0XPvkErrsOXnrpyPOWNIzxq7BNHF27QsmS+rnib9OnT2f69On+v5AfbdqkNaWio3UsedgwnYx03XVeR4bWZJ80SVsSAJs36yD30KE6ADNmjI7WG2MCImy7qkA/CKdO1c+ZKlX8F0Ob9A+tUNxzfPdu3Uri3Xd13OLRR+Gpp4JgL6K0NB2jGDNGK9Pu3asL9bp106RRvHiQDbQYE1qK0lUV1lNMnn5au7+HDgV/1iCcOHGi/07uJ3v3alWNt9/WkiF33KEl0E8+2evI0OCaN9dmUPnyuhS9WzftQwObGWWMx8L6J7BpU+2rf+cdXRx84on+uU61atX8c2I/OHBA78ebb+rn83/+o8MDnm2b7hz88ou2KtLSNJudcIKOyF94Idx4oyYPY0zQCOvEAfot+txz4eWX9du1P0yePBmATkFcpOnAAS0J8n//p91THTroPWne3KOAli/XbqiJE+Gvv3RAqmNHTSQiWkvKGBOUwnZwPMPZZ2up74wFxP7w7rvv8u677/rn5EUUH68b1tWtq2MXLVroF/xp0wKcNPbv1yTx7796/NVXmhyaN4fRo3Xh3oQJNm5hTAgI68HxDPHx0KSJfnj+/DOUKuXbGPbt2wdAZc9HlI/YuFG7oz75RCcldeyoiaNlywAGsWGDzuWdPl3XWCQnw6xZcMUVuk1jyZJBMApvTGSywfE8VK+uVSc6dtRv34MH+/b8wZQwli3TLrkvvtBij9266UK+xo0DcPF9+zRL1aypXVHnnaePN24MffrogFOrVvpYCI0LGWOyi4jEATrGev/92sd/9tm6zsNXJkyYAMAtt9ziu5MWwOHD2gs0dCgsWaIbWj3yiE6tjYry44WTk7V+/axZ8MMPuhHKgw/q3N6zztLf27e3LVeNCTMR0VWV4fBh7SVZskQXH19zjW9i8Godx9atWpbpo490570GDXSbie7d/dQDdPCgTpE94ww9btAA1q3Tpk10tN7cG27QgRRjTFCzrqp8KlVKE8YVV2i31aRJvlkZ/e233xb9JPmUlKQD2yNH6pd85zQB9u6tf69ivpzuEB+vWXbRIvjxRy38ddJJOoACWvWwYkVdX+HPFZbGmKASUS2ODP/8ox+y//2vVn99+OHgnszjnH5mjx4N48Zp/CefrC2Lu+4Cn2x3npCgAyQxMdrHVby4djt98IEOYkdHwyWXwMUXa6YK5htmjMlTUVocEZk4QMdxu3fXb+8336zTdQu71fSYMWMAuP322wt3glw4p5/jX32lvzZs0F32OnXSZHHZZfrZXqgTO6dNk59/1imxcXGwZs2RjdpXr9buqFWrdJXguedaWXJjwowljkLWTk9L08HyF17QHpeBA/VDuWTJgp3HV2McSUnaIzRzptbY2rBBq2u0awc33aTdawVa/Z6YqE2VtWs1Mfz3v/C//+l6icsv16myDz6oA9ktWuhc3ejowmdQY0zIiOzE0ayZi33zTTjlFF2oUbZsgc+xerXOuFq4EE49VffQ7to1/9W5k5OTAShZwIyTmqoVaBcs0GQxb55ueVu6NLRtq8nixhuPM3yQlgZ//60D1ps2wR9/wO+/Q5cuOvV1xQptLYC2GJo21STx4IO6YUnGKm1jTMQJmcQhIm2BwUAKsBO42zm3J8vzJwMjgMrpr+nlnIs73jmj69RxsX//feSBGjU0iXz/vX49X7xY95uuWRNq1dLfq1Y9qp/HOf0C/uqr2s1frpx+aF9zjY6HZOw8WljOaZXelSv183zRIu0pyqgU3qCBzlxt314rhJcrl/6mRYt0ytSOHVpCfPNmfcFdd2ntkKzrIUQ0eT73HNxzz5EmTKNGOiji05FzY0woC4nEISJlgP8CVzjnNovIo0Aj59z9WV4zHfjIOfeNiJwHfO6ca3K880Y3b+5ihw498q170yatffTNN5ocHngAPvww+5vKlNGppSIwaJB+OFeuDCecgKtUmY3/1mJwYm8mToRGuxdRhT3UiCpNndPLcHKDMlQ9tRJlz2lM1apQIX4DP37/DcWKOS66qB0JCbDvcFk2yqls3gyllv3M/o172LEpidRD/1KWQ8RTgw1Nr+eii+CRjY9S122ifMo+HXjZtUsz1ccfa6zly2usoB/8depoDZXnn9fE8vHHmhROOUWbS4VocRljIk+oJI4bgBucc3enH1cE1jrnTko/PhGIcc7Vz/KeGKCbc+63Y503zzGOAwe0O2fHDti+XX9PTNSa66AfwDNm6CDwvvQP7zp1YNMm0tLgQOv2VP75+2ynXENjmrAGgIW0pjWLsj2/hPNpxRKKF4eVxc7mjOT/ZXs+5eK2lPhxrh5ccolOk6pcWX9VqaIzl3r21Od//FEfr1lTWxdWUtwY4wOhkjgeASo65/pneWwnUMc5lywiZwNvOufaZXl+IvCBc25OjnP1BNI/WWkK+Kl8YcipBuzyOoggYffiCLsXR9i9OKKRc65iYd4YyK+vpdFxi6xSAHec51OBtJwncs4NB4YDiEhsYbNmuLF7cYTdiyPsXhxh9+IIESncdFQCW1Z9C1A340BEygNJzrmU3J5PdwqwMSDRGWOMyZdAJo7vgPYiUjP9uCcwLuNJ59zfwF4RaQMgIucCh51zGwIYozHGmDwErKvKOfePiPQGpotIGrAeuFdE7gZ2OOdmAHcAH4vIACAJuCcfpx7ut6BDj92LI+xeHGH34gi7F0cU+l6E/AJAY4wxgWUrwowxxhSIJQ5jjDEFEjKJQ0TaisgvIrJYRKaKSJUcz58sIt+LyBIR+UlEmnsVq7/l4150EpG5IjI//V6c5VWs/pbXvcjyuioisktEbgx0jIGSn3shIveKyAoRWSgin3kRZyDk42ckSkS+FpHZIvKziFzrVaz+JiIXpd+Hoz4TC/256ZwL+l9AGWAtUDf9+FHgwxyvmQ5cl/7n84DVXsft4b34D1Aq/c+XA0u9jture5HltZ8Aq4EbvY7bw/8X1wOzgHLpx+J13B7eiwlA1/Q/1wC2A8W9jt0P9+IzYAq6SLp5Ls8X6nMzVFocVwGLnHOb049HAB0ynkwvV3KGc+4bAOfcMiBRRBoHPFL/O+69AHDOTXLOHU4/jAFqBzC+QMrzXgCISDcgHvglgLEFWn7uxbPAQ865gwAu/dMiDOXnXmwFaqX/uSq6mvyoxcZh4H7nXEdyWS1flM/NUEkc9dDpuwA45w4AJUQko455XY5eKLgJqBOI4AKsHse/Fzk9DnwZgLi8UI887kV6ReauwHMBjy6w6nGceyEiJYCTgOtEZF56V+bVnkTqf/XI+2fkVeABEVmNfrnqFY6JNONLwjEU+nMzVCrm+axcSRjI614AmUUk30HvwwOBCS3gjnsvRKQZMBC42jmXIuG990he/y+qoV0yfzrn2qZvYbBQRC50uvg2nOTnZ2Qi8KjTStwno+vLOmRppUSCQn9uhkqLw8qVHJHXvUBEGgCzgWnOuXuzPhdm8roXfYAqwDwRiUO7K94SkTsCHqn/5XUvdgEJzrmvAZxzf6HftM8IdKABcNx7ISLVgFOydNH8BcwBrvQgVi8V+nMzVBKHlSs54rj3It0XaF/2tIBGFnh5/b/o4Zxr6Jxr7pxrDnyNfssMx9lEed2LFOAnEbkGQESqAs2AlYEONADy+hnZDZktUkSkAjqJJPv+B2GuKJ+bIbNyPH263ItoM2o9cC9wC+nlSkSkPvAx2vxKQgeFfvcqXn863r0A5gIHgJ9yvO2OcGyG5/X/IsdrRwFTnXNTAx1nIOTjZ6Q2WmaiItpN3d85N9OreP0pH/eiOfAGeh/KAcOdcyO8itffRGQ+0Mc5F5e1zFNhPzdDJnEYY4wJDqHSVWWMMSZIWOIwxhhTIJY4jDHGFIglDmOMMQViicMYY0yBWOIwJp2IVBORVoV8r0sv61GQ9+wSkco5HqsnIlsKE4MxgRIqJUeM8RkRGQb85pwbluOp5mhNqzZZXvsOOv+/OHAiR4rF3Zd+3MY5d2c+r1sdOD3LQ+WAFiKSkH6c74QhIhOA+c65D/L7HmN8xRKHiUR1gMT8vNA59wjwSHrF0G+cc/UznhOROwt43WbAw1mOZwG9shyPB5YU4Fz/FPD6xviEJQ4TUUQkCmgBnC0ibzjn4vP51hJoq6PQnHNzgbnpJS5uRqu4bgO+cs7tSo+vXl7nEZEHgZ1oWY3z0sthGxMwNsZhIoaI1AImo6XmnwVmikijfL69LlBb1G0iMgi4qRAxnAL8Fy0u+Du6J0RsfnZeS7/2Y0Bv4DagG/BlGJdHN0HKWhwmIojIPehOcM9kVIgVka3ANBEZ7JwbmccpzkXr+ZyJftvfSOE2yOoLfOKcG5Altt+BF4BOx4i9LLp730NAAnBJektpq+hWuKNF5DZ0l8MFzrlw3E7ABBFLHCZS/Ih2Ce3LeMA592NGhdR069AP39x0A0aj2432A34QkSSyDKTnUzFy3wPheDqiLYzXM0qBp+/edtg5t1JEWgB3AM+jW6CuKWBMxhSIFTk0EUVEugJDjvOSxc65bN/8RaQzcA9wN7AMOMs5F58+ON7GOXeniDigZF57n4jI6ejeD6PRkuanozO0OjvnlqePcfzknIvK4zxj0l/34fFeZ4w/2BiHiSjOuXHOuVq5/QK6oBs/ZRKRKsBg4In0/Qs+AYZKIbcTdM79iU773Q6ch87uusA5t7wIfy1jAsq6qow5BhEphm4K9KFzLmPDo1fQPU86Fva8zrm9gK2/MCHLEoeJKOmD5O9w7DUQP2f8wTmXJiJ3O+d+zfLYYRG5wjl3qCDrONKT0DqgVMZDGafM8isN6JzjfS3RnQtzqgxcLyIv5XwivfVkjN9Y4jCR6Kv8rvbOmjSyPHaooBdMT0IN0IThgDSXywBjznUczrlf0Cm7xgQNSxwmEt0iIu2P8/w5zrltvr6ocy6v2VPGhASbVWWMD4hIc+dcnA/OUwqo75xb7YOwjPELSxzGGGMKxKbjGmOMKRBLHMYYYwrEEocxxpgCscRhjDGmQCxxGGOMKRBLHMYYYwrk/wFjvUTA6dwfTgAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "p = 0.1\n",
+    "q = np.linspace(0.001, 0.999, 500)\n",
+    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
+    "mse = (p - q)**2\n",
+    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
+    "plt.text(0.05, 0.32, \"목표 희소\", fontsize=14)\n",
+    "plt.plot(q, kl_div, \"b-\", label=\"쿨백 라이블러 발산\")\n",
+    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
+    "plt.legend(loc=\"upper left\")\n",
+    "plt.xlabel(\"실제 희소\")\n",
+    "plt.ylabel(\"비용\", rotation=0)\n",
+    "plt.axis([0, 1, 0, 0.95]);"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 6.1 쿨백 라이블러 발산\n",
+    "\n",
+    "쿨백-라이블러 발산(Kullback-Leibler divergence, KLD)은 두 확률분포의 차이를 계산하는 데 사용하는 함수이다. 예를들어 딥러닝 모델을 만들 때, 학습 데이터셋의 분포 $P(x)$와 모델이 추정한 데이터의 분포 $Q(x)$ 간에 차이를 KLD를 활용해 구할 수 있다([ratsgo's blog](https://ratsgo.github.io/statistics/2017/09/22/information/)).\n",
+    "\n",
+    "\n",
+    "\n",
+    "$$\n",
+    "{ D }_{ KL }\\left( P||Q \\right) ={ E }_{ X\\sim P }\\left[ \\log { \\frac { P\\left( x \\right)  }{ Q(x) }  }  \\right] ={ E }_{ X\\sim P }\\left[ \\log { P(x) } -\\log { Q(x) }  \\right]\n",
+    "$$\n",
+    "\n",
+    "\n",
+    "\n",
+    "Sparse-오토인코더에서는 코딩층에서 뉴런이 활성화될 목표 확률 $p$와 실제확률 $q$(학습 배치에 대한 평균 활성화) 사이의 발산을 측정하며, 식은 다음과 같다.\n",
+    "\n",
+    "\n",
+    "$$\n",
+    "D_{KL}\\left( p||q \\right) = p \\log{\\frac{p}{q}} + \\left( 1- p \\right) \\log{\\frac{1-p}{1-q}}\n",
+    "$$\n",
+    "\n",
+    "\n",
+    "위의 식을 이용해 코딩층의 각 뉴런에 대해 희소 손실을 구하고 이 손실을 모두 합한 뒤 희소 가중치 하이퍼파라미터를 곱하여 손실함수의 결과에 더해준다.\n",
+    "$$\n",
+    "Loss = \\text{MSE} + \\text{sparsity_weight} \\times \\text{sparsity_loss}\n",
+    "$$\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 6.2 텐서플로 구현\n",
+    "\n",
+    "이번에는 텐서플로를 이용해 Sparse-오토인코더를 구현해보도록 하자. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reset_graph()\n",
+    "\n",
+    "################\n",
+    "# layer params #\n",
+    "################\n",
+    "noise_level = 1.0\n",
+    "n_inputs = 28 * 28\n",
+    "n_hidden1 = 1000 # sparsity coding units\n",
+    "n_outputs = n_inputs\n",
+    "\n",
+    "################\n",
+    "# train params #\n",
+    "################\n",
+    "sparsity_target = 0.1  # p\n",
+    "sparsity_weight = 0.2\n",
+    "learning_rate = 0.01\n",
+    "n_epochs = 20\n",
+    "batch_size = 1000\n",
+    "\n",
+    "def kl_divergence(p, q):\n",
+    "    # 쿨백 라이블러 발산\n",
+    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))\n",
+    "\n",
+    "inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
+    "\n",
+    "hidden1 = tf.layers.dense(inputs, n_hidden1, activation=tf.nn.sigmoid)\n",
+    "outputs = tf.layers.dense(hidden1, n_outputs)\n",
+    "\n",
+    "# loss\n",
+    "hidden1_mean = tf.reduce_mean(hidden1, axis=0)  # 배치 평균  == q\n",
+    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
+    "reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)\n",
+    "loss = reconstruction_loss + sparsity_weight * sparsity_loss\n",
+    "\n",
+    "# optimizer\n",
+    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
+    "\n",
+    "# saver\n",
+    "saver = tf.train.Saver()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch : 0, Train MSE : 0.13764,                 sparsity_loss : 0.90868, total_loss : 0.31938\n",
+      "epoch : 1, Train MSE : 0.05978,                 sparsity_loss : 0.02385, total_loss : 0.06455\n",
+      "epoch : 2, Train MSE : 0.05260,                 sparsity_loss : 0.02887, total_loss : 0.05837\n",
+      "epoch : 3, Train MSE : 0.05017,                 sparsity_loss : 0.19674, total_loss : 0.08952\n",
+      "epoch : 4, Train MSE : 0.04457,                 sparsity_loss : 0.00709, total_loss : 0.04599\n",
+      "epoch : 5, Train MSE : 0.03983,                 sparsity_loss : 0.35642, total_loss : 0.11111\n",
+      "epoch : 6, Train MSE : 0.03850,                 sparsity_loss : 0.04892, total_loss : 0.04828\n",
+      "epoch : 7, Train MSE : 0.03621,                 sparsity_loss : 0.02348, total_loss : 0.04091\n",
+      "epoch : 8, Train MSE : 0.03363,                 sparsity_loss : 0.05729, total_loss : 0.04509\n",
+      "epoch : 9, Train MSE : 0.03088,                 sparsity_loss : 0.04279, total_loss : 0.03944\n",
+      "epoch : 10, Train MSE : 0.02828,                 sparsity_loss : 0.32262, total_loss : 0.09281\n",
+      "epoch : 11, Train MSE : 0.02502,                 sparsity_loss : 0.03091, total_loss : 0.03120\n",
+      "epoch : 12, Train MSE : 0.02325,                 sparsity_loss : 0.08872, total_loss : 0.04099\n",
+      "epoch : 13, Train MSE : 0.02177,                 sparsity_loss : 0.06630, total_loss : 0.03503\n",
+      "epoch : 14, Train MSE : 0.02201,                 sparsity_loss : 0.11473, total_loss : 0.04495\n",
+      "epoch : 15, Train MSE : 0.02036,                 sparsity_loss : 0.04797, total_loss : 0.02996\n",
+      "epoch : 16, Train MSE : 0.01799,                 sparsity_loss : 0.18309, total_loss : 0.05460\n",
+      "epoch : 17, Train MSE : 0.01866,                 sparsity_loss : 0.02909, total_loss : 0.02448\n",
+      "epoch : 18, Train MSE : 0.01780,                 sparsity_loss : 0.23517, total_loss : 0.06484\n",
+      "epoch : 19, Train MSE : 0.01738,                 sparsity_loss : 0.02495, total_loss : 0.02237\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Train\n",
+    "with tf.Session() as sess:\n",
+    "    tf.global_variables_initializer().run()\n",
+    "    n_batches = len(train_x) // batch_size\n",
+    "    for epoch in range(n_epochs):\n",
+    "        for iteration in range(n_batches):\n",
+    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
+    "            sys.stdout.flush()\n",
+    "            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))\n",
+    "            sess.run(train_op, feed_dict={inputs: batch_x})\n",
+    "        recon_loss_val, sparsity_loss_val, loss_val = sess.run([reconstruction_loss, \n",
+    "                                                                sparsity_loss,\n",
+    "                                                                loss], feed_dict={inputs: batch_x})\n",
+    "        print('\\repoch : {}, Train MSE : {:.5f}, \\\n",
+    "                sparsity_loss : {:.5f}, total_loss : {:.5f}'.format(epoch, recon_loss_val,\n",
+    "                                                                    sparsity_loss_val, loss_val))\n",
+    "    saver.save(sess, './model/my_model_sparse.ckpt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from ./model/my_model_sparse.ckpt\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAD+CAYAAACnfntIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGfpJREFUeJzt3VmM3lUZx/FTttKVLmlnOmVpwQJVSEONUpQYY5BEIRI1JCgGjJpITDSRRA0mJuqNl9qYGDUaUPTCxKQSJEJcAmpB1EKhoWBKF6RMpyt0oZ2WzRs8/J4f856+M52Zdp75fq7Oyflv8ybzP/k/z1mmvPHGGwUAgIxOO9kPAADAWKGTAwCkRScHAEiLTg4AkBadHAAgLTo5AEBadHIAgLTo5AAAadHJAQDSOmOc78fyKqeOKSf7AYCJZmBgILzDXn/99a7OO/PMM0P92LFjtXz66aeHNl2F6rXXXgttU6dOreVXXnkltJ1xxluv81dffbXj/f08v8fZZ59dy4ODg6HNn1Wddlrnb6bW76TP3TrO2/r6+rp6h/ElBwBIi04OAJAWnRwAIK3xzskBQBpnnXVWLXserJVbmzLlrXSS57I096Q5OKfX8Pt7DlDvoTmwoa6jz+r312drned/kz6Pn9fKw2kO0HOJ3eJLDgCQFp0cACAtwpUA0CUPrWmI0IfXa/jOw5UazvONq/U6fl4rlKnhPL+mTlnwcKEf2wo76rF+nRZ9ttY0BKf3mzFjRtfnKb7kAABp0ckBANKikwMApEVODgC6pFMGSok5OR/irrknz0O1phDoNT3vpcdqns3b/Dy9v+cV/djWMlutKROte+gUAj9Pj239Fp477BZfcgCAtOjkAABpEa4EgC55iLA1HF7Da63pBR6i0/Chr1yiob3WvT2squcdb+cEDbvqjgTHe7bWFAb9e1uhW6dtPp2iW3zJAQDSopMDAKRFJwcASIucHAB0yfNnrdyaai3d5UPqW0P49Ty/n57nea/W8P7WLuKe22vlyPR5/DzN17V+Q9c6r1t8yQEA0qKTAwCkRbgSAEaoNYxfw4K+GoqG/Y4ePdrxmocPHw5tGkr04f0zZ84c8rhSYijzeGE/fTYPbWrY1f8mPdbDs63zun0WVjwBAMDQyQEA0qKTAwCkNeFzcv/4xz9CffXq1bW8ePHi0DZt2rRavvXWW0PbvHnzhiwDwP8NZxdvrft5+/fvr+WBgYHQtmHDhiGPK6WUOXPm1PKCBQtC29KlS2t59uzZoU3zd63dA7zuf5PmCL1Nd+72Xct1SoHnC/W38WuONA+n+JIDAKRFJwcASGvKaHwODsOo3+ySSy4J9U2bNo3oOuecc04tr1q16oSeaSSWLFlSy3fccUdoO//888filp2X/gYwpB07doR3mL4/PQynob3BwcHQtmfPnlp+/PHHQ5vWDx06FNoOHjxYywsXLgxtOoXgyJEjHZ9l7969oc1Di/q+8WkKOv1g3759oU3Dp1dddVVou/jii4d8zuPRMKeHMvv6+rp6h/ElBwBIi04OAJAWnRwAIK0JP4Xgd7/7XaivX7++lt/1rneFtqeeeqqWH3300dB2zz331PIDDzwQ2nRo7tatW7t+Nl1Kp5RSFi1aVMvPP/98x/M0P1dKKd/4xje6vieAseP/08pzRpqj82H7Op3Jh/vre8LvN3369I737+/vr2VfDkzrBw4cCG2+Y8CuXbtq2d9Feq7nEufOnVvL5513Xmg799xza1nHP5QSfzd/Fv3djrejeSd8yQEA0qKTAwCkNeGnEIwWHeK7bdu20Kbhyi1btnR9Tf/01jCEXrOUUnbv3l3La9asCW033HBD1/ccBqYQAMO0a9eu8A7TcJpPIdA2f8+2woe6K4GGNb3+0ksvhTZd3d+nCXS6dymlHDt2LNT7+vpqWVdYKSWGKH/4wx+GNg1J3nLLLaFt5cqVtTxr1qzQ5r+b0lCmr8yyYMECphAAACY3OjkAQFp0cgCAtCb8FILRosvXXHrppR2PW758+YjvodMWdFmfUkq58sora/naa68d8T0AjB3PrekQf59C0No1XKcN+HF6Tc9XaZ6/p6cntOlSXj59SvNunoNr7TDuu5bfd999tezTInScgS7jVUr8e1vTMHyagO4MPpwdxRVfcgCAtOjkAABpEa4cQy+//HKof/zjH69l/yz/wQ9+UMs+bBjAqak1hUDDlxp28zb/f2+FOXUXAA+PahiwtdK/74jg4UP9O5599tnQ9vTTT9ey/02XXXZZLfvUAz3W331a92uq1u/SwpccACAtOjkAQFp0cgCAtMjJjaG77ror1AcGBmp5/vz5oe2CCy4Yj0cCcAI0J1ZKHNbuQ+pb+SXNQ/kQfuV5fb2m5wC17kt3advxclu6XNj9998f2rZv317Lq1atCm0rVqyoZf+dlN+/29xl6/ds4UsOAJAWnRwAIC3ClaNs8+bNtXz77bd3PO6RRx4J9d7e3jF7JgCjw1fd0NCbD8XXofrDWeVDeUhSVzXxkKCG8/x+2uZhVb/HM888U8u+Meq8efNq+SMf+UhoW7x4cS1PnTo1tOnf2Pp7PZSpx7ZCoC18yQEA0qKTAwCkRScHAEiLnNwou/fee2vZ4/c33nhjLV944YXj9kwARofnhbTuuSbNi3keTHcT0LJfx++nx3r+SncX8OH2+izTp08Pbc8991yor1+/vpZ9CoNOE7j88stD24wZM2rZpzBojq41FcCXKtN8of+G3eJLDgCQFp0cACAtOjkAQFrk5E6Q593WrFlTyz5X5Hvf+14tj3TbCAAnj+eMdKfwVpvPW9O8m+ev9N3Qyq35/TR/57k0nwundu7cGeq6nY7m2Uop5aMf/Wgt67y4Utp/kz6r5y5b70LNw/mu7N3iSw4AkBadHAAgLcKVJ+jnP/95qP/tb3+r5U9/+tOhjWkDwMTWCh/6cP9WKFOv40P6tU2X8fJ7+P00ROk7G2hIcMeOHaFN31mllLJx48Zavuaaa0Lb0qVLh7xmKXEKg7fps7aWOHPsQgAAQAOdHAAgLTo5AEBa5OSGSZe8KaWUL3/5y6E+Z86cWv7ud787Ls8EYHx4HkyHuLe2sGkN4fch9XrstGnTOh7r05d0yS8fbq/5On+HPfTQQ6Gu2+l8+MMf7tjmz61/v+fPtN7aGVzzeqWMPJcXrjGiswAAmADo5AAAaRGu7IIO4/3Upz4V2nxo8M0331zLTBkAcvEwoIbhfNi+hu9auxe0dvhu7ZTtIUENX/pKJZs3b67le+65J7Tt27cv1K+//vpavuSSS0KbhgwPHjxYOvGdFTQE62FOrfsqURq+9Hdtt/iSAwCkRScHAEiLTg4AkBY5uSF4zPi6666r5f/85z+hbfny5aH+ne98Z+weDMBJ1VpayvN1mr/yttZyVa2dwTtdv5SYB/NdAHTprk2bNoW2lStXhrpOG/ApDJpnbO260FrWy+nf77+TnjfSnVv4kgMApEUnBwBIi3DlEHxI7YMPPtjx2LvvvjvUdUUAALl4yEyH7bfCjh6G63aHAm/TEGVrF4C1a9eGtgceeKCW/R31mc98JtR7eno6PvehQ4dq2dM6OoWitaqJrwyj12lNmWiFPFv4kgMApEUnBwBIi04OAJAWObk37d+/v5ZXrVrV8bhf/epXoX7FFVeM2TMBOLW1Vv7vdlfr1jJXraWs/Jrbt2+v5Ycffji0aS5Np0SVEnf7LiX+HQcOHOh4/9aQfv8tWudpXtP/Xl0OjCkEAAAYOjkAQFqEK99055131vKWLVs6Hnf11VeHeisMASAX36hUtXYa8JCkht585RKdCtDaaWDnzp2h7fe//30tr1mzJrTNnz+/ln2Fk9mzZ5dO/P4aTmxtBOu/k57nv5OGNj3MqdfxXR66xZccACAtOjkAQFp0cgCAtCZtTs5X4v72t799ch4EwITheahWPsmXr1Kao/PjNF/nO2Xr7gL//Oc/Q9t9991Xy1u3bg1t5557bi37sl6tpcNaOysMDg6GNv1tWkuVtZY48x3FW0ucdYsvOQBAWnRyAIC0Jm24UjcRLKU9s183RvVNBAFMHh4y09CiTxPQ0N7Ro0c7nudD6jVE59c8cuRILesqJn6PZcuWhbb3vve9tewhUJ/CoMP2W6uxuNaGqsrb/HmUhjLZhQAAAEMnBwBIi04OAJDWpM3Jtbzvfe8L9T/+8Y+1TE4OwP+1dvjWvFtrx2tfOktzW7rEVylx2L5PBbjmmmtq2ac6aE5u7ty5Ha9ZyttzdKqVF2vt8N1aDkzb/N56v9bOBi18yQEA0qKTAwCkNWWkn4AjNK43QxPbJwDD1N/fH95hrSHuGob0Nh2m76t8aFsrPOihzBa9jocLW7sC+Gosmq7R1Vf8Oh521OkNHsps9UGtVVR6e3u7eofxJQcASItODgCQFp0cACCt8c7JAQAwbviSAwCkRScHAEiLTg4AkBadHAAgLTo5AEBadHIAgLTo5AAAadHJAQDSopMDAKRFJwcASItODgCQFp0cACAtOjkAQFp0cgCAtOjkAABp0ckBANKikwMApEUnBwBIi04OAJDWGeN8vzfG+X7obMrJfgBgotm9e3d4h7322mu1PGVK/Jd64423Dn399ddDmx57+umnh7ZXX311yOuXUsrZZ5/dse200976ZvH76TX1uKGeW+nfMNS5ndr82bq93+HDh0N95syZtex/U29vb1fvML7kAABp0ckBANKikwMApDXeOTkAmLA816T5pVb+qpWjOuOM+BrWY88888yObZ4fO3r0aMfztO7nea5L6/43aZvfQ/l506dPr+UjR46ENv37Z82aFdr0733llVc63q+FLzkAQFp0cgCAtAhXAsAo8BCd1n3YvNY9lOnhS6UhO7+fnufX1DBjaxpAKXFKgz93K+yq1/VQpoYoW/dv/RY+1aJbfMkBANKikwMApEUnBwBIa9Lm5H7961+H+ssvv1zL69atC20//elPO17nW9/6Vqh/6EMfquUPfvCDJ/CEAE51muvy/JXmk3z4+9SpU2v52LFjoU3zUj68v7+/v5b37t0b2rZv317L+/fvD236bAsWLAht5513XqgvWrSolmfPnh3azjrrrI7Pprk2/3t1OTL/e3XJMad5Rz+vW3zJAQDSopMDAKQ1xYehjrGTugvBl770pVr+yU9+Mib3eOc731nLf//730PbOeecMyb3HCF2IQCGqb+/P7zDNCTpYTcd8u7D3wcHB2vZw347d+6s5c2bN4c2faf8+9//Dm2PPvpoLWt4sJRS+vr6annx4sWhbcWKFaG+dOnSWvaUi15n2rRpoU2nCfhUAA1zeltrtwY9VkO8pZTS09PDLgQAgMmNTg4AkBadHAAgrdRTCDQHV0r3ebgrrrgi1D/5yU/W8qZNm0LbL37xi1DfuHFjLf/2t78NbZ///Oe7uj+AU5MvuaU5JM+DaZvnoTSfpTm4UkrZsWNHLT/88MOh7a9//WstP/XUU6FNd9FetmxZaLvssstqed68eaHtxRdfDPWnn366lpcsWRLadDcB3fWglJhba+UgvU2nN3ieT6cNjHT8CF9yAIC06OQAAGmlC1f+97//reWf/exnHY97z3veE+r3339/LesneSnt4a/PPvtsqK9du7aW9+zZ08UTA5gofNUNfTe0wne+Kr+uDuLhO72mbzA6Y8aMWr7ppptC25VXXjlk2c/z8Ohf/vKXUN+6deuQ5VLi9APfTUBDjR5a1JCkr4ai12ntwEC4EgAAQycHAEiLTg4AkFa6nJzmwTyGq3m4P/3pT6FNh9+23HXXXaH+r3/9q+OxN9xwQ1fXBDAxeB5Kpwn4+0Zzaz69QJcA0+NKiTm6iy++OLTp1ICrr746tOmSgj6uQPOFvlTYCy+8EOo6hcCnIujf6DsUtBw+fHjIa5QS85W+rFenew8HX3IAgLTo5AAAaaULV65cubKWfQi/hgV8Zn23fFrCSDfyAzDxtFY88WkCGl7TFT9KaQ+b101NfSqAXlN3Cygl7nLiOyLoZqsPPfRQaPOUS2sqwJw5c2rZdwXQ30I3oS4lToXw8/Qe/tz6OxGuBADA0MkBANKikwMApJUuJ6dGayfuu+++u5afeOKJ5rHXXnttLV900UWjcn8ApwbPC+lwfx/+rvkkn3rQ2jVcl+DyqQA61cnbdLms3bt3h7b169fXsubnSnl7/mzhwoVD3q+UmFv085Tn1jTv2Fq6y38nXQ6sNb2ghS85AEBadHIAgLRShytH6vHHHw/1L37xi7XsK40vWrQo1FevXl3LPqQYQC46hcjDaRqW8zBna0NVDUP6VCcN3/k19+7dW8u+ufOGDRtq2Xch0PBoKXGj1N7e3tCmz+O7CWibhx31t/DwrE4p8DCnvkP9vG7xJQcASItODgCQFp0cACAtcnJDeOSRR0Ld83DqtttuC3VfNRxAHq0pBL6bgK68P2vWrNDWml6geTdv03t4TmxgYKCWn3zyydCmOTrPAS5fvjzUL7zwwlrW/Jzf36cwaG7NlzvUXRi8rbV0lz5r6z3cwpccACAtOjkAQFqEK9/0uc99rpZ/85vfdDzuq1/9aqh//etfH7NnAnBq8dU6dMi7hwFbITo9z6catYbbayhz//79oW3t2rW1vG3bttCmw/svvfTS0DZv3rxQX7FiRS37xqi6ipT/FvpsHsrU38Y3kNUQZWunAaYQAABg6OQAAGnRyQEA0pq0OblDhw6F+h/+8Ida9l18e3p6avmb3/xmaPNhwwDy8qW7NLfW2jXc2zQP59fU/JXmufxY32lAdwV44YUXQpvms3SKQClvnyagu3/71Ad9bn82n9LQ6Tynu4b7Mmb6LvbpFN3iSw4AkBadHAAgrUkbrrzxxhtDfdeuXR2P/cpXvlLLPtwWwOThIbrW5qcaWvQ2De21VlHxaQnbt2+vZd1ZoJRSnnnmmVrW1VZKiSHJ+fPnh7YLLrgg1HXTVN+hoLVRqoZkPeyo4UqfTqG/k+9CoL+F//bd4ksOAJAWnRwAIC06OQBAWpMqJ7du3bpafvDBBzse94lPfCLUb7/99rF6JAATiOfPPIekNJ+kw+RLiUtb+dB7rb/00kuh7bnnnqvlxx57LLRt3Lixln1Hb82zXX755aHNxxksXry4ln3Yvg7p92XFdOqBT4to5dM0X+fTtzTP57m8bvElBwBIi04OAJAWnRwAIK3UOTmPg99xxx213Irvvvvd7w51lu4CUMrbc02tHb41f9faJsbfL5qH0x29Synl3nvvrWWdF1dKzPNddNFFoW3ZsmW17PPk5s6dG+q6w7f/Tfr3+5Jfur2Ob6ej8/08j6lLLLaWOGNZLwAADJ0cACCt1OHKH//4x6H+5z//ueOxujM4UwYADMWHwrdSGZoS8eN0aLxfU0N0Tz75ZGjT3b/37t0b2j7wgQ/Usu/+reFKX/JLd/v25/blufS5NaxZSns5sn379tXywYMHQ5uGZxcsWBDaWrsXdIsvOQBAWnRyAIC06OQAAGmlzsn5Lt4t3//+92uZKQMAutHaubq1dJce622as+rv7w9tuhu458R0OS5f1kunU/nQf8/t6fN4vk6f24f767Pt2bOn4z28TZcV8xycTndgqx0AAAydHAAgrdThyuHQWfcjnVlfSgwh+CoHOqz26NGjHa/hK7WsXr26q3v7/TRcOxpDcYHJzldKav1ftXYGbw231xVBfKV/bfPwnYYZfZi+vt+O9y7Q3cB9dRS9jocdn3/++VresmVLaNP3naeDVqxYUcseStW/d6TvML7kAABp0ckBANKikwMApEVO7k06/PZE3HbbbbXc19cX2gYGBmr5Rz/60ajcr0X/pi984Qtjfj8gO88LaV7Mc/man/fh9lr3Nl06S3fbLiUue+V5fV0CTJfR8mfxpboOHDgQ6p4jVJqT9Jzczp07a7m1dNhVV10V2vR389yl5u/0dxkOvuQAAGnRyQEA0kodrrz55ptD/c477xzze/rOB93ST/HWBouf/exnQ90//dX73//+ET0LgO7oxqhaLqU9TajTNUqJoUUdXl9KDC1u27YttGmYcd26daFNw6oecvUNTjWt4mmcF198seN5PT09tawbqJZSyvnnn1/LS5YsCW1Lly7teJ5Oi2i9F1v4kgMApEUnBwBIi04OAJDWFI8Hj7FxvZn75S9/Wcu+PE/LE088UcvDGfr/ta99LdTf8Y53dDz2Yx/7WC0vXLiw63ucgJEt6Q1MYgMDA+Edpvkt302gtXSXDpv3qQeay/NlvTZs2FDLvnSW3t+nJegyX7o011B0Ka25c+eGNs27zZw5M7TpuALf4Vt/i0WLFoU2nV7geT79ff13WrhwYVfvML7kAABp0ckBANKaVOFKBIQrgWHq7+8P7zAN0XmIsNsh7/4O1nChb4yqK5l4eFRDe746iD6bn+ehVF2txDdN9edRujpJa4UX19r1Rc/zqQ+9vb2EKwEAkxudHAAgLTo5AEBaqZf1AoDR5LtxK88Zaa5rOPk6vY7vENBankt31fZcmvIcoE+n0hyZP7fev7VLuuYVS4l5QL9/636KZb0AADB0cgCAtAhXAkCXhhO+0yH1w6HhvBkzZoQ2vYcPvdfzfKWUI0eOdDzPw556D/8bWhuc6tQDD+vqeT5lQUOZvuKJam3m2sKXHAAgLTo5AEBadHIAgLTIyQFAl4YzhUCPbS1z5bmt1m7YuqxWazkwX9ardZ7nujQP53+T3qOVd/N7aN3/pla+bjTwJQcASItODgCQFuFKAOhSa+USn0Iwbdq0WvbV+/U6rXCh7xjQ7YojrZ0GfJh+K5SqG7iWUsr06dNrWacM+P1b0wt8CoP+Nv7c2jY4OFhGgi85AEBadHIAgLTo5AAAaY33zuAAAIwbvuQAAGnRyQEA0qKTAwCkRScHAEiLTg4AkBadHAAgLTo5AEBadHIAgLTo5AAAadHJAQDSopMDAKRFJwcASItODgCQFp0cACAtOjkAQFp0cgCAtOjkAABp0ckBANKikwMApEUnBwBIi04OAJAWnRwAIC06OQBAWnRyAIC06OQAAGn9D6JHofvGtLG5AAAAAElFTkSuQmCC\n",
+      "text/plain": [
+       "<Figure size 720x288 with 4 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "show_reconstructed_digits(inputs, outputs, \"./model/my_model_sparse.ckpt\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 7. Variational AutoEncoder (VAE)\n",
+    "\n",
+    "**VAE**(Variational AutoEncoder)는 2014년 D.Kingma와 M.Welling이 [Auto-Encoding Variational Bayes](https://arxiv.org/pdf/1312.6114v10.pdf) 논문에서 제안한 오토인코더의 한 종류이다. VAE는 위에서 살펴본 오터인코더와는 다음과 같은 다른점이 있다.\n",
+    "\n",
+    "- VAE는 **확률적 오토인코더**(probabilistic autoencoder)다. 즉, 학습이 끝난 후에도 출력이 부분적으로 우연에 의해 결정된다.\n",
+    "- VAE는 **생성 오토인코더**(generatie autoencoder)이며, 학습 데이터셋에서 샘플링된 것과 같은 새로운 샘플을 생성할 수 있다.\n",
+    "\n",
+    "VAE의 구조는 아래의 그림과 같다.\n",
+    "\n",
+    "\n",
+    "\n",
+    "![](./images/vae.PNG)\n",
+    "\n",
+    "\n",
+    "\n",
+    "VAE의 코딩층은 다른 오토인코더와는 다른 부분이 있는데 주어진 입력에 대해 바로 코딩을 만드는 것이 아니라, 인코더(encoder)는 **평균 코딩** $\\mu$와 **표준편차 코딩** $\\sigma$ 을 만든다. 실제 코딩은 평균이 $\\mu$이고 표준편차가 $\\sigma$인 가우시안 분포(gaussian distribution)에서 랜덤하게 샘플링되며, 이렇게 샘플링된 코딩을 디코더(decoder)가 원본 입력으로 재구성하게 된다.\n",
+    "\n",
+    "VAE는 마치 가우시안 분포에서 샘플링된 것처럼 보이는 코딩을 만드는 경향이 있는데, 학습하는 동안 손실함수가 코딩(coding)을 가우시안 샘플들의 집합처럼 보이는 형태를 가진 코딩 공간(coding space) 또는 **잠재 변수 공간**(latent space)로 이동시키기 때문이다. \n",
+    "\n",
+    "이러한 이유로 VAE는 학습이 끝난 후에 새로운 샘플을 가우시안 분포로 부터 랜덤한 코딩을 샘플링해 디코딩해서 생성할 수 있다."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 7.1 VAE의 손실함수\n",
+    "\n",
+    "VAE의 손실함수는 두 부분으로 구성되어 있다. 첫 번째는 오토인코더가 입력을 재구성하도록 만드는 일반적인 재구성 손실(reconstruction loss)이고, 두 번째는 가우시안 분포에서 샘플된 것 샅은 코딩을 가지도록 오토인코더를 제어하는 **latent loss**이다. 이 손실함수의 식에 대해서는 [ratsgo](https://ratsgo.github.io/generative%20model/2018/01/27/VAE/)님의 블로그를 참고하면 자세히 설명되어 있다. *(나도 언젠가 이해해서 포스팅할 날이 오기를...)*"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 7.2 텐서플로 구현"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/Chap15-Autoencoders/Chap15-Autoencoders.md b/Chap15-Autoencoders/Chap15-Autoencoders.md
index 6e9c21f..2286a1c 100644
--- a/Chap15-Autoencoders/Chap15-Autoencoders.md
+++ b/Chap15-Autoencoders/Chap15-Autoencoders.md
@@ -262,3 +262,329 @@ epoch : 4, Train MSE : 0.01720
 
 ![](./images/stacked-ae04.PNG)
 
+
+
+- [단계 1]에서 첫 번째 오토인코더는 입력을 재구성하도록 학습된다.
+- [단계 2]에서는 두 번째 오토인코더가 첫 번째 히든 레이어(`Hidden 1`)의 출력을 재구성하도록 학습된다.
+
+- [단계 3]에서는 단계1 ~ 2의 오토인코더를 합쳐 최종적으로 하나의 stacked-오토인코더를 구현한다.
+
+
+
+텐서플로에서 이렇게 여러 단계의 오토인코더를 학습시키는 방법으로는 다음과 같이 두 가지 방법이 있다.
+
+-  각 단계마다 다른 텐서플로 그래프(graph)를 사용하는 방법
+- 하나의 그래프에 각 단계의 학습을 수행하는 방법
+
+위의 두 가지 방법에 대한 코드는 [ExcelsiorCJH's GitHub](https://github.com/ExcelsiorCJH/Hands-On-ML/blob/master/Chap15-Autoencoders/Chap15-Autoencoders.ipynb) 에서 확인할 수 있다.
+
+
+
+## 4. Stacked-오토인코더를 이용한 비지도 사전학습
+
+대부분이 레이블되어 있지 않는 데이터셋이 있을 때, 먼저 전체 데이터를 사용해 stacked-오토인코더를 학습시킨다. 그런 다음 오토인코더의 하위 레이어를 재사용해 분류와 같은 실제 문제를 해결하기 위한 신경망을 만들고 레이블된 데이터를 사용해 학습시킬 수 있다.
+
+
+
+![](./images/unsupervised.PNG)
+
+
+
+위와 같은 방법을 텐서플로에서 구현할 때는 [Transfer Learning](http://excelsior-cjh.tistory.com/179?category=940399)포스팅에서 살펴본 방법과 같이 구현하면 된다. 이러한 비지도 사전학습 방법에 대한 소스코드는 [여기](https://github.com/rickiepark/handson-ml/blob/master/15_autoencoders.ipynb)에서 확인할 수 있다.
+
+
+
+## 5. Denoising 오토인코더
+
+오토인코더가 의미있는 특성(feature)을 학습하도록 제약을 주는 다른 방법은 입력에 노이즈(noise, 잡음)를 추가하고, 노이즈가 없는 원본 입력을 재구성하도록 학습시키는 것이다. 노이즈는 아래의 그림처럼 입력에 [가우시안(Gaussian) 노이즈](https://en.wikipedia.org/wiki/Gaussian_noise)를 추가하거나, 드롭아웃(dropout)처럼 랜덤하게 입력 유닛(노드)를 꺼서 발생 시킬 수 있다.
+
+
+
+![denoising-autoencoder](./images/denoising.PNG)
+
+
+
+### 5.1 텐서플로로 구현하기
+
+이번에는 텐서플로를 이용해 가우시안 노이즈와 드롭아웃을 이용한 denoising-오토인코더를 구현해보도록 하자. 오토인코더 학습에 사용한 데이터셋은 위에서 부터 다뤘던 MNIST 데이터셋이다.  아래의 코드에 대한 전체 코드는 [ExcelsiorCJH's GitHub](https://github.com/ExcelsiorCJH/Hands-On-ML/blob/master/Chap15-Autoencoders/Chap15-Autoencoders.ipynb)에서 확인할 수 있다.
+
+#### 5.1.1 Gaussian noise
+
+```python
+import sys
+import numpy as np
+import tensorflow as tf
+
+################
+# layer params #
+################
+noise_level = 1.0
+n_inputs = 28 * 28
+n_hidden1 = 300
+n_hidden2 = 150  # coding units
+n_hidden3 = n_hidden1
+n_outputs = n_inputs
+
+################
+# train params #
+################
+learning_rate = 0.01
+n_epochs = 5
+batch_size = 150
+
+# denoising autoencoder
+inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])
+# add gaussian noise
+inputs_noisy = inputs + noise_level * tf.random_normal(tf.shape(inputs))
+
+hidden1 = tf.layers.dense(inputs_noisy, n_hidden1, activation=tf.nn.relu, name='hidden1')
+hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name='hidden2')
+hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name='hidden3')
+outputs = tf.layers.dense(hidden3, n_outputs, name='outputs')
+
+# loss 
+reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)
+# optimizer
+train_op = tf.train.AdamOptimizer(learning_rate).minimize(reconstruction_loss)
+
+# saver
+saver = tf.train.Saver()
+
+# Train
+with tf.Session() as sess:
+    tf.global_variables_initializer().run()
+    n_batches = len(train_x) // batch_size
+    for epoch in range(n_epochs):
+        for iteration in range(n_batches):
+            print("\r{}%".format(100 * iteration // n_batches), end="")
+            sys.stdout.flush()
+            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))
+            sess.run(train_op, feed_dict={inputs: batch_x})
+        loss_train = reconstruction_loss.eval(feed_dict={inputs: batch_x})
+        print('\repoch : {}, Train MSE : {:.5f}'.format(epoch, loss_train))
+    saver.save(sess, './model/my_model_stacked_denoising_gaussian.ckpt')
+    
+'''
+epoch : 0, Train MSE : 0.04450
+epoch : 1, Train MSE : 0.04073
+epoch : 2, Train MSE : 0.04273
+epoch : 3, Train MSE : 0.04194
+epoch : 4, Train MSE : 0.04084
+'''
+```
+
+
+
+위의 가우시안 노이즈를 추가한 denoising-오토인코더의 MNIST 재구성 결과는 다음과 같다.
+
+![](./images/denoising02.PNG)
+
+
+
+#### 5.1.2 Dropout
+
+```python
+import sys
+import numpy as np
+import tensorflow as tf
+
+################
+# layer params #
+################
+noise_level = 1.0
+n_inputs = 28 * 28
+n_hidden1 = 300
+n_hidden2 = 150  # coding units
+n_hidden3 = n_hidden1
+n_outputs = n_inputs
+
+################
+# train params #
+################
+dropout_rate = 0.3
+learning_rate = 0.01
+n_epochs = 5
+batch_size = 150
+
+training = tf.placeholder_with_default(False, shape=(), name='training')
+
+# denoising autoencoder
+inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])
+# add dropout
+inputs_drop = tf.layers.dropout(inputs, dropout_rate, training=training)
+
+hidden1 = tf.layers.dense(inputs_drop, n_hidden1, activation=tf.nn.relu, name='hidden1')
+hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name='hidden2')
+hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name='hidden3')
+outputs = tf.layers.dense(hidden3, n_outputs, name='outputs')
+
+# loss 
+reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)
+# optimizer
+train_op = tf.train.AdamOptimizer(learning_rate).minimize(reconstruction_loss)
+
+# saver
+saver = tf.train.Saver()
+
+# Train
+with tf.Session() as sess:
+    tf.global_variables_initializer().run()
+    n_batches = len(train_x) // batch_size
+    for epoch in range(n_epochs):
+        for iteration in range(n_batches):
+            print("\r{}%".format(100 * iteration // n_batches), end="")
+            sys.stdout.flush()
+            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))
+            sess.run(train_op, feed_dict={inputs: batch_x})
+        loss_train = reconstruction_loss.eval(feed_dict={inputs: batch_x})
+        print('\repoch : {}, Train MSE : {:.5f}'.format(epoch, loss_train))
+    saver.save(sess, './model/my_model_stacked_denoising_dropout.ckpt')
+```
+
+![](./images/denoising03.PNG)
+
+
+
+
+
+## 6. Sparse 오토인코더
+
+오토인코더가 좋은 특성을 추출하도록 만드는 다른 제약 방법은 **희소성**(sparsity)를 이용하는 것인데, 이러한 오토인코더를 Sparse Autoencoder라고 한다. 이 방법은 손실함수에 적절한 항을 추가하여 오토인코더가 코딩층(coding layer, 가운데 층)에서 활성화되는 뉴런 수를 감소시키는 것이다. 예를들어 코딩층에서 평균적으로 5% 뉴런만 홀성화되도록 만들어 주게 되면, 오토인코더는 5%의 뉴런을 조합하여 입력을 재구성해야하기 때문에 유용한 특성을 표현하게 된다.
+
+이러한 Sparse-오토인코더를 만들기 위해서는 먼저 학습 단계에서 코딩층의 실제 sparse(희소) 정도를 측정해야 하는데, 전체 학습 배치(batch)에 대해 코딩층의 평균적인 활성화를 계산한다. 배치의 크기는 너무 작지 않게 설정 해준다. 
+
+위에서 각 뉴런에 대한 평균 활성화 정도를 계산하여 구하고, 손실함수에 **희소 손실**(sparsity loss)를 추가하여 뉴런이 크게 활성화 되지 않도록 규제할 수 있다.  예를들어 한 뉴런의 평균 활성화가 `0.3`이고 목표 희소 정도가 `0.1`이라면, 이 뉴런은 **덜** 활성화 되도록 해야한다. 희소 손실을 구하는 간단한 방법으로는 제곱 오차 $(0.3 - 0.1)^{2}$를 추가하는 방법이 있다. 하지만, Sparse-오토인코더에서는 아래의 그래프 처럼 MSE보다 더 경사가 급한 쿨백 라이블러 발산(KL-divergense, Kullback-Leibler divergense)을 사용한다.
+
+
+
+![](./images/kl.PNG)
+
+
+
+### 6.1 쿨백 라이블러 발산
+
+쿨백-라이블러 발산(Kullback-Leibler divergence, KLD)은 두 확률분포의 차이를 계산하는 데 사용하는 함수이다. 예를들어 딥러닝 모델을 만들 때, 학습 데이터셋의 분포 $P(x)$와 모델이 추정한 데이터의 분포 $Q(x)$ 간에 차이를 KLD를 활용해 구할 수 있다([ratsgo's blog](https://ratsgo.github.io/statistics/2017/09/22/information/)).
+
+
+
+$$
+{ D }_{ KL }\left( P||Q \right) ={ E }_{ X\sim P }\left[ \log { \frac { P\left( x \right)  }{ Q(x) }  }  \right] ={ E }_{ X\sim P }\left[ \log { P(x) } -\log { Q(x) }  \right]
+$$
+
+
+
+Sparse-오토인코더에서는 코딩층에서 뉴런이 활성화될 목표 확률 $p$와 실제확률 $q$(학습 배치에 대한 평균 활성화) 사이의 발산을 측정하며, 식은 다음과 같다.
+
+
+$$
+D_{KL}\left( p||q \right) = p \log{\frac{p}{q}} + \left( 1- p \right) \log{\frac{1-p}{1-q}}
+$$
+
+
+위의 식을 이용해 코딩층의 각 뉴런에 대해 희소 손실을 구하고 이 손실을 모두 합한 뒤 희소 가중치 하이퍼파라미터를 곱하여 손실함수의 결과에 더해준다.
+$$
+Loss = \text{MSE} + \text{sparsity_weight} \times \text{sparsity_loss}
+$$
+
+
+### 6.2 텐서플로 구현
+
+이번에는 텐서플로를 이용해 Sparse-오토인코더를 구현해보도록 하자. 
+
+```python
+import sys
+import numpy as np
+import tensorflow as tf
+
+################
+# layer params #
+################
+noise_level = 1.0
+n_inputs = 28 * 28
+n_hidden1 = 1000 # sparsity coding units
+n_outputs = n_inputs
+
+################
+# train params #
+################
+sparsity_target = 0.1  # p
+sparsity_weight = 0.2
+learning_rate = 0.01
+n_epochs = 20
+batch_size = 1000
+
+def kl_divergence(p, q):
+    # 쿨백 라이블러 발산
+    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))
+
+inputs = tf.placeholder(tf.float32, shape=[None, n_inputs])
+
+hidden1 = tf.layers.dense(inputs, n_hidden1, activation=tf.nn.sigmoid)
+outputs = tf.layers.dense(hidden1, n_outputs)
+
+# loss
+hidden1_mean = tf.reduce_mean(hidden1, axis=0)  # 배치 평균  == q
+sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))
+reconstruction_loss = tf.losses.mean_squared_error(labels=inputs, predictions=outputs)
+loss = reconstruction_loss + sparsity_weight * sparsity_loss
+
+# optimizer
+train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)
+
+# saver
+saver = tf.train.Saver()
+
+# Train
+with tf.Session() as sess:
+    tf.global_variables_initializer().run()
+    n_batches = len(train_x) // batch_size
+    for epoch in range(n_epochs):
+        for iteration in range(n_batches):
+            print("\r{}%".format(100 * iteration // n_batches), end="")
+            sys.stdout.flush()
+            batch_x, batch_y = next(shuffle_batch(train_x, train_y, batch_size))
+            sess.run(train_op, feed_dict={inputs: batch_x})
+        recon_loss_val, sparsity_loss_val, loss_val = sess.run([reconstruction_loss, 
+                                                                sparsity_loss,
+                                                                loss], feed_dict={inputs: batch_x})
+        print('\repoch : {}, Train MSE : {:.5f}, \
+                sparsity_loss : {:.5f}, total_loss : {:.5f}'.format(epoch, recon_loss_val,
+                                                                    sparsity_loss_val, loss_val))
+    saver.save(sess, './model/my_model_sparse.ckpt')
+```
+
+![](./images/sparse.PNG)
+
+
+
+## 7. Variational AutoEncoder (VAE)
+
+**VAE**(Variational AutoEncoder)는 2014년 D.Kingma와 M.Welling이 [Auto-Encoding Variational Bayes](https://arxiv.org/pdf/1312.6114v10.pdf) 논문에서 제안한 오토인코더의 한 종류이다. VAE는 위에서 살펴본 오터인코더와는 다음과 같은 다른점이 있다.
+
+- VAE는 **확률적 오토인코더**(probabilistic autoencoder)다. 즉, 학습이 끝난 후에도 출력이 부분적으로 우연에 의해 결정된다.
+- VAE는 **생성 오토인코더**(generatie autoencoder)이며, 학습 데이터셋에서 샘플링된 것과 같은 새로운 샘플을 생성할 수 있다.
+
+VAE의 구조는 아래의 그림과 같다.
+
+
+
+![](./images/vae.PNG)
+
+
+
+VAE의 코딩층은 다른 오토인코더와는 다른 부분이 있는데 주어진 입력에 대해 바로 코딩을 만드는 것이 아니라, 인코더(encoder)는 **평균 코딩** $\mu$와 **표준편차 코딩** $\sigma$ 을 만든다. 실제 코딩은 평균이 $\mu$이고 표준편차가 $\sigma$인 가우시안 분포(gaussian distribution)에서 랜덤하게 샘플링되며, 이렇게 샘플링된 코딩을 디코더(decoder)가 원본 입력으로 재구성하게 된다.
+
+VAE는 마치 가우시안 분포에서 샘플링된 것처럼 보이는 코딩을 만드는 경향이 있는데, 학습하는 동안 손실함수가 코딩(coding)을 가우시안 샘플들의 집합처럼 보이는 형태를 가진 코딩 공간(coding space) 또는 **잠재 변수 공간**(latent space)로 이동시키기 때문이다. 
+
+이러한 이유로 VAE는 학습이 끝난 후에 새로운 샘플을 가우시안 분포로 부터 랜덤한 코딩을 샘플링해 디코딩해서 생성할 수 있다.
+
+
+
+### 7.1 VAE의 손실함수
+
+VAE의 손실함수는 두 부분으로 구성되어 있다. 첫 번째는 오토인코더가 입력을 재구성하도록 만드는 일반적인 재구성 손실(reconstruction loss)이고, 두 번째는 가우시안 분포에서 샘플된 것 샅은 코딩을 가지도록 오토인코더를 제어하는 **latent loss**이다. 이 손실함수의 식에 대해서는 [ratsgo](https://ratsgo.github.io/generative%20model/2018/01/27/VAE/)님의 블로그를 참고하면 자세히 설명되어 있다. *(나도 언젠가 이해해서 포스팅할 날이 오기를...)*
+
+
+
+### 7.2 텐서플로 구현
+
diff --git a/Chap15-Autoencoders/images/denoising.PNG b/Chap15-Autoencoders/images/denoising.PNG
new file mode 100644
index 0000000..d1dd676
Binary files /dev/null and b/Chap15-Autoencoders/images/denoising.PNG differ
diff --git a/Chap15-Autoencoders/images/denoising02.PNG b/Chap15-Autoencoders/images/denoising02.PNG
new file mode 100644
index 0000000..9ec1321
Binary files /dev/null and b/Chap15-Autoencoders/images/denoising02.PNG differ
diff --git a/Chap15-Autoencoders/images/denoising03.PNG b/Chap15-Autoencoders/images/denoising03.PNG
new file mode 100644
index 0000000..21505d6
Binary files /dev/null and b/Chap15-Autoencoders/images/denoising03.PNG differ
diff --git a/Chap15-Autoencoders/images/kl.PNG b/Chap15-Autoencoders/images/kl.PNG
new file mode 100644
index 0000000..999d908
Binary files /dev/null and b/Chap15-Autoencoders/images/kl.PNG differ
diff --git a/Chap15-Autoencoders/images/sparse.PNG b/Chap15-Autoencoders/images/sparse.PNG
new file mode 100644
index 0000000..23576dc
Binary files /dev/null and b/Chap15-Autoencoders/images/sparse.PNG differ
diff --git a/Chap15-Autoencoders/images/stacked-ae04.PNG b/Chap15-Autoencoders/images/stacked-ae04.PNG
index e6354c2..3643c7b 100644
Binary files a/Chap15-Autoencoders/images/stacked-ae04.PNG and b/Chap15-Autoencoders/images/stacked-ae04.PNG differ
diff --git a/Chap15-Autoencoders/images/stacked-ae05.PNG b/Chap15-Autoencoders/images/stacked-ae05.PNG
new file mode 100644
index 0000000..477527a
Binary files /dev/null and b/Chap15-Autoencoders/images/stacked-ae05.PNG differ
diff --git a/Chap15-Autoencoders/images/unsupervised.PNG b/Chap15-Autoencoders/images/unsupervised.PNG
new file mode 100644
index 0000000..0e9d94a
Binary files /dev/null and b/Chap15-Autoencoders/images/unsupervised.PNG differ
diff --git a/Chap15-Autoencoders/images/vae.PNG b/Chap15-Autoencoders/images/vae.PNG
new file mode 100644
index 0000000..dfb5884
Binary files /dev/null and b/Chap15-Autoencoders/images/vae.PNG differ