reinforcement_q_learning_pole_simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solve cartpole with DQN\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple\n",
    "from itertools import count\n",
    "from PIL import Image\n",
    "import time\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "\n",
    "\n",
    "env = gym.make('CartPole-v0').unwrapped\n",
    "\n",
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Transition = namedtuple('Transition',\n",
    "                        ('state', 'action', 'next_state', 'reward'))\n",
    "\n",
    "\n",
    "class ReplayMemory(object):\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.position = 0\n",
    "\n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition.\"\"\"\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)\n",
    "        self.memory[self.position] = Transition(*args)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0694, -0.0100]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class DQN(nn.Module):\n",
    "\n",
    "    def __init__(self, h, w, outputs):\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 8, kernel_size=3, stride=1)\n",
    "        #self.bn1 = nn.BatchNorm2d(8)\n",
    "        self.conv2 = nn.Conv2d(8, 16, kernel_size=3, stride=1)\n",
    "        #self.bn2 = nn.BatchNorm2d(16)\n",
    "        #self.conv3 = nn.Conv2d(32, 32, kernel_size=3, stride=1)\n",
    "        #self.bn3 = nn.BatchNorm2d(32)\n",
    "        self.conv4 = nn.Conv2d(16, 4, kernel_size=3, stride=1)\n",
    "        #self.bn4 = nn.BatchNorm2d(4)\n",
    "        \n",
    "        self.pool = nn.MaxPool2d(2)\n",
    "\n",
    "        self.head = nn.Linear(204, outputs)\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = F.relu(self.pool(self.conv1(x)))\n",
    "        x = F.relu(self.pool(self.conv2(x)))\n",
    "        x = F.relu(self.pool(self.conv4(x)))\n",
    "        return self.head(x.view(x.size(0), -1))\n",
    "    \n",
    "    def update_params(self, new_params, tau):\n",
    "        params = self.state_dict()\n",
    "        for k in params.keys():\n",
    "            params[k] = (1-tau) * params[k] + tau * new_params[k]\n",
    "        self.load_state_dict(params)\n",
    "        \n",
    "class Flatten(torch.nn.Module):\n",
    "    def forward(self, x):\n",
    "        batch_size = x.shape[0]\n",
    "        return x.view(batch_size, -1)\n",
    "\n",
    "\n",
    "class DQN_basic(nn.Module):\n",
    "    def __init__(self, h, w, outputs):\n",
    "        super(DQN_basic, self).__init__()\n",
    "        self.l1 = nn.Linear(4, 20)\n",
    "        self.head = nn.Linear(20, outputs)\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        return self.head(x)\n",
    "    \n",
    "    def update_params(self, new_params, tau):\n",
    "        params = self.state_dict()\n",
    "        for k in params.keys():\n",
    "            params[k] = (1-tau) * params[k] + tau * new_params[k]\n",
    "        self.load_state_dict(params)\n",
    "\n",
    "test = DQN(1, 1, 2)\n",
    "test(torch.zeros([1, 3, 40, 150]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAACICAYAAAD+r7D/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAARwklEQVR4nO3df5RcZX3H8fcnu8kmgYX8hsAGFzDyy4OEppBU21KEEqiCPbUFahUsLbWlB2hB5Mc5LVbOqSgC7dEiURALFMWACKmKGEItpxpICIFIiAmIZCGQjRIIvzbZ7Ld/3GeXm92Z3dns7szc7Od1zj1z73Ofufc7z8w888xzfzyKCMzMrHjG1DoAMzPbNa7AzcwKyhW4mVlBuQI3MysoV+BmZgXlCtzMrKBcgVvVSTpb0sO1jqOeSGqVFJIaax2LFYcr8N2MpOckvSXp9dz05VrHVWuSjpPUNoLbv1LSbSO1fbNS/Gu/e/pwRPy41kEUjaTGiOisdRwjYXd+baOZW+CjiKQbJC3KLV8taYkykyUtltQu6ZU035LL+5CkqyT9X2rV3ydpqqTbJb0m6VFJrbn8Iel8Sc9K2izpi5JKft4kHSrpAUm/kbRW0p/18xr2lnSTpI2SXkgxNQzw+vYAfgDsl/tXsl9qNS+SdJuk14CzJR0j6aeStqR9fFnSuNw2j8jF+rKkyyUtAC4HTk/bXlVBrA2Srkll8yzwRwO8d59J29iayuiDue1cLumZtG6FpFm59+A8SeuAdQOVtaSmFNPz6bV9VdKEtO44SW2SLpK0Kb2mT/YXs1VBRHjajSbgOeCEMusmAr8AzgZ+F9gMtKR1U4E/SXmage8A9+Se+xCwHjgY2Bt4Km3rBLJ/cv8JfCOXP4ClwBTggJT3r9K6s4GH0/wewAbgk2k7R6e4jijzGu4BbkzPmwE8AvxNBa/vOKCt17auBLYDHyFrzEwAfguYl2JpBdYAF6b8zcBG4CJgfFo+Nret2wYR66eAp4FZqYyWpjJrLPGaD0lltF9abgUOTvOfBp5MeQS8D5iaew8eSNufMFBZA9cD96b8zcB9wL/myq8T+BdgLHAK8CYwudaf+dE81TwAT8P8hmYV+OvAltz017n1xwC/AX4FnNnPdo4CXsktPwRckVv+EvCD3PKHgcdzywEsyC3/HbAkzZ/NOxX46cD/9tr3jcA/l4hpH6ADmJBLOxNYOtDro3wF/pMByvNC4Lu5fa0sk+9KchX4QLECDwKfyq37Q8pX4O8GNpH9WI7ttW4tcFqZmAI4PrdctqzJKv83SD8Mad184Je58nsrH1+KaV6tP/OjeXIf+O7pI1GmDzwiHkl/2WcAd3anS5oIXAcsACan5GZJDRGxIy2/nNvUWyWW9+y1uw25+V8B+5UI6V3AsZK25NIagVvL5B0LbJTUnTYmv59yr68f+RiR9B7gWmAuWYu+EViRVs8Cnqlgm5XEuh99y6ekiFgv6UKyH4kjJN0P/GNEvFhBTPl99FfW08le74pcvAIacnl/HTv3o79J3/fcqsh94KOMpPOAJuBF4JLcqovI/oYfGxF7Ab/X/ZQh7G5Wbv6AtM/eNgD/ExGTctOeEfG3ZfJ2ANNyefeKiCO6M/Tz+srddrN3+g1kXRuzUzlczjtlsIGsC6mS7QwU60b6lk9ZEfFfEfEBsko4gKsriKl3XP2V9WayH+Ejcuv2jghX0HXMFfgoklqXVwF/AXwcuETSUWl1M9kXeIukKWR/q4fq0+ng6CzgAuDbJfIsBt4j6eOSxqbptyUd1jtjRGwEfgR8SdJeksZIOljS71fw+l4Gpkrae4CYm4HXgNclHQrkf0gWA/tKujAd8GuWdGxu+63dB2oHipXs38H5klokTQYuLReQpEMkHS+pCXib7H3q/lf0deBzkmYrc6SkqWU2VbasI6IL+BpwnaQZab/7SzppgPKyGnIFvnu6TzufB/5dZReI3AZcHRGrImIdWevy1lQxXE92oGsz8DPgh8MQx/fIuh8eB/4buKl3hojYStb/ewZZq/klstZlU5ltfgIYR3YQ9RVgETBzoNcXEU8DdwDPpjNMSnXnAFwM/DmwlaxC6/nRSbGeSNbf/xLZmR1/kFZ/Jz3+WtJj/cWa1n0NuB9YBTwG3F0mHlJZfJ7svXmJrHvo8rTuWrIfgx+R/fDcRPY+9lFBWX+G7ED1z9JZOT8m+1dmdUoRHtDBhp+kIOuGWF/rWMx2V26Bm5kVlCtwM7OCGlIFLmlBupprvaSyB2Fs9IkIufvEbGTtch94uiT4F2QHddqAR8kunHhq+MIzM7NyhtICPwZYHxHPRsQ24FvAacMTlpmZDWQoV2Luz85XebUBx5bJC8C0adOitbV1CLs0Mxt9VqxYsTkipvdOH0oFXuoKvT79MZLOBc4FOOCAA1i+fPkQdmlmNvpIKnmrhaF0obSx86XALZS4VDoiFkbE3IiYO316nx8QMzPbRUOpwB8FZks6UNn9ks8guxWlWfFE9EzRtYPo2jHwc8xqbJe7UCKiU9Lfk10O3ADcHBE/H7bIzMysX0O6nWxEfB/4/jDFYmZmg+D7gduo9Obm7JjQhp9m96Dq2t7Rs27W+08HYM993l39wMwGwZfSm5kVlFvgNip1H6Tc2rYGgK4d23rWdWzZBLgFbvXPLXAzs4JyC9xGpaa99gFg7B6TAOh4rb1nXcfWzTWJyWyw3AI3Mysot8BtlCp/F840rKVZ3fMn1cysoFyBm5kVlCtwM7OCcgVuZlZQrsDNzArKFbiZWUH5NEIbnaT00LcN43uBW1EM2AKXdLOkTZJW59KmSHpA0rr0OHlkwzQzs94q6UK5BVjQK+1SYElEzAaWpGWzwmgcN5HGcRNpmrQvTZP23WndG+2/4o32kkMQmtWVASvwiPgJ8JteyacB30zz3wQ+MsxxmZnZAHb1IOY+EbERID3OKJdR0rmSlkta3t7eXi6bmZkN0oifheJR6a0uSSAhjelzINODGltR7GoF/rKkmQDpcdPwhWRmZpXY1Qr8XuCsNH8W8L3hCcfMzCpVyWmEdwA/BQ6R1CbpHODzwImS1gEnpmUzM6uiAS/kiYgzy6z64DDHYmZmg+BL6c3MCsqX0tuoFtHVNzFdZm9W79wCNzMrKFfgZmYF5S4UG9XG75VdXPZqLq3zzWypq3MbAGMax1U7LLOKuAVuZlZQboHbqNbUPK1P2vaeFngH4Ba41S+3wM3MCsotcBvV+j+N0KcTWn1zC9zMrKBcgZuZFZQrcDOzgnIFbmZWUJXcTnaWpKWS1kj6uaQLUrpHprfii8imUtKoPWb1qpIWeCdwUUQcBswDzpN0OB6Z3syspioZlX5jRDyW5rcCa4D98cj0thtoaJpIQ9PEnrExpTHEjk5iRydd296ma9vbtQ7RrKxB9YFLagXmAMuocGR6j0pvZjYyKr6QR9KewF3AhRHxmirsG4yIhcBCgLlz55bpbDSrjQnTWgBQwztfhc6ONwDo2JqN1T2ueWr1AzOrQEUtcEljySrv2yPi7pTskenNzGqokrNQBNwErImIa3OrPDK9FV9/Z6EgfDm91bNKulDeD3wceFLS4yntcrKR6O9Mo9Q/D/zpyIRoZmalVDIq/cOUb4Z4ZHozsxrxlZhmZgXlCtzMrKBcgZuZFZQrcDOzgvKIPDbK9XNtmW9kZXXOLXAzs4JyC9xGtabm7BY+jU179KR1j0r/1uY2AJpnHlL9wMwq4Ba4mVlBuQI3Mysod6HYqDZm3Hhg57sRRnQBsKPjzZrEZFYpt8DNzArKLXAb3creiRCfRmh1zy1wM7OCquR+4OMlPSJpVRqV/rMp/UBJy9Ko9N+WNG7kwzUzs26VtMA7gOMj4n3AUcACSfOAq4Hr0qj0rwDnjFyYZmbWWyWj0kdEvJ4Wx6YpgOOBRSndo9JbITWOHUvj2LGMkXom0ZVNEpWO/WpWC5WOidmQRuPZBDwAPANsiYjOlKUN2H9kQjQzs1IqqsAjYkdEHAW0AMcAh5XKVuq5ks6VtFzS8vb29l2P1MzMdjKo0wgjYoukh4B5wCRJjakV3gK8WOY5C4GFAHPnzu3nnC2z4XXxxRf3zK9cubJknqbGrIvk/BNbetJmTJ4OwA/vuwuAWz99bd8n9jJnzhwArrnmml0L1mwXVHIWynRJk9L8BOAEYA2wFPhoyuZR6c3MqqySFvhM4JuSGsgq/DsjYrGkp4BvSboKWAncNIJxmg3asmXLeuYffvjhfvMe1PK5d+YPORWADc/fCcCDD35jwH1t27ZtV0I0G5JKRqV/AphTIv1Zsv5wMzOrAV9Kb7utpqamivN2jZmSm98bgG2aUi77kPZlNlx8Kb2ZWUFVtQW+fft2Nm7cWM1d2ijW0dFRcd5nVy/smV/9xD0AtL3w1KD35c+3VZNb4GZmBeUK3MysoKrahdLZ2YmvxrRqGcypfff/bNWw7Mufb6smt8DNzAqqqi3wCRMmcOSRR1ZzlzaKNTc3V31f/nxbNbkFbmZWUL6Qx3ZbgzmNsEj7MuvmFriZWUG5BW67rfnz5/fMjx8/fkT31X07WbNqcgvczKygXIGbmRWUu1Bst/WFL3yh1iGYjSi3wM3MCkoR1RumUlI78AawuWo7HR7TKFbMRYsXHHM1FC1eKF7MIxXvuyJieu/EqlbgAJKWR8Tcqu50iIoWc9HiBcdcDUWLF4oXc7XjdReKmVlBuQI3MyuoWlTgCwfOUneKFnPR4gXHXA1FixeKF3NV4616H7iZmQ0Pd6GYmRVU1SpwSQskrZW0XtKl1drvYEiaJWmppDWSfi7pgpQ+RdIDktalx8m1jjVPUoOklZIWp+UDJS1L8X5b0rhax5gnaZKkRZKeTmU9vwBl/A/pM7Fa0h2SxtdbOUu6WdImSatzaSXLVZl/T9/HJyQdXSfxfjF9Lp6Q9F1Jk3LrLkvxrpV0UrXjLRdzbt3FkkLStLQ84mVclQpcUgPwFeBk4HDgTEmHV2Pfg9QJXBQRhwHzgPNSnJcCSyJiNrAkLdeTC4A1ueWrgetSvK8A59QkqvL+DfhhRBwKvI8s9rotY0n7A+cDcyPivUADcAb1V863AAt6pZUr15OB2Wk6F7ihSjHm3ULfeB8A3hsRRwK/AC4DSN/DM4Aj0nP+I9Ur1XYLfWNG0izgROD5XPLIl3FEjPgEzAfuzy1fBlxWjX0PMe7vpTdlLTAzpc0E1tY6tlyMLWRfzOOBxYDILiRoLFX2tZ6AvYBfko6/5NLruYz3BzYAU8huP7EYOKkeyxloBVYPVK7AjcCZpfLVMt5e6/4YuD3N71RnAPcD8+uhjFPaIrLGyHPAtGqVcbW6ULq/AN3aUlrdktQKzAGWAftExEaA9DijdpH1cT1wCdCVlqcCWyKiMy3XW1kfBLQD30jdPl+XtAd1XMYR8QJwDVnraiPwKrCC+i7nbuXKtQjfyb8EfpDm6zZeSacCL0RE75GxRzzmalXgKpFWt6e/SNoTuAu4MCJeq3U85Uj6ELApIlbkk0tkraeybgSOBm6IiDlkt1aom+6SUlK/8WnAgcB+wB5kf497q6dyHkhdf04kXUHWpXl7d1KJbDWPV9JE4Argn0qtLpE2rDFXqwJvA2bllluAF6u070GRNJas8r49Iu5OyS9LmpnWzwQ21Sq+Xt4PnCrpOeBbZN0o1wOTJHXfabLeyroNaIuIZWl5EVmFXq9lDHAC8MuIaI+I7cDdwO9Q3+XcrVy51u13UtJZwIeAj0Xqe6B+4z2Y7Id9VfoetgCPSdqXKsRcrQr8UWB2Omo/juxgxL1V2nfFJAm4CVgTEdfmVt0LnJXmzyLrG6+5iLgsIloiopWsTB+MiI8BS4GPpmx1Ey9ARLwEbJB0SEr6IPAUdVrGyfPAPEkT02ekO+a6LeeccuV6L/CJdKbEPODV7q6WWpK0APgMcGpEvJlbdS9whqQmSQeSHRh8pBYx5kXEkxExIyJa0/ewDTg6fc5Hvoyr2PF/CtlR5WeAK2px8KGCGD9A9hfnCeDxNJ1C1q+8BFiXHqfUOtYSsR8HLE7zB5F9uNcD3wGaah1fr1iPApancr4HmFzvZQx8FngaWA3cCjTVWzkDd5D10W8nq0jOKVeuZH/vv5K+j0+SnWFTD/GuJ+s37v7+fTWX/4oU71rg5Hop417rn+Odg5gjXsa+EtPMrKB8JaaZWUG5AjczKyhX4GZmBeUK3MysoFyBm5kVlCtwM7OCcgVuZlZQrsDNzArq/wHOf7kyBtUKqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "\n",
    "\n",
    "def get_cart_location(screen_width):\n",
    "    world_width = env.x_threshold * 2\n",
    "    scale = screen_width / world_width\n",
    "    return int(env.state[0] * scale + screen_width / 2.0)  # MIDDLE OF CART\n",
    "\n",
    "def get_screen():\n",
    "    # Returned screen requested by gym is 400x600x3, but is sometimes larger\n",
    "    # such as 800x1200x3. Transpose it into torch order (CHW).\n",
    "    screen = env.render(mode='rgb_array').transpose((2, 0, 1))\n",
    "    # Cart is in the lower half, so strip off the top and bottom of the screen\n",
    "    _, screen_height, screen_width = screen.shape\n",
    "    screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)]\n",
    "    view_width = int(screen_width * 0.6)\n",
    "    cart_location = get_cart_location(screen_width)\n",
    "    #print(cart_location)\n",
    "    #if cart_location < view_width // 2:\n",
    "    #    slice_range = slice(view_width)\n",
    "    #elif cart_location > (screen_width - view_width // 2):\n",
    "    #    slice_range = slice(-view_width, None)\n",
    "    #else:\n",
    "    #    slice_range = slice(cart_location - view_width // 2,\n",
    "    #                        cart_location + view_width // 2)\n",
    "    # Strip off the edges, so that we have a square image centered on a cart\n",
    "    # screen = screen[:, :, slice_range]\n",
    "    # Convert to float, rescale, convert to torch tensor\n",
    "    # (this doesn't require a copy)\n",
    "    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "    screen = torch.from_numpy(screen)\n",
    "    # Resize, and add a batch dimension (BCHW)\n",
    "    return resize(screen).unsqueeze(0).to(device)\n",
    "\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(get_screen().cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "plt.title('Example extracted screen')\n",
    "plt.show()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "GAMMA = 0.75\n",
    "EPS_START = 0.9\n",
    "EPS_END = 0.05\n",
    "EPS_DECAY = 200\n",
    "TARGET_UPDATE = 10\n",
    "tau = 0.01\n",
    "MEM_SIZE = 40000\n",
    "learning_rate = 0.001\n",
    "past = 3\n",
    "\n",
    "# Get screen size so that we can initialize layers correctly based on shape\n",
    "# returned from AI gym. Typical dimensions at this point are close to 3x40x90\n",
    "# which is the result of a clamped and down-scaled render buffer in get_screen()\n",
    "init_screen = get_screen()\n",
    "_, _, screen_height, screen_width = init_screen.shape\n",
    "\n",
    "# Get number of actions from gym action space\n",
    "n_actions = env.action_space.n\n",
    "\n",
    "policy_net = DQN_basic(screen_height, screen_width, n_actions).to(device)\n",
    "target_net = DQN_basic(screen_height, screen_width, n_actions).to(device)\n",
    "target_net.load_state_dict(policy_net.state_dict())\n",
    "target_net.eval()\n",
    "\n",
    "optimizer = optim.RMSprop(policy_net.parameters(), lr=learning_rate, weight_decay=1e-5)\n",
    "memory = ReplayMemory(MEM_SIZE)\n",
    "\n",
    "\n",
    "steps_done = 0\n",
    "\n",
    "eps = []\n",
    "\n",
    "def select_action(state, eps_max=1):\n",
    "    global steps_done\n",
    "    sample = random.random()\n",
    "    eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n",
    "        math.exp(-1. * steps_done / EPS_DECAY)\n",
    "    steps_done += 1\n",
    "    eps_threshold = min(eps_max, eps_threshold)\n",
    "    eps.append(eps_threshold)\n",
    "    if sample > eps_threshold:\n",
    "        with torch.no_grad():\n",
    "            # t.max(1) will return largest column value of each row.\n",
    "            # second column on max result is index of where max element was\n",
    "            # found, so we pick action with the larger expected reward.\n",
    "            return policy_net(state).max(0)[1].view(1,1)\n",
    "    else:\n",
    "        return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)\n",
    "\n",
    "\n",
    "episode_durations = []\n",
    "\n",
    "\n",
    "def plot_durations():\n",
    "    durations_t = torch.tensor(episode_durations, dtype=torch.float)\n",
    "    plt.title('Training...')\n",
    "    plt.xlabel('Episode')\n",
    "    plt.ylabel('Duration')\n",
    "    plt.plot(durations_t.numpy())\n",
    "    # Take 100 episode averages and plot them too\n",
    "    if len(durations_t) >= 100:\n",
    "        means = durations_t.unfold(0, 100, 1).mean(1).view(-1)\n",
    "        means = torch.cat((torch.zeros(99), means))\n",
    "        plt.plot(means.numpy())\n",
    "    \n",
    "losses = []\n",
    "losses_avg = []\n",
    "def plot_losses():\n",
    "    plt.title('Losses')\n",
    "    plt.xlabel('steps')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.plot(losses)\n",
    "    # Take 100 episode averages and plot them too\n",
    "    avg_length = 100\n",
    "    while len(losses) > len(losses_avg) + avg_length:\n",
    "        if len(losses_avg) == 0:\n",
    "            losses_avg.append(sum(losses[:avg_length])/avg_length)\n",
    "        else:\n",
    "            i_rem = len(losses_avg) - 1\n",
    "            i_add = i_rem + avg_length\n",
    "            l_curr = losses_avg[-1]\n",
    "            l_curr += (losses[i_add] - losses[i_rem])/avg_length\n",
    "            losses_avg.append(l_curr)\n",
    "    if len(losses_avg) > 0:\n",
    "        plt.plot(losses_avg)\n",
    "    \n",
    "    \n",
    "def plot_stuff():\n",
    "    if is_ipython:\n",
    "        display.clear_output(wait=True)\n",
    "    plt.figure(figsize=(16,4))\n",
    "    plt.subplot(121)\n",
    "    plot_durations()\n",
    "    plt.subplot(122)\n",
    "    plot_losses()\n",
    "    plt.pause(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_model():\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    transitions = memory.sample(BATCH_SIZE)\n",
    "    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for\n",
    "    # detailed explanation). This converts batch-array of Transitions\n",
    "    # to Transition of batch-arrays.\n",
    "    batch = Transition(*zip(*transitions))\n",
    "\n",
    "    # Compute a mask of non-final states and concatenate the batch elements\n",
    "    # (a final state would've been the one after which simulation ended)\n",
    "    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,\n",
    "                                          batch.next_state)), device=device, dtype=torch.uint8)\n",
    "    non_final_next_states = torch.stack([s for s in batch.next_state\n",
    "                                                if s is not None])\n",
    "    state_batch = torch.stack(batch.state)\n",
    "    action_batch = torch.cat(batch.action)\n",
    "    reward_batch = torch.cat(batch.reward)\n",
    "\n",
    "    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the\n",
    "    # columns of actions taken. These are the actions which would've been taken\n",
    "    # for each batch state according to policy_net\n",
    "    state_action_values = policy_net(state_batch).gather(1, action_batch)\n",
    "\n",
    "    # Compute V(s_{t+1}) for all next states.\n",
    "    # Expected values of actions for non_final_next_states are computed based\n",
    "    # on the \"older\" target_net; selecting their best reward with max(1)[0].\n",
    "    # This is merged based on the mask, such that we'll have either the expected\n",
    "    # state value or 0 in case the state was final.\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()\n",
    "    # Compute the expected Q values\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n",
    "\n",
    "    # Compute Huber loss\n",
    "    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n",
    "    losses.append(loss.cpu().detach().numpy())\n",
    "\n",
    "    # Optimize the model\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in policy_net.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAAEWCAYAAABIRevRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeZikZXnv8e9de++z9cAwDA6boAhuiGvUgCZoFFyjcYkal2jC0YScJOQcjYmJiVmOmBizIGiMxh0jiBgjKioKyCA7iAzDwPSsPT29d9f6PuePd6m3uqu7q7uql6n5fa5rrqnlraqnR5yeX9/3cz/mnENERERERESkXSRWewEiIiIiIiIiraSgKyIiIiIiIm1FQVdERERERETaioKuiIiIiIiItBUFXREREREREWkrCroiIiIiIiLSVhR0RY5iZpY0swkzO6mV14qIiIiIHM0UdEVWUBA0w1+emU3H7r9xse/nnKs457qdc4+18loRERGZzcx2m9mLVnsdIrKw1GovQORY4pzrDm+b2W7gHc65G+a63sxSzrnySqxNRERERKRdqKIrsoaY2V+a2ZfM7AtmNg68ycyebWa3mNmIme03s380s3RwfcrMnJltD+5/Lnj+W2Y2bmY3m9nJi702eP4lZvYLMxs1s4+b2Y/N7K0r+yciIiKy9pnZO81sp5kdMbNrzeyE4HEzs8vN7FDw/fRuM3tS8NxLzez+4HvwXjP737H3e5mZ3Rl87/+JmZ0Te+6Pg+vHzexBM7tg5b9ikbVPQVdk7Xkl8HmgD/gSUAbeB2wCngtcCPz2PK9/A/ABYAPwGPAXi73WzDYDXwb+MPjcR4DzlvoFiYiItCszOx/4a+DXgS3Ao8AXg6d/BXg+8HhgHfA6YCh47irgt51zPcCTgO8F7/c04FP43+s3Av8GXGtmWTM7A7gEeEbwul8Fdi/zlyhyVFLQFVl7bnLOfcM55znnpp1ztznnbnXOlZ1zu4ArgBfM8/qvOud2OOdKwH8CT1nCtS8D7nTOXRM8dzlwuPkvTUREpO28EfiUc+5nzrkC8CfAs4MOqhLQA5wJmHPuAefc/uB1JeCJZtbrnBt2zv0sePydwL8F3/srzrnPAAXgWUAFyAavSzvndjvnHl6pL1TkaKKgK7L27InfMbMzzeybZnbAzMaAD+FXWedyIHZ7Cuie68J5rj0hvg7nnAMGGli7iIjIseYE/CouAM65Cfyq7Vbn3PeAfwI+ARw0syvMrDe49NXAS4FHzewHZvbs4PHHAX8QtC2PmNkIsA04wTm3E/g94M+AQ2b2xbBNWkRqKeiKrD1uxv1/A+4FTnPO9QJ/Ctgyr2E/cGJ4x8wM2LrMnykiInI02ocfTgEwsy78luO9AM65f3TOPR04C7+F+Q+Dx29zzl0MbAa+jr9lCPwfNH/YObcu9qvTOfeF4HWfd849L/hMB/zNSnyRIkcbBV2Rta8HGAUmzewJzL8/t1WuA55mZi83sxT+HuH+FfhcERGRtS5tZrnwF35AfZuZPcXMssBfAbc653ab2TPM7JnBEMlJIA9UzCxjZm80s75gi9AYflsywCeBdwevMzPrMrNfM7MeMzvDzM4PPicPTMdeJyIxCroia98fAG8BxvGru19a7g90zh3EH5jxUfz2q1OBO/D3CGFmLwxaqQjuf8DMvhG7/z9m9kfLvU4REZFVcD1+wAx//RL+YMer8TuiTgVeH1zbix9ch/Hbm4eAvw+eezOwO9iW9G7gTQDOuR34+3T/KXjdTuCtwWuywEfw52YcwK8G/59l+SpFjnLmb70TEZmbmSXxW7Ne45z70WqvR0RERERkPqroikhdZnahmfUF7VEfwD/m6KervCwRERERkQUp6IrIXJ4H7MJvj7oQeEVwbIKIiIiIyJqm1mURERERERFpK6roioiIiIiISFtJrfYCmrFp0ya3ffv21V6GiIi0idtvv/2wc05HaTVB35tFRKSVlvq9+agOutu3b2fHjh2rvQwREWkTZvboaq/haKfvzSIi0kpL/d6s1mURERERERFpKwq6IiIiIiIi0lYUdEVERERERKStKOiKiIiIiIhIW1HQFRERERERkbaioCsiInKMMrMLzexBM9tpZpfVef6tZjZoZncGv96xGusUERFZrKP6eCERERFZGjNLAp8AXgwMALeZ2bXOuftnXPol59wlK75AERGRJqiiKyLShu7aM8I9A6OrvQxZ284DdjrndjnnisAXgYtXeU0A7DkyxUf/50EeG5pa7aWIiMhRSkFXRKQN/dX1D/A3//3z1V6GrG1bgT2x+wPBYzO92szuNrOvmtm2em9kZu8ysx1mtmNwcLDphe0dmeYfv7eTgREFXRERWRoFXRGRNlSseORLldVehqxtVucxN+P+N4DtzrlzgBuAz9R7I+fcFc65c51z5/b397d4mSIiIounoCsi0oY8z1HyZmYWkRoDQLxCeyKwL36Bc27IOVcI7n4SePoKrU1ERKQpCroiIm2o4hzlirfay5C17TbgdDM72cwywOuBa+MXmNmW2N2LgAdWcH2z68siIiIN0tRlEZE2VPH8qq7IXJxzZTO7BPg2kAQ+5Zy7z8w+BOxwzl0LvNfMLgLKwBHgrSuxtno91SIiIouhoCsi0oY8z1FSRVcW4Jy7Hrh+xmN/Grv9J8CfrPS6REREmqXWZRGRNlRxjpLXfNDdsfsIV/5oVwtWJLJ46kkQEZGlUtAVEWlDnucoV5qPCf91x17+4bsPtWBFIo0zU/OyiIg0R0FXRKQNVZyj1IKgWyx7aoEWERGRo46CrohIG6q0aI9uqeK1JDCLiIiIrCQFXRGRNuS3LjcfdIsVj4rnqGiCs6wCp//sRERkiRR0RUTakD+MqhWty/57qH1ZVpK26IqISLMUdEVE2lDFo2UV3fjvIiIiIkcDBV0RkTbkOYfnaLrluFiuBL8r6IqIiMjRQ0FXRKQNea41LcfhICq1LstqcDpJV0RElkhBV0SkDYWV3HLTFV0/4JbKChyycrRFV0REmqWgKyLShrww6DZZiQ2DbrFSaXpNIiIiIitFQVdEpA1VgtblZodIhS3LRVV0ZRXoeCEREVkqBV0RkTbkBfm2XGkuKRTC1mXt0ZUVpOOFRESkWQq6IiJtKKzoNht0w4qwgq6IiIgcTRR0RUTaUDiMquS1qnVZQVdERESOHgq6IiJtxotNWm62ElsdRqWgKytPW3RFRGSpFHRFRNpMJTbBp9nW5VLUuqzIIStJm3RFRKQ5CroiIm2m0qKKrue5KOCqdVlERESOJgq6IiJtxotXdL2lV2Lj7coaRiWrwel8IRERWaJlD7pmljSzO8zsuuD+yWZ2q5k9ZGZfMrNM8Hg2uL8zeH77cq9NRKQd1VR0m6jExsOt9ujKStLxQiIi0qyVqOi+D3ggdv9vgMudc6cDw8Dbg8ffDgw7504DLg+uExGRRYoPWi41U9GNhWS1LouIiMjRZFmDrpmdCPwacGVw34Dzga8Gl3wGeEVw++LgPsHzFwTXi4jIItQOo1p6QFXrsoiIiBytlrui+zHgj4DwX0gbgRHnXDm4PwBsDW5vBfYABM+PBtfXMLN3mdkOM9sxODi4nGsXETkqtWoYVancumOKRJZCO3RFRGSpli3omtnLgEPOudvjD9e51DXwXPUB565wzp3rnDu3v7+/BSsVEWkv8WFUzRwLVKxUqrfVuiwrSO1cIiLSrNQyvvdzgYvM7KVADujFr/CuM7NUULU9EdgXXD8AbAMGzCwF9AFHlnF9IiJtKV7RLXtNtC7HKrpFnaMrIiIiR5Flq+g65/7EOXeic2478Hrge865NwLfB14TXPYW4Jrg9rXBfYLnv+d0roCIyKLVti439tfo8GSx5nWgPboiIiJy9FqNc3T/GLjUzHbi78G9Knj8KmBj8PilwGWrsDYRkaNebevywgF1uljheX/zPa67e1/N45q6LKtOP+4WEZElWs7W5Yhz7kbgxuD2LuC8OtfkgdeuxHpERNpZTetyAxXdiUKZyWKFwfFCzeMlVXRllejQBRERadZqVHRFRGQZLbaiG15Tntm6XFbQFRERkaOTgq6ISJuJZ9KZ4bWesOo7c49uIRZ0C2pdllXg1LssIiJLpKArItJmaoZRNRBQw6FTM9uca1uXFTjakZldaGYPmtlOM5tzNoaZvcbMnJmduyLrWokPERGRtqagKyLSZmpalxuo6IaBtjLjKKKwdTmbSjQUmOXoYmZJ4BPAS4AnAr9hZk+sc10P8F7g1pVa29Ckv1/887fuWamPFBGRNqOgKyKyxu0+PMnuw5MNX187jGrhgBpWcmft0Q1e25NL1Rw1JG3jPGCnc26Xc64IfBG4uM51fwH8LZBfqYU9OjQFwA0PHFypjxQRkTajoCsissZ94Jp7+cA19zZ8fSVW0W1kj24xqujWb13uyqY0jKo9bQXiJdOB4LGImT0V2Oacu24lF6bWZRERadaKHC8kIiJLN1koL+p6LxZYGzn/dqGpy52ZlM7RbU/18mT0H4GZJYDLgbcu+EZm7wLeBXDSSSc1vzAdLyQiIk1SRVdEZI0re66hymyopnXZa7x1eWZFN6z0dmeTal1uTwPAttj9E4F9sfs9wJOAG81sN/As4Np6A6mcc1c45851zp3b39+/jEsWERFpjIKuiMgaV6q4WROR51PTutzA66oV3frDqNS63LZuA043s5PNLAO8Hrg2fNI5N+qc2+Sc2+6c2w7cAlzknNuxOssVERFpnIKuiMgaV/G8hiqzofiljRwLNNce3WLZI5kwcqkkpbKOF2o3zrkycAnwbeAB4MvOufvM7ENmdtFqrk2dyyIi0izt0RURWeMWU82F2opuI5XYaOpynXN0M8kE6VRCFd025Zy7Hrh+xmN/Ose1L1yJNYGGUYmISPMUdEVE1rjSIqq5UDuMqpFKcGmeim46aaSTRkHDqGQlqaQrIiJNUtAVEVnjKhXHYmq68cC6mNblUp1hVJlUkqwqurLCFHNFRKRZCroiImtcyXO4RSTdsHU5lTDKi2hdrswaRuXIphKkkwq6srJU0BURkWZpGJWIyBpXrix2GJUfXP1K7CKmLldmV3T91uVEy8/RvebOvVzw/26sabNeyCOHJ/ncLY+2dB0iIiLSnhR0RUTWuLK3tOOFsulkQ5XYuffoVsikEmQaDMyLcf++MR4enFzU+bz/9bMB3v/1exuqUouIiMixTUFXRGSNK1fcolqHK7GKbrmBimkYYmdeW6o4MkHrcrHi4RbTP72AsXwZYFFBtxBcm9dgrLZn2qUrIiJNUtAVEVnjyp7XUGANeUEgzaWTDVU/55+6nCCTtOC61gXd8XzJf89FhNawfXq6WGnZOmRt0h5dERFploKuiMgaV/YcFc81XFENs202laDYQDgNw/DMfcDF8BzdpP+topUDqSYKi6/ohp+fLynotjvlXBERaZaCrojIGlaJTVxutKIaH0bVSEW3GE1dnl3RDffo+p/fuqA7HrQul8qNV4nDiq6CroiIiCxEQVdEZA2Lh8uZQXQu0TCqVLLBPbphRbdO0I1VdFs5eTlsXS5WGg+tYdCfVtBte2pdFhGRZinoioisYfFwW2rwiKFoGFW6sfNvy3Ps0S1VgopuGHSXoaJbXExFN2pd1jCqdqdhVCIi0iwFXRGRNSx+rFCjRwx5rtq63EjQDVuX65+jG29dbt0wqoklTF2OhlGpoisiIiILUNAVEVnD4lXcRs+PrR4vlGwoHNebujxdrHBgNM+m7mzLW5c9zzFRLNd8diPCazV1+Riggq6IiDRJQVdEZA2rbV1udOry4iq6YYCOh+qbdh6mUPY4/8zNpKPjhVoTdCeK5WjA1mLCc/j5hbKCbrtTzhURkWYp6IqIrGHxcNloRTdqXU4nGmo3LtWZunzD/QfpyaV45ikbotblQosquuH+XFhi67Iqum3PNI1KRESapKArIrKGxVuPGz5eKLjMn7rcyB5dr+azPM/x3Z8f5IVnbCadTNCdTQHVs2+bNREPuosIz0VNXRYREZEGKeiKiKxh8aDaSGiFmVOXHc7NH5BnTl3edXiSwxNFnn/6JgD6OtIAjE6XFrf4OYRHC8Ei9+iWNXX5WKF6roiINEtBV0RkDYufbdvw1OXYMCpY+PzdsFIcflbYGry+MwMsR9BdakVXU5ePFepcFhGRZinoioisYbWtyw1WdGPHC/mvq77H8GSR//2Vu2qqqsWoouvV3E8Hr+8Ngu5Yi4Lu2FIrutE5ugq6IiIiMj8FXRGRNaxmGFWDU5fDim5H2q/oxqcU/+AXg3z19gHu3TtWfd9wj27wuvAzw2nLuXSSbCrRsopufK/vfBXdewZGeeOVt0TrL5YVdEVERKQxCroiImtYzfFCi6joJhNGLgi68T2tOw9NALXhd+bU5TBQhhVh8NuXR6eWoXV5nnbsn+4+wo93DjE4XgjWqanLxwq1LouISLMUdEVE1rB42/FCe22r10HSLAqq8VAbBt14+C3NWdGdEXSXYRjVfBXd8LrwWKPoeCFVdNueaRyViIg0admCrpnlzOynZnaXmd1nZn8ePH6ymd1qZg+Z2ZfMLBM8ng3u7wye375caxMROVrUTF1u+HghRyJB3YruQ4fGgZkV3dqpy8sfdMv0BEcWzVelDiu/hVLt3mFNXRYREZGFLGdFtwCc75x7MvAU4EIzexbwN8DlzrnTgWHg7cH1bweGnXOnAZcH14mIrDnliscv/e33uO7ufcv/WUtpXfYcSTNyaf+v+HBPa6ni8ejQFFANj/7j1dZl51zUTrxcQXciX6avM03C5q/ohufthqE8XKf26LY/tS6LiEizli3oOt9EcDcd/HLA+cBXg8c/A7wiuH1xcJ/g+QvM9K1ORNaefNljz5FpHhmcXPbPildxGx1GVfEciYRFxwuFwfDRocnoPeIV3XIsQFc8F51Xm1mmoDuWL9OTS5NOJuav6Bb8zyuWPSqeiyrOal0WERGRhSzrHl0zS5rZncAh4DvAw8CIcy6cRDIAbA1ubwX2AATPjwIb67znu8xsh5ntGBwcXM7li4jUFQbBxRyNs1TxENro53nRMKpwj67/unB/bvwxqB0IVfZctXU5Vf1ZY29HumXHC43nS/TkUmRSiZp1zL4urOh6NV+7KrrtTz/nFhGRZi1r0HXOVZxzTwFOBM4DnlDvsuD3et/VZpUvnHNXOOfOdc6d29/f37rFiog0qBTsmy01WGFtRryK2+ge3Wrrcm1F96GD1aAbD4szjzCaa4/ueKHc8ECs+Yzly/TmUmQWqOiOxYJuMXadKroiIiKykBWZuuycGwFuBJ4FrDOzVPDUiUC4yW0A2AYQPN8HHFmJ9YmILEa4V7Q0TzWyVWqGUXmNV3QTsYpuPljnwPA0/T1ZElZb0S1XPBLBjxorFRc9l5lxvBDQkqru6FSRdZ0ZMqnEAnt0w6nLlZo/67yOF2p7queKiEizlnPqcr+ZrQtudwAvAh4Avg+8JrjsLcA1we1rg/sEz3/PObf85RIRkUUqzziOZznFjxcqLbKiG+7RLQQV0MMTBTb3ZMmlkzMquo6OoPpb9rzoc2bu0QVq9ul++bY9DE0UFv01DU+VWNfRwB7d2NTlYlRltii4S/tS57KIiDRrOSu6W4Dvm9ndwG3Ad5xz1wF/DFxqZjvx9+BeFVx/FbAxePxS4LJlXJuIyJKF4azYoj26dw+M8Mkf7qr7XKWmdbnRqcuQTBjZGRXdwYkCm7qzZIO9sWP5EocnChQrHh2ZZPR5c7UuQzXoHhzL80dX380Xb9uzmC+VfKnCdKnC+q6gottI0C17lMr+n0NvLs20KroiIiKygNTClyyNc+5u4Kl1Ht+Fv1935uN54LXLtR4RkVZpdevyNXfu47M3P8o7n3/KrOfKM/bPNmLmObphRXdwvMDjj+shm0pSKHn85XX3c//+MYCo+hvu0U2YH5ZDfZ21QffQmF/Jfejg+KK+1vD16zr9im6xXP9rKlW8aC9uoVyJAnFvR5qhySLOOQ0samOm5mUREWnSiuzRFRFpJ6UWty6XKn5rbr3dGs21LlenLjvnODxRoL8nSy6dIF+ucHCswK7giKR4RbdY8WqquTC7ons4aFneOTjBYgxPFQFY1zF/RXeyUI5uF8petJe3N5eKHpPmmdmFZvagme00s1mdVGb2bjO7x8zuNLObzOyJK7OulfgUERFpZwq6IiKLFAbOVrUuz9cKvaTW5WAYVSaZwMxvFx6dLlGqOPq7s1FFd6pYZipoA67u0XWUyq5mfy7MDrqDQdB9+NAk3iIC//Ck//r1nWkySZuzKh62LYO/R7cUq+iCjhhqBTNLAp8AXgI8EfiNOkH28865s4MTFP4W+OiKrG0lPkRERNqagq6IyCJFFd0WBd2wfbfeBOKSt4TW5aCia2bkUv7gqcFxP5huilV0JwvVsBhOaK54fqhMpxqr6E6XKuwbnW5oXf7rg4pu5/wV3bF8dehVTetyLgy6qui2wHnATufcLudcEfgicHH8AufcWOxuF3WO/RMREVmLFHRFRBYpPM+20VbiBd8vCLP12nHDzzJr/Hihiuei/bXZdIJ8yYsqsDMruqFcrKJbLHuzKrq5dJJsKsFI0Hp8eLwYPbfzUOPty8NTM/foNlDRLXtR5be3w29d1lm6LbEViE8TGwgeq2Fmv2tmD+NXdN9b743M7F1mtsPMdgwODja9MLUui4hIsxR0RUQWKayyznc0zqLeL2xdrhd0gypuNpWIQu9CPOdIBEkhl0pSKFcruv09WbLpBIVyhcnY9OKodbnigoru7KSxqTvL0GQQdCcKUZV3cUHXf/36zgyZeY4XigfdYtmbVdHV5OWWqBcnZ/1H5pz7hHPuVPxTE95f742cc1c45851zp3b39+/TEsTERFpXENTl82sH3gnsD3+Gufcby3PskRE1q6wutiqoDtf63K54pFKGOlEYnHDqIKKbi6s6I7XVnSHJopMxQY+LTSMCmBjd4YjsaB72uZuHjk8ycOLGEg1OlUik0qQSydIp+au6E4UZrQul2fs0S0r6LbAALAtdv9EYN88138R+JdlXZGIiEiLNHq80DXAj4AbAP3rQkSOaWGVtdEK60LCwFy3ddlzpJJGKmmNty47SERBN9ijO1Egk0zQ25EK2pkrTJXqVHSDPbozW5cBNnRlGJrwg+7geIFT+ruoeI6B4cb36A5PFVnfmcbMyCbn3qMbVnR7sim/dbkSnqPrf9vKq6LbCrcBp5vZycBe4PXAG+IXmNnpzrmHgru/BjzEClDrsoiINKvRoNvpnPvjZV2JiMhRIgymrarohgG2fkXXkU4kSCUbr+j6w6j829lUgkLZ4/B4kf6ebDSgamS6RPw0o1xN67KrX9HtyvKLA/65uYcnCpx38gYmCuVFtREPT5VY35kBaGiP7qaebN2py9qj2zznXNnMLgG+DSSBTznn7jOzDwE7nHPXApeY2YuAEjAMvGUl1qacKyIizWo06F5nZi91zl2/rKsRETkKlFo8jKoUtC4X6rTjlj2PZNJIJaxmyvN1d+/jv+89wD+94WmzXlM7jKpa0d3Ukw0eqw6VCoVBt+IFe3ST9fboZjg8WaRU8RieKrGpO8uB0TwjU6VZ185ldKoU7e3NpObeozuWL5FJJujJpWpblzV1uaWC7+vXz3jsT2O337fiiwJMJV0REWlSo8Oo3ocfdvNmNh78GlvwVSIibajVFd3iPMOoShVHKpEIWperwfrWXUf473sP4NzssF2JD6NKJ8mX/T26/d1+JTWbSjDzpKL4ObqF8tx7dItlj8eOTAHBUUWZ5KLOtPVblxeu6E7ky/TkUlFFOhpGpanLIiIi0oCGgq5zrsc5l3DO5YLbPc653uVenIjIWhRWVhs913Yh0R7dOsG54vnVVX8YVfX56VIlCqUzefGKbipBoVRhaKLApm6/ohtWb+Oq5+j6Fd1Mqt4eXf/1Dwbty/3dGTrSyUVVV4enSqzvild06/8ZjgdBNxME3dLMqcsKum1N9VwREWlWo63LmNlFwPODuzc6565bniWJiKxtxcrcU5KXIhxqVagTGMuV2DCqWCgMq6jj+fKs4Fpx8anLfsX1yGSRDV3Vim4onTRKFRdNXS6HQXeOii7Az4Ogu6k7S0c62XDodM4xOl2kr8N/n0zSKFY8nHOzWlUHxwus68yQTSUZnS7NmrpcUNBta+pcFhGRZjVU0TWzj+C3L98f/Hpf8JiIyDGnWtFt8Tm6dSq6JS9oXU4kaj4vDLqTsSOCQp4XP0c3weB4gbLnoqAbD8Zb13XUPFbxPErl+sOoNgUV3Qf2+ztXNnVnyaUTDQ+jmixWKFUc6zurFV3/66+t6nqe4959o5x1Qm9Qka5WdHuCqcs6R1dERETm0+ge3ZcCL3bOfco59yngwuAxEZFjTnWP7tJal+8ZGOXSL91JJWh9nm+PbsULztENKq+hsF14ok7QnVnRnQxC4cbu2RXdbRs6o+ugWtFN12ldDl9/88NDdGWSbNvQ6bculyvRXuG///aDfOue/XW/7vAs341BC3UYpmcG/EePTDGeL3POiX3VPbrBn01HOkkqYWpdFhERkXk1GnQB1sVu97V6ISIiR4to6vISW5dv2nmYr92xl6HJQvB+4Tm6s8NbqeJIJf3jhSqxPcHTsdblmSoeUUU3HmrDPbbZVLWie+J6v6LbEZu6XJxj6nJYEZ4olHnytnUkE0Yuk8S56hnA/3Hzbv77vgN1v+79I/55uyesywGxiu6MP8e7B0YAOHvrOrKppD91ueLIJBOY2aL3BcvRR63LIiLSrEaD7l8Dd5jZv5vZZ4Dbgb9avmWJiKxdUUV3ia3LYdvx2LQfUsvz7PktV/yKbiphNcOoFmpdDjuP423KG6PW5epf/af2dwPQG7QE++fo1t+jm0sn6c761z31JP9nn2FAzpcqlCoeY/kyk4X61da9YdDt88N1GHRnVnTvGRglm0pw+nHdZNPVYVRh+M5l/H3BP33kCJ+9eXfdzzrWmNmpZpYNbr/QzN5rZusWet1aZRpHJSIiTWp06vIXgGcBXwt+Pds598XlXJiIyFoVTlteautyvhxWY/3zZ4tRRbdO0ItOMQUAACAASURBVPX8YVTpZKJmynNY0V24dTle0Q1bl6vh97VP38YX3vksTgxamCueozjH8UJQbV9+yrb1wfsno/UMB2fzThVnrwlg/2gegOP7/Ipu1Lo8s6K7d5QnntBLOpkgm/KPICqWq5Ogc+kE+VKFL972GJff8FDdzzoGXQ1UzOw04CrgZODzq7skERGR1TNv0DWzM4PfnwZsAQaAPcAJwWMiIsecMJhVPIc3zxFD37pnPw8PTsx6PB/smR0L2o5L8+zRLVcc6fAc3VjlM5zQPL7AMKp4qN1Qp6Lbk0vx7FM3kgqCsb9H19U9XgiqVeGnbKut6E4XKwxP+sF9co5BUftGptnUnYnCcTZW0f3FwfHoXOD79o5y9ta+aP3Vim4i+sx8qcJkoTxnqD4Gec65MvBK4GPOud/H/74tIiJyTFroeKFLgXcB/6/Ocw44v+UrEhFZ4+LTjx86NMHX7hjgsgvPnHVEzh985S5e94xtfPDlZ9U8Hu4vDSu6pflalz0vmrocryBHFd16e3TrVHS7s6lYwPR/78wkSQTXJaOg6wV7dOsH3S19HWzfWKS/p/ZM3nzJY2Tar+jWa6cG2Dea54RgyjNUK7qliscnf/gI37n/AM8+9XwmixVOCirM2ZS/N3m6VKkJutOlCuWKI1/yqMTODT6GlczsN4C3AC8PHkuv4nqac8z/zykiIs2aN+g6594V3HyJcy4ff87Mcsu2KhGRNSx+nu237t3Pv/1gF29/3sls7qn+tVgse0wVK3WHJlVbl8t4nouGTM01jCqX9qcuxwN2eLxOvVBZ8RzJsKIbBNGwmus/5gfGzkz1W0BY0S1Fe3TrJ433v+wJNUf7hOfvTpeqFd2puYLuyDSnBXuCgWgfcLHscXAsz1i+zP5Rfx9vNCE6WOt4vhxVgLPpJNPFStTqPV2qRHuHj2FvA94NfNg594iZnQx8bpXXtGTKuSIi0qxGh1H9pMHHRETaXnx4UjhQauYAprBaWy+8VodRlWoGWtU/Xsg/0zaVTEQB2zkXheV6e3Q9z0WV2jAcxoNuLqjodmWrbc2pWOh0jnkruqfEwmp8GNWRYIp0vdZl5xz7R6bZsq76w4B0qlrRPTTu/yz1oYMTwXprJ0RP5Mu1rctlL2pbVvsyOOfud8691zn3BTNbD/Q4547a8+5ndkeIiIgs1rw/Ajez44GtQIeZPZXqD1l7gc5lXpuIyJoUr+iGgXZmZTU89qdeeK22Lpdr2pFnTh8GPwRGU5e96tCq4Nja+scLuWpFN2wt3linotsRm8gcVnTDluh65+jWE7ZGTxcrDE3OPYxqbLrMZLHC1ljrcljRLZQ9DgVn7D54YLxmvWFQHy+USaeq7dj50Ur0w4XpOfYEH0vM7EbgIvzv63cCg2b2A+fcpau6sCVSzBURkWYt1Ov1q8BbgROBj8YeHwf+zzKtSURkTYsf8xMGzYlCmbv2jNDbkebkTV2MhROV6wbdcBhVqeYM2UKdNudw6nIqYVHAjl9Xv3WZqKKbq9O6XK3oVr8FhHtcw7XNVdGdqSM+dTkIuqWKo1Cu1AzC2he0JG/piwXdILhOFiqMTPl/Xg8e9IPupu5scE3Yulxic7AvONyjG1azpxR0Afqcc2Nm9g7g0865D5rZ3au9qKVSQVdERJq10B7dzwCfMbNXO+euXqE1iYisafEq7HjBD2gT+TIf+Pq9nHF8D//0hqdFLc31qrT5cryiGwu6da6teM4fRpWsDqMKq64wR+uyi52jG7Yud9fbo1sNomEFOKzGzrVHd6ZcrHU5rOgCTBVmBN3wDN1Y63L4/GNHpqLHfhEE3ZlHIU0UylE1uCM4R3dSQTcuZWZbgF8H/u9qL6ZZOkdXRESa1dD0Dufc1Wb2a8BZQC72+IeWa2EiImtVqd4e3WKZI5PFKHiOz1PRLdTs0XWxx+doXU7WDqPKx4JuveOF6g2j2livohsbRpVIGAmD6aL/GXMdLzRTOIwqHztHF/w/j/Wxz9wXnKEbn7ocBtcdu49Ejz12ZIreXCr6/LB1eSJfjj2WZHS6FJ0rrD26AHwI+DbwY+fcbWZ2CnDUHjKsiq6IiDSroX/JmNm/Aq8D/hf+1pnXAo9bxnWJiKya6+7eN+cROVB7vFAYaMfzZcbz5ai6OBYNo5q7dXk8X65pXa5X/S1XXLBHN0FlRkU3mbC664wPo+qIgm42ej6q6MaGUQGkEolFty7nYq3LQxOxiu6MKuu+kWnSSaO/u7qO9V0Z+nuy/OThoegx52Bj9+y1lj0XBfOOTLLmBwiq6IJz7ivOuXOcc+8J7u9yzr16tdclIiKyWhqduvwc59xvAsPOuT8Hng1sW75liYisjkeHJrnk83fw3/cemPOaUrlahR0L9ugOTRQpVrzYROWFh1GN5Us11eFinQnN/h7dBOlkdRhV+BkbuzJM5Mv8+48f4Z6B0eg18WFUp2/u5rKXnMmLzzouej4cAhWv6IIfnKcXG3RT4TAqj+GpIsf1+iF1ZgDfPzLNcb25KICHHn9cN6PT/g8FOuoNzoq1P5+zra/mupCGUYGZnWhm/2Vmh8zsoJldbWYnrva6lkoFXRERaVajQTc8Q3fKzE4ASsDJy7MkEZGlu2/faN19q406OOZP/50qzR2eSnUquuH5r2Homq91eTpW0Y1XcetVf8ueRzphpJLVYVTh6/t7sgxNFvizb9zPX13/QPSaiuei4VKJhPHuF5xKby4dPZ9IGBuCampcKmHR+hsNuqlkgkwywXSpwpHJItvW+wP5Zx63tG8kX9O2HHr8cT2AH7JPP84/tmhjfD9xrIX6Gds3AP7UZcPjbNvF76e+yi99/zUwcaih9baxTwPXAifgn5bwjeAxERGRY1KjQfcbZrYO+DvgZ8Bu4AvLtSgRkaUolj1e+c8/4T9veXTJ7zE04QfdQqnCvpFpvvvAwVnXlCpeFMDCAVHhHtRq6/I8w6hie3TD8Jqw+qG4XHEkEwlSiQRlz+Gci/byburORp9/864hHhvyhzp5zs2qnM50ze8+l3f8Uu3PK5PJakU3nIjciFw6weB4gVLFceJ6P8xOztg3u290mhP6crNeGwbdTd2ZaKpyvdZlgHNO7IO9t/PLv/gwt2Yv4RvZ93NJ8r8oWQYmBxteb5vqd8592jlXDn79O9C/2osSERFZLQsGXTNLAN91zo0Ek5cfB5zpnPvTZV+diMgijE6XKJY9RoJW2KU4HEwOLpQ9PnfLo/z2Z2/Hiw2MAj98xicWg9+aC7UhFmaHV+cchbKHGUwUy1EVtyuTqh90PY90MIzKv++iMBoewWPm//rqzwYAv6K7QM5l24ZOOme0Li+logv+ntlwqvK2DX5FNz4gquI5DozOX9Hd3JOL9hHXa11OUSb7zffCJ8/n5P3X81PvDH6/+B7OLfwLXznnSjjurIbX26YOm9mbzCwZ/HoTMLTgq0RERNrUglOXnXOemf0//H25OOcKQGG5FyYisljhXs9m9mweHg8qumWPqWKFsucYz5fp60xzaCzPyHSJYsWjM5NieKoaqMOgN12qregWZuy7DYPths4MQ5NFjgTBuiubqt+6XAnO0U2GFeTqPuBNPX4gfPzmHrasy/Hpmx7h/DM347nqcUGLkUwYUyV/3YsJurl0kt1DkwA8bmMXUNu6fHiiQNlzbKkTdMN25c092ahlOR50M4Vh3pT8Dm/K/ADu2AXP/T2+s/6NXPLV6kDhSe3RBfgt4J+AywEH/AR426quqBnapCsiIk1q9F8y/2NmrzbTwH8RWbvCoDszXC7G0GQYdCvR+4TH5nz0O7/gtz97O+WKi47VCYVha7pUwTk359TlfGx/LRAF3e7c7Iquc84fRpVIsL7T32N7eLxY3aMbVHSfetI6PvzKs1nfleGNn7wFYMHW5XpSicSijxcCfzjU/qB1+8zj/QptvKK7N/ghwNZ1s1uXe3NpnrCllyds6Y0q1Bu7s1Apw/c+zAlXPYW/TH+aU9an4VVXwov/nHRnX817TOt4IZxzjznnLnLO9TvnNjvnXgG8arXX1Qqfa2IrgoiIHLsa/ZfMpcBXgIKZjZnZuJmNLeO6REQWLWwXztc5j7ZR4RE5hZIX7YUNg+7wVJHD4wVKFW9W63LIOT/cztW6HK5tc68f+o4Ewdqv6NYG9ErQMp1KWFQpffTIZPQeYTB82knr2bqug39+49OiwL3Uim4YxDOLrOiC3z592ma/QjsRq+juH5l9hm7ctZc8l0tf/PioontS8SH41K/AD/8WO+uV8O6byPyvW+Gc19Z8nn87oeOF5nbpai+gFd7/9XtXewkiInIUWrB1GcA517PcCxERadZoFHSbaF0OhlEVKx75IHiGe36nihUmimWy6cSsI27iposVxmPDqJxzhA0x+RnV2KGwoptNzgrF5TDoJhNsD4Lu7qGp6D3OO3kD55+5mV8+czMAZ53gV0UPTxSWWNFd/PFCUD3u57ieHLl0ks5MkqlCmbF8iZseOsyde4YB2NJXP+iGn/Xs/jJX9n+Fc66/Bjo3wquvgrNfM/vzMtW19fdk552QfYw7aruw7OhduoiIrBENBV0ze369x51zP5znNduA/wCOBzzgCufcP5jZBuBLwHb86c2/7pwbDtqi/wF4KTAFvNU597PGvxQROda1IujWq+iOBBXdfKmCczAyVZqzogv+0URh67JzfmANh0mF4XlzcN5s+HldmdSsCc3h52dSCTb3ZMmmEjw2NEkuncQMtvTl+NRbnxFdb2ZccOZmvrRjT93BVgtJJiyqIofrbUQumIwcTlzuyqa4e2CUZ/3Vd6Nqa3c2RW+uzrcc5+CRH8JtV7L5wet5kVeBc38LLvgAdKyf4/OS0Rr7OtI6R3dubuFLRERE2lNDQRf4w9jtHHAecDtw/jyvKQN/4Jz7mZn1ALeb2XeAt+JPcf6ImV0GXAb8MfAS4PTg1zOBfwl+FxFpSDSMqomgOzhR3aMbhtLhyWpFF/zgOnNicdxUocxEoUwunSBf8iiUvahqGbYdhxXdsC26O5eiVHF4XvVooPDr6etIk0gYj9vYye6hKR63oZOOdJJ6YxN+OQi69+0bXfTXnopVcRc7dRmqE5e7Mkl+uvsIAH/3mnP4y28+wJa+XO16vQrc/3X48T/A/rugYwM86z3w9LfBxlPn/bww6HZlU3SmU0w2cW7y0c7MxqkfaA2oX0IXERE5BjTauvzy+P2gWvu3C7xmP7A/uD1uZg/gH2J/MfDC4LLPADfiB92Lgf9wzjngFjNbZ2ZbgvcREVnQaJN7dAvlastxoRyr6NYJ0PF9or25VDRlGfyw7Jy/h3ZgeJqJfJlbHh7igidsjqrNMyu63Vn/r+NixSOXSAaf6z+3rsMfRHXShi4eG5riuN5szefHnX/mZl711K287bkn131+PqlYu/NihlGFawkruuEPAbau6+C1527jmSdvjH5oAEBhHL78Fnj4u7DxNHj5P8I5r4P07GFV9YSt0l2ZFB2ZZPTDgmORthaJiIjU1/i/ZGoNAE9q9GIz2w48FbgVOC4Mr8Hvm4PLtgJ7ZnzG1jrv9S4z22FmOwYHB5e0eBFpT822LocTkMEPutEe3SBIxVtk463L4QTl0KExvyocDov61r37ecd/7GDHo8OzJiYPxYZRhZ878+tZF0xc3r6xk0ePTDJVrMy5RziTSvDR1z2Fs0/sq/v8fJKxoLuUPbrb1gcV3ax//5xgDSdt7IzOy+XAvXDli2DXjfBrH4XfvQ2e/paGQ27887qzKbqySQ2jakM640FERJrV6B7dj1NtjUoATwHuavC13cDVwO8558bmOaGo3hOz2rGcc1cAVwCce+652n8kIpHq8UILV3RHp0qM5UtRuy34R/eECqVKbOry7IpuPOhu6s7y8OAk3dkUE4UyB8fy0eMAh4KzeX/w4CBP2toL+K3KuXSierxQFHQrgB9sR6Zqg+7jNnWRL3k8NjRFNr3Un1PO7Qlberhzzwiw+OOFAE7cUN2jC1TDtufBIzfCjk/Bz6+Hrk3w5q/BKS9c0jqrrctJOtIp7dEVERGRWRrdo7sjdrsMfME59+OFXmRmafyQ+5/Oua8FDx8MW5LNbAtwKHh8ANgWe/mJwL4G1yciUt2j20Dw+ctv3s8de0a44dIXRI8dDqqr3dmU37pcrh1GFa8cdswIuuC3I08MljkQBN3+Hv+4nOEgzP7wocHo+J2OdJLeXDoKwV3B+8WHSIUt071B6/LjglD+4IFxTtpYDeit8sGXn8VPHh7i0aEpsktoXY4qupkUGUo83+6CH/wX3PUFOPKwvw/32b8Lz32fH3aXKFxbVzblT3jWOboiIiIyQ6N7dD9jZv3B7Yb6hYMpylcBDzjnPhp76lrgLcBHgt+viT1+iZl9EX8I1aj254rIYkTn6JYXDrq3PzocBeNQuF/2hHU5imUvaoEemSpR8VxNCK3Xunx8b45dg5PsG5kG4LjorFz/fe/ZO8re4LlcOklPLlUNuuEe3XjrchCw+4KgG7b/jhfK8x5vtFS5dJLv/8ELOTJVXFTr8uOP7+HkTV1s6cvByGO8avgq/jx7LZu+Hxy3vu2Z8II/hidevKgW5bkkEkYunaArEwZdVXSXyswuxD/xIAlc6Zz7yIznLwXegf9D7kHgt5xzjy73up560rrl/ggREWlz8wbdIKx+ELgEv7U4YWZl4OPOuQ8t8N7PBd4M3GNmdwaP/R/8gPtlM3s78Bjw2uC56/GPFtqJf7zQ2xb/5YjIsazRPbqj0yV2HZ6MqqihvcPTmPmVyd1Dk9WK7nRx1iTnTDJBKmGUPcembr9yGwbbMMxu6fPvhy3IzsF3HzgI+KEyrNRC/T264TFG2ZS/zuP7cjzrlA3csuvIsrQugx8iwwp1oy46+3gu6rwfvvQGeOjbnO/g3p7nsOmi98G2Z8x5TFAzcukkXVl/GFWh7FHxXM0eY1mYmSWBTwAvxu+qus3MrnXO3R+77A7gXOfclJm9B38Q5euWe23ZVJJ1neno/zsiIiKLtVBF9/fwA+sznHOPAJjZKcC/mNnvO+cun+uFzrmbmPuw+gvqXO+A321o1SIidcSnLjvn6h6/A3DfXv/onZl7eXcOTrB1XQd9nWkKB2MV3cnSrHboVDJBKhkG3WrrMsDAcG1Fd3iqyLpO/7zXuwf8z86lE/Tk/KCbTlp0Fm1xxjCqdbEwDPDap2/jll1HuGdg8ccHtZxzcPu/w48/BsO7oasfnncp9vS3cva6bQu9uinvev4pPOmEPn5+wK8aTxXL0Z+nNOw8YKdzbhdA0FF1MRAFXefc92PX3wK8aaUWpx9biIhIMxYqCfwm8BthyAUIviG+KXhORGRNKFW8mmnE8w2kujsIumXPUa5Ur9t5aILTNneTTVXPv02Y3yo8nq+tLGWSiai9d2N3tXUZ/EpsTzYVDZganirRk0tx8qYuyp4/Qy+XStKb859PJxNkkrPXPTJdoq8zU/O5Lzn7eKBaAV414wfhP18D1/0edG2GV18Fv38/XPABWOaQC/A7LzyN5z++n47gKCMNpFqShk47iHk78K16TyzHiQjzDK8UERFZ0EL/Uko75w7PfNA5NxgMmhIRWRPCau5xvVl2D02RL1XmPGs2Xg0tVjxSyQQVz/Hw4ATPPXUjZc9Fwba/J8vBsQL7R/M175FKWhR0zziuh+edtonnnraJZMKoeI4N3ZlocvHIVJFN3d2c0t/Fzw+Mk0kmSCQsqkCmEkZPEHrj+4ZHp0r0ddT+Nd2ZSfG133kOG7tqA/CKqZTgzv+E734IipPw0r+HZ7xj1c6DCdvPtU93SRo67QDAzN4EnAu8oN7zy3EigmKuiIg0Y6GgW1zicyIiKyoMiJt7c0HQnbuie8/eatDNlzw6MzAwPEWx7HH6cd3sGqzuzz2+r4ODY4VowFQonUyQTvr/FF/fleZz73gm4E9TniiUWd9ZDbplz9GVTXHKJn/icri/tjcIsZlUItrPe2C0+jkj08XoNXFPO6n1e14X5Bzcf40fcI88DCeeBxf/E/SfsfJrielU0G1GQ6cdmNmLgP8LvMA5V1ihteksXRERacpCQffJZjZW53EDmh+dKSLSImHQDduH5xtINTRRiM68LQQTmncemgDgtM3d7B2uhs3je7PcBVFFN6zYppNGKuEH1njlOBcE3Y1dGTKxycWdmSSn9HfVXN8b7dFNsKk7Szpp7ItVjkemStEZuqsqP+a3KN97NWx+Irz+C3DGS9ZEEjl5Uzdvfc726IcGsii3Aaeb2cnAXuD1wBviF5jZU4F/Ay50zh2a/RbLafX/+xIRkaPXvHt0nXNJ51xvnV89zrk18K8vERFfvHUZ5j5iyDnHdKkSHdlTCCq/UdDt7yEbC65b+joAovC7OThKKJ1MkEn5k5fTMwItwIauTM37dGVSnNLvV2fDwVNhu3IqaSQSxnG9OfbHKsej0yX6VjvojuyBT/0q3Pd1OP/98O6b4MyXromQC3DG8T382UVnceL61p8r3O6cc2X8UxW+DTwAfNk5d5+ZfcjMLgou+zugG/iKmd1pZteu1PoW+5/Y7Y8eYftl32T/6PTCF4uISNvTj8BFpC2MRUE3rOjWb10uVjw8B+s60+wdmY5alHcemmBTd5a+zjTZVDW4hi3F4ZFBm3uy7B/N+1OXEzbrPNvw/oaZFd1staLbUaeiC3BCX0dU0c2XKhTKHus6VmkvLkBh3B84NbYP3vRVOPX81VuLLAvn3PX4x/vFH/vT2O0XrfiiAov9UcrnbnkMgJsfHuJVTzux9QsSEZGjyvIcxCgissLG82XAHx4Fc0/hzRf9YBu2BIety/tGp9m2wa/eZuJBd53/2MDwFODvAQZIB5Xc3IyzeHOxim78fbqzKXpzaTZ1Z6PW5bCiGwbiLetyUTUqPD+0r2OVKrqlPHztXXD4F/C6zyrkyopbbEXXP6XQd9eeEYYmVmw7sYiIrEGq6IpIWxgLpyR3z9+6PB3s3Q0rpWHld7JQiYJnvKLb15GmK5Nk34hfaY1al1OJmvNvQ51BiF3flal5n87gGJyzt/aSCP4F39tRW9Hd0tfBgdH9eJ5jZNqf97cqe3Qnh+BLb4THbvanKp/ywpVfgxzzbIl7dM3g4k/8GIBH/vqlOqZIROQYpaArIm1hPF8mnTTWBefOFuYYRhUF3RkV3eliJdrfm03FhkulEmzozrDniF9pDVujw725s1qXg4ruzGFU4TE4H3vdU6PH4nt0AU5Yl6NUcRyeLEQV3XUrWdEdPwC7b4LvfxhG98JrPg1PetXKfb5IjPKpiIg0Q0FXRNrCeL5ETy4dVVjn2qMbtjRHQTe4bqpUjqqu8UpsNp1kQ1c2Crr9sWFUqeTce3TXd2VIJIx00ihVHJ1Z/73jw6Vm7tENB1/tH8lHw7VWZBjV5GG4+h2w60bAQddmeMs34KRnLv9ni8whsYik+6OHBvn6nbNORuLb9x1kYHiKJ29bxzO2b2jl8kREZI1T0BWRtjA2XaYnl4r2v04vVNENWpfDYVRThUo0MTkba0fOpRNs7MpEt7et7yRh/h7ci5+ylWK5NlCHnx++JpNMUKpUoopu3Kw9usHgq/2j01GrdFhBXjaTh+Gzr4DDD8ELL4PTfwWOPxuSGqwvR483X/XT6PaNDw5Gt9/9uduj2z//iwtrjgITEZH2pqArIm1hPF+iN5eO/iE71zm64eNhpTS8P1WMBd1Y63I2lWRDEFo7Mymee9pGfnLZBRzfl+M3zjtp1vuH77E+DLqpBJPFCl3Z2X/ddmVSJAzSUeuyX9HdN5Jn5+AE6zrTUWBeFjtvgK//DkyPwG98Hk5btQG7IrMstXX5mjqVXQAvNqxKRETan4KuiCyLgeEpenLpFZsaPJ73K7odUdBdoHU5PEe37OF5/tm6HUHrcnxacjZVreh2pJOYGcf3zV1lXd+VoTuboidb+15d2dmVpETC6M6mSAUV3fWdaTrSSfYMT7Hz0ASn9XcvzyCd0QH40Udhx1XQ/wR409V+FVdkDWn0P/3RYD/7QpRzRUSOLTpeSESWxZuv+ikfu+EXK/Z5Y/kSPblUtL92ropu2LocVlwL5Ur0WFdU0Y23Llcruh112o9nevvzTubq9zwnCqhh0A33/87U25GOWpfNjLO39vGzx0b8oLu5e8HPWxTn4Ccfh394Mtz+aXjmu+Fd31fIlTUpPnV59+HJOa+77Gt3N/R+5YqSrojIsURBV0SWxf7RaQbHV+4cy/F8md5cmkTCyKQSNccL3XD/QW7bfQSI79GtVnSngipv/dblRKx1eeGg29eR5ozje6L7YYjtmiPoXvaSM3nbc7dH98/dvp57BkY4MllsbdAtjMNX3gL/83444yXwvrvgJX8D6Y7WfYZIC8Urui/8+xv55t37Z11TLHs8eHC8off7w6/e1aqliYjIUUBBV0RarlCukC95UZvwSvBbl/3wmkslyMc++yP//XP+9caHgdge3Y7q1OWpYhkgal3Ozmxd7g6HUS1+kE0YmjvrtC4DvOycEzg3Ng32Gds34AWFp1NbFXQf+SH82wvggW/Ai/8Cfv2zsG72/mKRtWRm5/J9+0ZnXfP493+LXYNzV3vj/uf+gy1YlYiIHC0UdEWk5cKjcaaWKeh+7+cHeed/7IjuVzzHRKEcTTHOpZM1e3THpkuMF/wwG4bvrmyKdNLIlyvROrtmTF1OJYxUMsHGLv9IoUYqujOFrcvddYZR1fO0x62PKlmn9TcZdIcehs+9Bj7zcnCef2TQc9+rA0rlqDBzf7oaj0VEZDEUdEWk5caioFtelvf/8c4hvnP/wag6O5H3P6c3qNJ2ZJI1rcvj+TKTYdANXpNLJ8mmkkFFtxK9DqrtxmFld0NsGNViVffoNvbavo40ZxzXQ0c6ydZ1TbQVD9wOV70YBn4KF3wQfudm2P68pb+fyApr5x/H3LJriO2XfZP7942tIXktNgAAIABJREFU9lJERNqWgq7IUepnjw1T8dZmjWO5K7rh+4/la3+PKrqpZBSCSxWP6VK1ajtdqpBJJUgmjFw6QaFciQJ5ODAqGwTa6Ezc7saHUc2UXWAYVT1vfOZJvO4Z20gklvhP/bH98PnXQrYH3vl9+KVLtRdXjj4z/vNf7qnJh8by/MnX7pl1NvZy+PZ9BwC4edfQsn+WiMixSkFX5Ci058gUr/rnn/DdB9bmnrN40H1saIqXffxHDE20bjDVSHCcyHhQyQ2Dbm/UupxgOmhdDqu9E0FFN1+sRJXZbCo5xzCq2opuZ8Y/tmhJrcvJBLm0H6wb9eZnb+fPLjpr0Z+Fc7DvTvjqb0FpGt7wFdh46uLfR2QNmPn/GNei5uXJQrnuVPYPXHMvX/jpY3zv58v/96qOOhIRWX4KuiJHoaHJIgBHgt/XmrHpapvwPXtHuXfvGL84ONHUezrn+NgNv+DAaD5qjQ5/DwNvOIwqm65WdMPn4q3L1aCbIF+qRPt2wyCbShgJqx0+9bevOYfffPb2Ra87m07MOXG5pXbfBP/+a3DFC2DgNnjZ5dD/+OX/XJFlMusM6RaFw7M++G1+5fIfznq8Gj5Xrmm6nduzRURW2wr860tEWm0qCG3L1RrcrNHYHt2JQm178VINDE/zsRseojeXZmTaD/hhiA1/7w2CbmcmydBEseZzp4oVPM8xXfKqe3FTCQplj8kZrctmRjaVjPbXArz8yScsad0bu7Js7s0t6bUL8jzYcwvc+Nf+ZOXu4+BX/xqe/Hro3LDw60XWsJlNEK0sgj52ZGrWYyqyioi0FwVdkaPQZGy/6VoUBt18yavup51uLuiGYXZoshC9ZzXo1u7R7etIR0eOhNcATBbLTBcrUaU2l/Zbl6OKbuwIoGw6saTjhGb6wwvPqDnqqCWKk3DjR+COz8L0MHT1+wH33LdpL660DVuleqeGkouItAcFXZGjUDg8KWzHXWtGY6F2cNzfmzuWb26t4R7boYni7GFU07VBd11HmpGpsOpbXctkoUK+VKEjXd2DWyhVmCwEQTcWbDPJRM15ukvVm0tHleamTQ/Djk/BrVfAxAE465Vw2ovhrFdApqs1nyGyRszqXJ6xsXUpf/95sQF+P3n4MM85ddOsa1Zy/6xCtYjI8lHQFTkKhS3LrWxdLpQr3L57mOecNvsffosVD7oHx4Kg22RFN2yB3jsyHZ2RG4bYmXt013VmGMuXKVe82RXdUiVqXc6mk4xNl5gqlckkE6SS1WCbTSei6curbmQP/OTjcMfnoDQJp/wyvPbf4XHPXu2ViayYmQH0rA9+e9Hv8fHv7Yxur+aMg5mhXUREWk/DqERWwJuvupUv/vSxlr1fNFiphUH32jv38YYrb2X/6HTT7xUPuofG80Dze3TDwBq2JMcfGy+UyaUT0Z7adZ3paB21FV2/dbneMKp42zJAZzpVU+FdNQfu9QdM7fgUPPEiePdN8JtfV8iVtvfzA+M196+86RE+eM29Tb3n5Tf8Irr9yR89wo7dR2Zds5JVVhV0RUSWj4KuyDLzPMePdx7m+nsPtOw9w0puOESpFfYM+wE3PLqnGWM1QTes6C5trSNTRQ5PFKLW5b0j1SAefs7oVCmq5gKs7/TPvR2ZLtW0TE8Ex4rkYkG3WPaYLFRmhdo/v/gs3nvB6Utac9PKBdj5Xfiv98BVvwLJLPzOzfDKf4Xjz16dNYmspIP3cWX67zjDan9A+JmbH23ZR9y1Z4TX/OvNLXu/xVA9V0Rk+al1WWSZTRbLeA7uGRjBOTf7yIwlvie0tqJ7cNSvvLZi3+/odMnf/1r2OBS2Li+xonvZ1fdwZKrIBWdunvVcfEDVpu5s9HhY0R2ZKs7aoxs/XigaRlUqR+3MoWedsnFJ612yqSNw1xfgzs/DofvBeZDt9fffvuCPYP32lV2PyGqaHuZFyTu4qvLSFUuFt+waWtT1A8NTnLi+c95rDo3nOe/D3+WTv3kuL37icdHjYedyK74fiIhIfQq6IsssbOMdnioxMDzNtg3z/8OoEVOF1ld0D4wFQbcF4XlsusSWvhy7h6aiSuxS9+jet38Uz6sOowqZVQdcHRovsLknHnSDiu5UqXaPbmHGHt1UgkK5wlSxQld2lf46LE7B9z8MP70CKkXYei780v+GLU+G014E6WU6mkhkLTP//6NJvBX5uENj+Zq/Kxbyk52HecOVt/Kx1z2FVzx165zX3bdvDIDP3fJobdAN0rtyrojI8lHQFVlm8Zbde/aOtiToLktFd6y1Fd2zT+xj91D1rMq5pi7fMzDKt+7dzx/+6hmzqhv5UoWB4Wk608lZ/wg9vjcXVYkHxws8/rie6Ln1QUV3OAi6PdkU44UyE7P26CbJlzymCtXHVkRpGu7+Mtx7Ney7Awpj8JQ3wbPeA8c/aeXWIbJWJVYu6FY8x3l/9d1FvSbcP3znnpF5g+5ClHNFRJaP9uiKLLN4y+7dA6Mtec+wotvKqcsHWhR0SxWPyWKFLX2157nOVdG97u59/POND9cMsArtHprEOb/KPDxVOyF12/pOxvNlPM8xOGdFt8hYvsTxfX5VdKJQplD2qnt000FFt/T/2Tvv+Cjq9I+/v9uy6R1ISCCA9CogVcTeT8+uqKe/07PrWe5O7jzLqZwoZxcL9t6xgYL03ntNSEJ673X7/P6Y3clusqkkEOD7fr14ZXfmOzPf2SyZ+czzPJ/HQZCpi4WuywlFB1Xn5Dcnwy8PQHU+jLwGbl0If5wrRa5E4sEd0dUdBaH7/KKD7RpfVmvD7mzfvGRNrkQikRx9ZERXIuliPALObNSxJ7eiU/ZZZ+9coWuxOzUTqiPdpyfy2jPMN+W2uRrdkhpVwGaX1WsC1YO3w3J2WZ3PuvgIM1szy6mot+NwKcR6Cd3QAAM60ZC63DPMzKGiGkpr1Hph79Rlu1OhxuKgb3QX9KGtL4esTbD/R9j/s9oaCCD6FLj5R+h/psxdlEj8oVOfwx+NiO681ek+7wUw/IlFhJgNfHbbRAZ6ZYsoisLYZ5Zo7zNLa2mJ5v53t9Rd6PNNmSRGBnHGoNjWpi6RSCSSFpBCVyLpYjyRzBHx4RwubvmmqK3UuaOudZ1Uo+tJW4amtbDtxSPs48IbhK4Q6n5dLgWdzvfWr7RWFZ855XWMTAj3WZdeXKO9zi6vJy7cTH6lhTCzgfBAI1X1dq19kbfQ1ekEEUEmyt1mVPERZoJNek1Ue5tRgZri3KmthGx1sOU9WDlbFbemUBh5FfSZAvFjIGaQlpopkUj80EqNbl7FkbdBa4lam5Nam5PzXl5NxuxLmh23Irn4yA7k50HXYz+oLZRaOq5EIpFIWqfLhK4Q4gPgUqBIUZQR7mVRwNdAEpABXKsoSrlQC/NeBS4G6oBbFUXZ3lVzk0iOJp7a1D5RQSQ36gvZUTyGUZ0V0S2obBC6RyqePULXO6LbM9RMQZWFaquD8ECjz/hST0S33DdiC74R3eJqKxP7RZFfaSE8yEhYoJEaq0Nzde4R6htBjggyahHd0AAjQQEGSjwRXa/2Qp45d0rqct4ONTV5z3dgqYCBF8DUB6D3ODAGtr69RCJRcT8IEn6SfpNmLuT20/t12aG9H/wB7M6pID4ikJiQgBYjsQBzV6SSW1HPf6/wbQOmNNpQpjJLJBJJ19OVNbofARc2WjYTWKYoykBgmfs9wEXAQPe/O4C3unBeEslRxRPRjY8IpNrqwOk68lscjxi1Olydsr8Crxu7Wmvz4rnW6uDlJSlYHc2P8QjdmBATJr36JyY+QhWh/up0y2pVoZtT3jRCk1ZcQ3RwQzpzTEgAoWYDEYEmQs0GXIpaxwu+EV2AiEAjFfU2VeiaDYQEGCh29/Q1mxrMqDwEHYnrcn0FLHkS5p0FOz6HAWfDrb/CjK8h6XQpciWS9iJaTl1+b+3hLjv0/35P8Xl/2RvruPS1tUDzAnVvbiWfbshgzuJkvtjU0Pu3tfZBsnBBIpFIuo4uE7qKoqwGyhotvhz42P36Y+CPXss/UVQ2AhFCiLiumptEcjSprLcTGmDQervWtKOFRXN4i9HOSF/2REVDAgwtmlGtTC7m1WWH2JZRri1zOF088s0u9rvbaHiEbnigUauFjY9QhV7jOl1FUbQoa+MaXEVRSC+uZWzfSG1ZSICBmJAAwgONhJrVzzOtSE1v7tFI6EYGmSiptlFvdxJqNhIc0DR12RPRBYhqVB/cJlwu2DAXXh4O616BU2+Cv6XANR9C0lRZfyuRdJSj3F7IG3/GeAWNoryNufT1tTz+075m12/JaLgdsjqcPmJYIpFIJF3D0XZd7qkoSj6A+2cP9/LeQLbXuBz3siYIIe4QQmwVQmwtLj7C2hiJ5ChQZbETFqim2nreN0duRT2rUtTv9V2fbuPnXXl+x9XZGlyCO6PFUEGVhUCjnl7h5hZ78+ZXqlHXYrc4VZdZ+H57DnNXpAK+Qtczx94eoVvvu+9amxOrQ72RzW4U0U0rrqHa6mDqgGhtWXCAgb9M68+MiX0INasR2PSSWoJM+iZ9cMODjGS5xXOo2UCwqSF1OTFKnU+AUf0TGBMSwHUTElv4hPyQsQ4+vAgW/wv6ToU7V8Plb4A5rH37kUiOIUKIC4UQyUKIVCHETD/rzxBCbBdCOIQQVx+1iena7rp8mW4d2wLu5BzdNmYavujSaTVOQW4rFnvDeby+LFV7LZ+FSSQSSdfRXdoL+ftT7/dqoijKPEVRxiuKMj42VjoSSro/VfUOVei6I5AeIVhZb28SOXh/zWHu/HQrTpfCon0FrEwuarI/l0uhzuYkJkSNYNZ6Cd3SGmuLacXNUVBloVe4meAAQ4upy3kValTDkwIMDanHv+8voKzWpqUnh3lFdHtH+o/oelyQI4OM5JTX+dxErk8rBeCsIT0wuwVpiNnAjIl9uHhknPZ5phXVNInmqvs0Ue92p+4bHUSIWwhHBZsY1EN1UZ12Six/O38QK/42Xdtfm9j6AXx0CVRkwmVvqCnKcaPbvr1E0g0QQuiBuajlQ8OAG4QQwxoNywJuBbpWQTaZXNtdl18zzSVaVPO+6UXuMixgnvFFMswzEJ0cDZ63Oq3dtbXeNzeeh5IV9Tav9VLpSiQSSVdxtIVuoScl2f3TcxefA3iHUxIA/6EsieQ4o6reTpjZQFigKrSqLHZsDhdXv7We2z7a4jO2ot6Gxe7S0uT8OYta3EI2JkRNtfWkLiuKwgWvrOH9DtSu5ZbXa87ELaVCeyK6Rd5C193f1u5U+GlnLlX1dkwGHWajXovoxod7IrqNhK5bJI9OjMBid2mpxQAb0kqJDzfTJyqISHdacahX1NYj9PMqLU3qc0EVzwAT+kVx9pAeWg3upP5RmvNzeJCR+84eqKVBt0pxCnzzJ1jwEAw8H+7fDmNvlmEZyfHKBCBVUZR0RVFswFeopUQaiqJkKIqyG45yDrEnoitaPqy/iO/5+m0AHDbf1KlT+mJTltaGra14C2OHS51rB4PCEolEImknR1vo/gzc4n59C/CT1/I/CZVJQKUnxVkiOd6pstgJ94roVtU7eHdNOoeKatiaWe5Tm+rpQZtZohoseSKo3ngirh6h53FeLq21UVJj9Wvq1Bq5FfX0jggkyGSgpqWIbmXTiG65W6yGmg0sP1hEZb1dc1YOMqrisqFG11dEexyXRydEAGqLIVCj1hvTS5k0IBohhNZfN8TcIHSHxoVyyUi1lL+x4zJAUkwwgUY9z/5xBEIIQgLUG+fJA2Ja/0AaU1cGq16At0+HtBVwxj/gus/AFNT+fUkk3Yc2lw21RqeXFbWxRneIOHq1rkXVVk6btbRT99nRZ2SL9hawYHfb4wH78ipJmrmQ3C5uyySRSCTdiS4TukKIL4ENwGAhRI4Q4jZgNnCeEOIQcJ77PcCvQDqQCrwL3NNV85JIjjZV9WqNrkf8FddYeWN5KuPcJksLdjc806l2p/YedjsJ51fW42rkquyJuHqimB6hm+sWuP6cjVvCYndSXG0lITKIkIBWIroVnohugwD3pC4PiwujqMrqK3Td4jIu3IxO+ApkaEhdHpOoCl1PnW5yYTXldXYm91frcz3R2RCviK4QgtlXjWR0YgTjkyJpzCUj49j2+LkM6qmmKQeb1G09+2wTlipY9rRqNrViFgy6AO7fBmc/BoYOmFdJJN2LNpcNtUanlxVpNbotT+cs3U4AHrHdxenWV5usN9G+v4ct0ZF2bt4fsNLopzfZZXVsPtzYv7MptVYHN723ibs+28Z9X+xo8zw85lfLDzYth5FIJJITlS7ro6soyg3NrDrHz1gFuLer5iKRHEuqLA7CzA0R3ZSCaurtTm6Y0AeHS2HhnjzuPnMA0BDRzXBHdO1OheIaq09P2sYR3Xq3MPU8qa9uxdX5QH4VkUEmeoWr+/SkR/eOCKSgytKs67LN4dJMqBrX6Op1ggE9Qli0t4DoEFOD0HWnLocFGjm1TyRrU4uBIdq2ntTlUQnhQENE92CB6uDsEcCRfiK6AKFmIz/dO9XvfIUQBJkaxk8fHEtZnY0BscHNfzgeXE5IWQS/PQqV2TDiajj9Ieg1ovVtJZLjh+5bNuSu0W3NjOrvxm8AWOwaTw1BjLG8wxhdKh+Z5gCQYr6Frxxnsk9J4lPn+V0750ZsyyznTx9sbnGMRwhPe2EFABmzL2lx/NIDhaxNLWn3XGS2tEQiORnpLmZUEskJicPposbqIDzQqIm0Q0XVAPQKM3P+sJ7sza3STKk8IvVwSUM6c+NUM0/ENcYd0fUI3zxN6DYfwVAUhZvf38QLiw822X/vyEB3eyH/UYvCKguKAmajzqdGt7zORmSQiR6hAZTV2iitsWlCN9BoINikR68TnDtUPdcdWeXMXZHKruwKSmtshAQYiA4JICrYRHaZOpd8d4q0J+XZ05op9Ah63U4bGMtL145pta8lqUvh1dHw1QwwmOG2JXD1+1LkSk5EtgADhRD9hBAm4HrUUqJjTxtTl/e4kgCoQS0jqCCUla5TudL6lDbmesNKnjF+1AWTbJlHv9/td7l3jW5FOzNwOornmNJNQCKRnExIoSuRdCEe4RoWaECvE4QGGDhUqPZ97RUeoLXd8bS98YjUDHfqMjQ1pPK4LMd6zKjczsKe2tzGdbDepBXXUFJj09KcoSHlOSEykCCTnnq7E6er6fN/j0HWyN7hVNTZNXfnslob0cEmrU72cGmtJnRPS4rkjEFqGuN5w9RuYjPe3cScxclcPncd32zNJipYPY/EyEAtoptfYSHMbNBaBjUX0e1UKnNh/p3w2VVgCoFrP4G710PihK47pkRyDFEUxQHcBywGDgDfKIqyTwjxtBDiMgAhxGlCiBzgGuAdIUTzzWI7E13rrsuX6dYzUpfhd912ZZCfpV0T15zy3LI2jduq9dJtmMfs3w6y5lBDTbO/v73etPqgzg9qqYni3r7dm0skEslxixS6EkkX4mmn40lbDgs0aum6PcPMRLvFammNDUVRNGGcVdoQ0W0sdD2pys2nLjcfIdiSUQ74uibnlNej1wl6hZm1OlZ/dbqeeYxyG0d5HJLLa+1EBhu1mmGbw0WYW5BeP6EPb900DoABsSEkRQdRb3fy4jWjuWx0PDVWh/YZJEQGaWI9v7Jei+ZCQ0Q35Agiun5RFMjfBUuegNfHwb4f1BTlO1bAsMtlHa7khEdRlF8VRRmkKMoARVFmuZc9oSjKz+7XWxRFSVAUJVhRlGhFUYYflYmJ1vvovmZ6o8VdJFl8OyKFUdvMyCPDY9LnTVG1BV0jUbnsQBEWu7OJ6/LH6zO013/9qmnd7bbMMp5fdLDJcg+KovDFpiws9qbZOOtTS5gwaxmL9xUC3aOdUUphNTuzK471NCQSyUlAF4ZHJBJJpVdPWVCdiQGCTXpCzUYtmllWa8Vid+FwP823OV2EBxpxuZQmzsue1OLokOZSl5uP6G5xRxTUNGQFIQS5FfX0CjNj0Ou0CGqdzdmk5Y4nndhTT1tcbaV3RCBldTYG9Qzx6WXrieh6I4TgXxcPpajaylXjErhoZC/SimsY7DaLSogKZMn+Qu2c48Ib6pIvHhlHtcWhRcCPGFsdrH8Ntn0M1XmAgOFXwLlPQWTfzjmGRCLpOLqWU5dDaXgY2N/yWbO7ud92Hw8ZvqO/roDbDL/xsuOazp1nM0yYtYyESN+/V59vymJDWinpJb6Ce+mBBoMob3NCD1e9tQGAS0fF8cCXvkL4g7WHeXrBfgBWpRTxzs3jfdbvya0EGkwDuwPnv7waaL0eWSKRSI4UKXQlki4ktUhNU46PUEWbR/D2dIs4T1S2pMamRX89RAQZMRv0zdbohpoNmI066t1P8T3j6mxO7E4XRn3ThI2t7ohunc1JjdVBqNlIbnk9vd03ZMFul+Qaq4OejbbNKa8jzGygf0wIAEXuVObyWrVG17uXbZgfoQtw/vBe2usgk4Gf7p2K3h32SIwMwuZ0UVRtJb+ynjF9IrSx8RGBPHSev1TEdmK3wIGfYelTUJULAy9QHZQHng8hPY58/xKJpHNopUb3Yv0mAKqUQFwtJKf94ppCnSOA900vEktl58+zBfy1emsscltiZ3YFG9NLtfeXvLa2yRiPyAW0qG1LHI3U5fTiGjakl3LjRPnQUCKRHFtk6rJE0oWsTikmOtjE0F5hQEMKcy+3i7Kn9rS0xtYk5Tgi0Eh8hJltmeV8vD4DxZ3vVl6njgsJMBBkMlBnc1BrdVBRZ6dnmCo2a9xR3WUHCtmRVY7LpVBYZSGrrI7R7ohsYZWavpxTXkeCO1LqcSmu82NItS+viqFxYZqgLa6x4nIplNfZiAo2aaId/Ed0/WHQ67SaM0/041CR2looPrxpb9wOYbfA9k/g82vg+SSY/xcIjoH/+w1u/AZOvUmKXImku+GO6CaJAibr9tG4vnai7gAA19qebHVXK11jcCqCGYblZJhndGrLoa7kj3PXMfu35lOW/XGosJqkmQtJKaz2u95b52aU1GrXFYBvtmZz1v9W+ozfkVXO0v2tC2hv/vD6Wh77YW+7tpm1cD9JMxe2axuJRCJpDSl0JZIuwuVSWHOohNMHxqBzRy3DAlUh6RG6JoOO8EAjpbXWJiZS4UEmbpmSREyIiSd/3keK28QqpbCaxKhAzEY9QSY9tVanlrY8xC2oqyx2Smus3PbxVq54cz2P/bhHS1u+ZFQcoEZk7U4XBVWWJhHd2kY1unani/15VYxKCCcmxIQQaqp0lcWOS1EFu8mg0/rdtlXoepMYpbqmbnH3kowLP4I05fpy2P2tai718jD4+X4oTYWxf4Ibv4O/rIC+Uzq+f4lE0rW42wtda1jFl6ZZZJhvZJjI0FaPEukAJCsJre7KiR69aBB0Lxnf5ERtuPOLO/V5oftn47PcnFHGutQSvt6SxZn/W8l323K0df/4bjeHG0Wcr3hzPbd/srVdc6jtQL/hd9ccbvc2EolE0hpS6EokXcT+/CpKa22cMTBWW+aJ6Pb0ilZGB5sorbVptbWeqGxEoJEzB/fghatHA5BV5ukxW60J2piQAIqqLeR4hG6cWu9abXFotb1x4WZ+2pnH+rRSAo16zhqsRi8Lqy3kVdTjUqCPW2R6zKg8vXTrbA42pZeSXFCN1eFiZEIEBr2OUQkRrE0t1eq+PLXGnmhvR4Sup/52s1uQx0V0IKJbWwo/3Qf/Gwzzb4dDv0P/s+BPP8P92+HiF2DgeVq0SCKRdFNE09uTsbpD2utTdGq7X6UDtzGX6jeRYb6RvqKg4/PrrigN7soFlRafnucA87fncuN7m3j0+z0Afk2hDjUTDW7b4Y/9AwRFUdiWWdYt5iKRSI4tUuhKJJ2M5+K6Mlk1GJk2KEZb56ld9UR0AaJDTJTWWLXUZY/o9ERHPSm9OeV1WOxO0otrGBqnCt34CDP5FRatRdBQr4hufqW67JpxCdTZnHy/LYdT+0RobsaFVVZNPGtC121GVWtz8t22HMY+s4Tr5m3kH9+p/SBH9VbTns8f1pNd2RUkF6g3RJFuoetpMdRcjW5LmI16EqMC2Zap1hHHtyei63LCzi/grcmw6ys1Hfn2ZfD3VLUHbv/psq+GRHI8IUST2ttnjR8yXGQQS/sde+c6LqNKCfJZtirg4SOaYnfEI+1eWXqISc8t4/217Y+U/rI7nw1ppVzy2hqf5Yv3FZA0cyGpRdXU25y8vCQFm6Ohhnrz4TIW7W14eNCa0FQUhZXJRa22VGov327L4aq3NrBwT1Njr8Y4XQqvLE1p4pHRXaizOVidUtz6QIlE4hcpdCWSTiS/sp4xTy9h0d4CftiRy7i+kZr4A7S2Oz29hW5wgLtGV42iJkaqN2Ph7vrd6GATgUY92WX1HCqswaXA0F5q5DYuPJC8ynpyK+ox6AQDYlWjqKp6h9b39oqxCRh0AqvDxWlJUQQHGAgNMFBQaWkQutEeoatGOn/fV8C/5u9hVO8IxvaJYH9+FaFmA33d484dqlpVfetOe4sKOvKILsD9Zw/E7lRvenq1VqNrq4ODv6oR3BcHw493Q1g8/GU5XPoSJIyXkVuJ5Lim6cOphQH/Yov5HgD+Y7+5zXua47ieUdb3Om1m3ZXXl6e2a7w/ifnaskM89sMe9uVV+Sx/dakaUd+ZXclbK1N5ddkhhj6xSGtrdO07G7j78+0N+25Fv/6+v5BbP9zC+2vT/a63O13MXZHqt21SSmE1rmYEcnqxmn7tub61xOJ9Bbyy9BD/XXig1bHHgn/O38OfPthMenHNsZ6KRHJcIoWuRNKJ/LIrj8p6OzPn7yatuJbrxif6rNciuuGNIrq1DWZUnlrVCPdYIQQJkYHklNdxoEC98RjijujGhZux2NX62bgIs9ZvttpiJ7/SglEv6BsVxNi+kQCclhQFQI8wNeU5q6wOk15HT7cYjw4OYHDPUBbszic6xMTbN4/jsUuGAWpbIY9x1KDfpRtmAAAgAElEQVSeISRGBbL8YJG2P0BrMdRRoXv12AROS4okLtyM2ehHpLpckL0ZfrgLXugPX90A+3+CpGlw3edq7W3cqA4dWyKRdC90tFzrudQ1tt37HGD5lIGWT7T3Txg+YYRI5yb9knbv60RAUdSev4+6s3Y8uPyo1P356vUnq6yOOncdrtOl8OmGTP/7buXYBe6WddllTd2pQTXHmrM4mTcaifdd2RWc//Jq5q3xL5CVVo5sd7qwO9VItCciXe9HTHcH0twCt9aPQaTkxKWizqaVkEmODNleSCLpRBbszickwEBFnZ0gk56L3cZPHs4YGMsNExIZ6q6lBTViW15no6LOjhBoxlAe0Qqq+M0ur+dgfjWBRr2WauxJQ96eVc6wuDCtT2+1xUFBpYWeYWZ0OsFFI3pxIL+KU90te3qGmSmssqIoav9aj1mWyaBj0YPTOJBfTWSw2uc3KtjEg+cOZHh8uDYfIQR/O38waw+VcOnoeC1C/YfR8SAgyNSxSKpOJ3jvltMorfGqK1MUKE6GPd/Cjk+hphCMwTDmBhh6GfSdCgZTh44nkUi6P7fbHmGpaxzX6FcyxzgPgLcdl5KtNG6C1jpO9DiBJ+238B/jx/zZsIg/GxYBYMXIt84zO3HmxwMK172zsYkJVUapbzT0to+2+Lx/zysl2u7y3wKqxuKg3u6kZ1gAG9JLmdw/WntYCg2pzc1VldS7xXRjc0RP26ZdfuqL3aek7tdPRgDA+GeXqgaLT1/of/tuhCwzPjkZ8/QSeoQGsPmxc4/1VI57pNCVSDqJzNJadudUMvOiIaxKLmZ4fBghAb7/xXqFm3nuSt+IY3RIAIqiPiUPCTBoacDeQjchMpAtGWXszTUwqFeo1ns2zh0ZrrY46B0ZqB3PU6PrWX/L5CSuGpeg1eD2DDOz+XAZFrtTE80ehBAMiw/zWfbguU172F4+pjeXj+nts2xE73BG9A5vMrY9hAcaCRf1kLoUUn6HlEVQkQkIGHQBjLhaNZQKjGh1XxKJ5Phns2swAIddDX24rRzZw62PnRfwH+PHPsvmGOfxrXM6/lKmT1S+3JzdpnHL3Nk70NC+zkNKQTUz3t3YZJvRT/8OwDN/HMHjP+5l7oyxmus/NER8/aUmA5oo/nBdBpFBJh44Z6B7uXt7LxFoc7jQCbVlnaJt7/9cKuvV7KmvNmdpWUne+yqsshARZCTAoMfpUvhqSxbXjU/EoNexP6+K3/cX+L0mdiXSYuLko6iRkdyRsjWjjNjQAPpGB3fqfrs7MnVZIukkViarhhGXjIzjyzsm8e9Lh7Vpu+gQ9YYto7SWMLORUYnhTO4fzcjeDUIuMTKIaouDLZllTB/YYG7liegCJEQEYtDrCDbptYhuL7ehk04nNMdngKFxoeRW1JNSWN1E6B4TytJh87tqO6A3ToPZfeCzq9T+tz2GwaWvwMP7YcbXMOoaKXIlkpOIKlTvga3KYG3ZZ45zjni/51ufb7Isw3zjEe/3ROeDdb4GVz+6Xf2bI9MdLfa0wfOw310D/M3WhhZHzy5Q++mmF9f4PG54aUmK9tqzfNG+An7amQvAoH//xhVvrgfQegi3pg1nzt/D7356BE/87zLu/XwHAF9syuSxH/Zq5/zHuet4ZemhZuuDvfltTz5JMxeSW+E/NVtycrJ0fyFfb8k6Ksey2J3a/4er397A9Dkrj8pxuxMyoiuRdBKFVRYMOqG5JLeV6GC1rjWzpI7ekYH0CDXz5R2TfMYkRqn7VBS4dHS8tjwmJACDTuBwKZroDTUbqapXa3TPGxbg95jXjEvkxd9TsDpcx0bo1pZA1gbI2qhGbosPqstDekL8WBh5DfQeq6YlG4+gn65EIjmBECRZvui0vaUoiSRZviCcGkbqDvOZ6TkAhovD7FP6ddpxJCqzfj3ATZP6au+/9erh68GTEn32i6t4ws/D4oMFVbyytKHN1F+/2qllFu3JrQQaHjoD7Mur1MpuXlh0UKsLbkxKYTXrUkuYeor6IHnpAVUAe6K/FXXqT0+adlsyij09ig/kVWnt844lFrsTi91JRJAs9WmJBbvzGNsn0ieQ0Jl4+lJfd1qfTtnfv3/cw0Uj4rTvrjePfr+bn3bmsfOJ8zrlWMcjMqIrkXQSpTU2okNMPjVIbcHjVFxtdWg1to1JcDsxD+wRwqCeDfW9ep3Q6mM9tb1hgQayyuqwOlxaRLcxkcEmrhybADSYX3UpVfmw5ztY8BDMnQhzBsDXN6lR3OBYuHA2PLADHkmGGV/B9H/AKedKkSuRnMTYDGEscE7s8uNUEsJa10iqFfXvzcKAx7r8mCcT3vW8Q59Y1ObtdjSqwd2SUcaFr6whuY19fp/77SCXvLaW+dtVwfnmyjTm78j1O/ZgQTU3vrepyfLG13PPu/b06C2qtpJa1D7XZEVRmk3p7ig3vreJMU8ff6ZrtVaHZtbZ1SiKwn1f7ODqt9b7XV9YZfFpqeWPpfsLfdpVrUguIrO0toUtVGqsDqwOZ4vfLbvT1WT9Zxuz/H53QW35BWjmcUeLepuTqbOXsy615Kge1x9S6EoknURprZWoYP8R1JYYEBvMOUN6AJBZ6r8dQp/oIEwGHX88tXeTdfERbqHrFdE95L6oxrXQoufu6QOYNjCGcW5H5k6nIltNPf74MnhpCHx/G+z+FsIT4Zwn4c+L4Z/ZcOsCmHQ3RPWXhUgSiUTjlh7fcp/9r0fteBOtc7XXGeYZjBJpnKnb4bavkhxtftmV5/P+mrc3+B1ndbT8+3n4m10UVfuP5LYVj7TwZCy/uTKt1W08l7N//bCHc19a1a7jvb/2MEMeX9SpdZqeHvVtweZwtSrojhZjnv6dkU/93ubx61JLuPeL7e16GOHB8/vNr2r6fbE6nEz87zIe/d7XoXzJ/kL+88s+QE3Pv/2TrTzw5Q5t/f99uKVNKcMjnlzMlW+ub7ZFWG5FPQMf+42vt7Strt6b1j6JK95cx5TnlrV7vwCzFqrlBt6kFdeQW1HfrAA/mkihK5EcIfO351BYZaG01kZMSPtTgoQQvHjtaAAuHhnnd0yY2ciSh87gzjP6N1kX547aetJswswGymptQMu9aPtEB/HpbROJCWm/OG8Wa41qIPXdn+HVUfDz/Wr97Vn/hjtWwqMZcNN3MO1h6DMJDJ14bIlEckLhbEMdZGdSh+/fy58DHucj0xzSzDezxnT0BLekfZTU2LTXnghWY15cnOJ3eWMm/nep3+WNNdNXm1uusVTrclt/cGtzuPzW+/7sFvnFbRC67W1F8+rSQ1q99NwVqSTNXNhEFE6ZvYzBj//W5n22lZIaa5v/X7tcCt9ty8Hu9B1fVmtj/LNL2OtOVS+qtrAru0Krhb7xvU0s3J3fbsdqh9NFRZ36XfL3m/MI/9/3Ffgs/8snW/lwXQbQ0KaqcQ/qtrIvr4rvtzdN6Qe0XsoLduf7XZ80c6F27+fBcx6Hi30jymW1Nkq8ulvsyKogz53W/59f9pE0cyEul0JWaV2rD5LeXXO4ybLkgrZlXRwNpNCVSI6AomoLD3+ziy82Zampy8Edq32JCDKR/OyFPPmH5g2s+kYHY9A3/S97+sAYzhwcq/WdHdc3ErNRR++IQAbEhnRoPm2ivgIOr4b1r8P3t6smUs8lwBfXwKElMPleuHcLPLgHpv8d4k8FvbQFkEgkbeOikb1aH+SHjvbxBhhk+djv8kRdMePFQTLMM8gwz6BtVZqSo8HU2cu119e+4z/q+/XWtkXBCqt8haUnKrtkfwHrvdIwFdQHMSmF1Qx/YhE7shqipcsOFDJ19nKtzrcx93y+jYVusTLo379x/1c7moxpTSL/tiefqbOX43C6GPP0Ek6btVRLj3W5FBzO5qOxLy9N4dLX1wIwZ3Gyej6Nvs4lNbYWheKnGzNZe6hpWuqFr6zmhx0NQu2GeRu11PGSGivjn13K/35P1taX19r4x3e7tFZS3ny/PYe/fbvLZ1lqUTVjn1lCSY2NV5cdYld2BRNmLePyueu074HO48rd/PT98q8f9jDu2aXufTT8Bt5bk847q9K83Lz9/3ZcLkX7zNrygOKRb3b5Xe79uf+4I5ekmQt9Ut/XppZQXG2lrNbWxO18UqOobKlb+N70fkNkNWnmQsY+s4Txzy7l/bWHyWjUWswj2vv/61fOmLOCf36/hx1Z5T6p9HMWH+SxH/Y0f27f+j+3Y4G865RIjoDUQvWPT055PaU1HUtd9hBg6Fjv2WvHJ3Lt+ETt/X1nD+S+swd2eB5+cdjUyGzOZkhbAXk7oNzrKV5YAsSNbjCRSpomo7USieSI8HbjbQ+9wsyaiVB7sWFkuOV9vjQ9yyidb6Tiu4Cntde9KSGX2A4dQ3J8kDRzIX+arJpnpRXXMsMrDbOy3s4zC/bz0foMAOatTuetm8axN7eSH5qpA/bw654Cft1TwCWjLgFg4e585s5oWL90fyG7cir9bmuxOxEC/vnDHirq7FS5Wz3V2ZxMn7OSjNmXcNXb69mRVUHGbHX/+ZX1mBvdX3gif0J0rFfv4z/uBdCO4eFgQTUPfb2LK05VPUA2pJeyIb2UK8cmaBHEZQcKuWlSX/Ir6vl5Vx7fbM1heHw4t0xJwuVS0OkEO7LK/aZa/7qnIZq6ZH8hSxq5Zr+5MlVLP3YpCsKltltsbMD12558yups2Bwu/m+qajz3446GVHlvLfvswgOA2gIS1IcQ1769gVumJPm0y7pu3ga2ZPjO+edG6fenzWrIGPh+ew4vXjuaxfsKWOHVvsvl9Qt58OudANz+8Raf3ta7sitILqxu4nZuc7iotTq0VpLWVtLPn1mwn883ZbY4Zv6OXObvyOX60xKZfdUoluwvZO4KNXV/1hUjtXFZpXU89cs+5s4Y2+L+jjZS6EokR0CqO5UktbiGWptTaxV03GKrhZIUKE5WnZCLk9V/5RmguJ/mhcZDwngYe7MqbnuNhhB5wyeRSDqXjmbIBJrUm/rLRsfz2g2nNqkfa41aArnMNkt7H4CNZPOtPmNeMr1FttKDx+x/JoR6Sjmy/uGS7sknG/yLgDqbUxO5ALU21dHYEyltL5e/sZaf7jsdaHDl9eaVpYd475bxDHl8ETEhAZo4bRxbLK+1sSPL18Rr8nPLtShnc7RF6yqKwgfrMrjCj1cIwJpDxX6XN2yv/hQIpj2/HO8MZrvTRXJBNRe8sroNM2meFxY1RIsVBeYsSeatlWms/vtZxISaCDKpsufuz7dr4644tTcRQSYUr0/B7lRImrnQR8h/vinLcwJszihjc0YZxdUNWXiNRS7gU6vb3N+hOz/d1up5ZTTyb5n16wHOH97T79jhTy5m1hUj6NfGfrkeR/HW2JtXicXu5C9e30/vlPf7v9zOrpzKdhnOHQ2k0JVIjgBPOsmBfLUeoyM1uscURYHKbEhfBXu/h/SVaJc8nQGiT4Gew2HElRAzCHqOgB5DpWmURCLpch46bxBr2+jaOfWUaNalqtGNGRP68PxVo+gb3TmO8lZMDLB8ylX61exz9WNhwL+YqDvIRA5ytb7hxrxYCSdWVPKA7T5+dk0BQOBC8VMl9plxFqfr9zHG8g4VhDZZLzm+WJ1SzIWtiLSWHrjsyqlk2BOL2PvUBX7XLz1QqNXhetdWnvqMr4vyu2vS/W7fXFmsR6cUVln8ttOxO10s3V/IucN6sie3kmcW7Pdx0q2st7MhrYQLR8Rx8/ubmz2/pJkLeeGqUYB6+9B4Ps8uPMDdZ9r8bNlAe82xJv53KeVuEXfGnBUIAYefu4TPNvo+vPDUDPuLbN/8flMzJYdXzfBTv+xv9vieqHd7aUva8+GSWt5Z5f93DfDYD20/duO63ubYm1vFkMd9RWy/f/6qvW4uC+FYI4VuI6wOJysOFnPB8J7tbhMjOfnwCF3PH+AjSV3ucmx1ULgPCnZB/i71dXEK2NymARF94PSHIH4MxA5RXZD1Ha91k0gkkiMhPLDttyi3n95fE7rXnpbYyuj240TPN86zWhwTK9QbvddMb/Aab/isG2L5kIPm/wPgfcdFnK5XXVp3mu8kyfI5AgUFQVsMjCTdk8ZRt9Z4e5Wvc3Odzcn0/61odvzwJxe3us96rzrKb7Zmc824hGbHnjmn4VhTZi/nh3umEBce6GM+9OaKNF5emkJiVCBTB6h9Wr1b/fz1qx2sTC5mzT98/2889fM+n4g3wBq3QD5c4r/VzlutOFkP+vdvXDSi7XX75Y0ilYri/2HDuGeX8sh5g3D4eRqwxk8dcn0b2z59urHllGCAp/0I5dbSjbuSzm5p1R0QHbHf7i6MHz9e2bq1aYrHkTB/ew4Pf7OLb++azGlJUZ267xORr7dkMaFfNP1i2pYicaIxYdZSyutsmivg/HumMLZPF7XraStOO1Tnq+nGBXtUUZu/S01JVtx/QAMjG6KzsYOhz2ToMUxGaiUnPUKIbYqijD/W8zie6axrc3ZZHdNeaP7G35vDz12sRRca1w22N3W5NU4ROSwN+AfP2G/kdsNvRFKNWXS8z+eHjgv4P8NiypUQTrXOAyAQC1FUyzpgyRFxwfCeLN7n3xSro4SZDVpt8IDYYNKKazmlR0i7ewVLTg4a/z3uKB29NsuIbiM8/UfXp5ZKodsKlXV2Hv1+D7dM7st/Lh9xrKdz1Kmst1NUbWXKgGjNECDmaEd06yugNBWKDkDuVjX1uDwTn4qb0Di1lnboZerPuNEQniBFrUQi6dZ4am29+fz2ie3uzbjrifMZ/XTb+3C2RqqSQJLlCwDed6o3cUYc/G76O9GiijBR36b93GN7gDdNr/F/BjVSFylqyDDPwKoYCBAN7WI8x/ImABuzje/yseMCdiqnaMuDsHCaLplVrtEdPj/JiUNni1xAE7kA1e7XUuRKuitS6DbC06dqQ3oJf6WTnWuPIyx2J7d+uJm/njOIyQOiURSlSSr3wQK1LjW9mTSUEx3PH/bpg2I1odvlZlTWalXUHvoddn4JVV6upAFhqtvxqOsgrLcqZnuNhJAeXTsniUQi6QICjU2F7tRTYlrcJiKoablFeJCRi0f28nFs7WzsGDjL9jIAwdRTSyBGHLgQpJlvbjL+Hccl/OqaBLzWZJ23yAXc7Yz8c4V+HUucYzlPv73JuoGWT7A3us0LpwYXOs7TbeUH1+l+64ePFB0uXLJ75UlBURvqSSWSY4kUuo1IdzdV3p5ZgcXu1HqTnmzsy6tkY3oZAYY0Ugqrmbc6neV/m+7TAueguyF0evHJKnTV858+OJbnfjtIgEFHkJ8IRIepKYbMdWorn+JkKD6gpiMDIGDgeTDhLxAzUK2pjewHOnlzIZFITgwaX389dYBLH56OzeHi4tfWaOuEEPz3ipFMGRDtd1/PXTGqS4WuN7Wopj4ekZlk+YLRIpW9Sj8MOImghkLUjLFDrt4M1PlvR+NQdBhE6/V6/kQuwCHzn1rc7iXeZpDlYx43fMrNhoa2J94R5ABsGHBSixnv+uGzdDtY7xqOFd+Hu68a3+By/XoqlGDGWN9tde4SiUTSlUih64XTpZBZWsegniGkFNawLbO81afHJyq73e5pqw8Vsz2rnGqLg60Zvp+HJ6KbW1FPvc3pN83sROZAfjVBJj2DeoQSE2IiwKDvmIGZokBtsSpiszZAzhZV2JakqOt1RlXMxp8Kp96k1tbGjYaw+E49H4lEIulO6Bv1REmMUl2UT+kRAtAkSjtjYp9m9xUeZOSrOyax5XAZLy5J6YLZtswud3qxE70mcgHOs83hfv18Frgm84BhPpfp1vOV82y2ugbxg2saD+jn87DxOwDedvyBuwy/aNu+4riSBw3zfY5zp+0hxuhSudtrXEukmG9psswTQR5k+bhJW6V+ls+YY5ynuU2fZplLMaovRZLI53L9egAiRC0X6jbztukVFjgncp/9r22aj0QiObGorLcTHnjsjE2l0PUip7wOm9PF9af14aUlKfz1q528fN1opg08+cwgdudUEhJgoMbqoNriQK8TrE4p9hG6B/Kr0blt4g+X1DIsPuwYzvjoc7CgisG9QtHpBL0jg2izsZvDptbTHl4DGWsgdxvYvdwaowaoBlFjZkDf01VRazjO2hZJJBJJJ3LvWQOaLHvzxnHtMpqa1D+aSf2juWxMPNPnrOzE2R0ZrzuvBOAh+708xL0+615zXslr7vUAsx03+Kx/xXF1k/0tdp3G644r2G/+s9/jnWOdwyTdAWYZP2hxXv5E8GHzTT7vt5jvbTLGw9umVwC4VL+JIF7gbP1On/VjLW9TRhj/MnzONN1urrc9TiXqQ4wYKjlXv40UVwLblUGAwpfGWRxUEvmPw3deIdQhgGo6p52URCLpPFalFHPZ6GMXmJFC1wtPCu6ohHC+vnMS93+5g0e/283aR89G11q3bS8ySmr5dW8+d54xoMkT6WNFaY2VqGBTmyOOu3MqmNQ/CqNeh9mop7DKwqqUYv558VAAXC6F5IJqJvSLYmN6GasPFbNgdx4PnTcIo/7ET59VFIWDBdVcNCIOgEcvGIyzOaGrKKrr8Z5v1VTkooPgqAcExI2CsX9SxW1EH/W9jNRKJBKJD3+/YIjf5Y9eOASnq33tOPpGB5Mx+xJ2ZJXz/KKDbEwvA2DawBi/7USOR+owNzGxCqWOGswo6Ehz9uYG/XJG6DK40Dqbg0pDNHy4OMzCgMc6fOwkyxdN6oobi1yA7ea7fN7vMt/R6r4ns5+huiwm6Q40WTfJ8joFRAMKIdSz13w7AJ85zmGO4zrCRC2FShSJogg7BlYHPMQoyzyqCCGGSraa79b2Nd36EplK21vZSCSS7okUum52ZlewKqUYgP6xIUQFm7j/7FN46Otd7MguZ1zfhlSjjJJanvplH6MSIrhhQiJx4b5Nth/7cQ/rUksJCTDwp8lJTY6VV1HPpsOlRAcHMG1gDEIIaqwO9uRUMql/VKf3792QVsrN72/ir+cM5P5zWjfYqrbYSS+p5fIxvXnAPf7tVWnM/u0ghVUWeoaZySqro97u5KIRcWxML+N/i5NxuBQSIoNaTB87EdibW4lBL6ioszM0LhSAKZ5It8sFNQWqYVThPvVfzhYoS1NTkPtOhvH/B32nQtJUtc2PRCKRSDrE3Wc2jfS2lVP7RPLclaP4w+tr+fm+qaxLKz1hhK4/Gkc8L7X91++4fUo/qpVAQkU93ztP5xH7PQBcplvPH/QbeNtxKduUwYDCAtNjmLFxii6PX50TeNehulB7i+yWzLQ6gj+RC7DRfL/f5TcZlnGTYZnfdbubEderAh7mcfutxIkyPnRcoKVnX69fTrIrkR1Kx81Kx4uD7FYGYEP2qT8R0OPEiAMLR7nrxnFC5jE2rD3phW5ZrY3//LKPn3bmAapjY1SwmiZ67tCemAw6vt6Szf68KvrHhjAgNoS7P99OenENq1OK+WDtYSb0i2JHVjkAkwdEsy61lDCzgTmLkukfE8LgXqFaJ5ec8npu+2gLpbU2AJ76wzBunpzE3Z9tY82hEqYNjOHUxAgAekcGctVYtdn3v3/cS25FPW/MGEuY2cCK5CLKau30iwlmT04F5w3vRVyYmbI6G0a9TsuHzyyt5f4vd+BwKcxbnc7E/tGsSy3BoBNcNS6BwioLK5KLCTTquWpcb2b/dpBN6WUoCoxMCNc+p3OG9GD2bwf5x3e7eefmcVoj8In9o+gdEUhuRT0mvY43lh/iqnG9Mel17M6pZPPhMkYmhGs1VaDaWXiiy3U2B99uzWH6oFj6RgdRVmvzboxDSIBBMyTZfLiMtYeKiQgyccOEPgSa9OzPq2JbZhnTBsbSJyqI//yyj105lfz7kqH0iwluEsW22J2sSimmqt7OH0/trUafU36H7R+D3gh6Ew5hYFdeHZkVNsYmxZLUIxx0BtAZ2Ztfw5L9+QQb4O8GO+dnx0JOOVTlqf+q88Dl5ZgZGqfW1E65H4ZdDkGyZZVEIpG0lZiQAEpqus7ZtV9MMHv/cwGgRnof/3Fvlx3reGKk9T0mioNsUoZqy352TeFn1xSvUaJZsexNc+2RPPW/u1z9udz2LAYcnKPbwTuml7XtoqlkmzvSeqn1WRYE/BuABc6JvOy4mmUBf+/gGfrnV+cELtZvBuAZ40cA3GP4uc3bN2cgttI5mtccVzA/4Clt2eP2W7Vj/Nd+A/Ocf+AC3RYu0G/hUfsdfs3EbrX9HQcG1rpGastSAm7GJJxcbn2aWwy/c6V+Lcucp/Kh80KeNHzCVbanqCIYgPeMc4gUNWQpPXjRcS05SiwJopiLdRv51jkdJzqMONlmvpsDrkQusj3PaJGKAz37lH4AWu31MMsH1GHW5pEoCnnd+DqrXaN42XE1Cjr0OAnESg1B6HHyq+mfvOq40u047p8oqrhZv4QvnWdTRCQjRTr5SjQlqPeksZQDgmIimmwbgI0warUHE0Yc2NFjwoFAaWKe5iEQCyNEBtuVgTjRY8LOFN0+VrtGkSiKyFZ6+HUSF7g0V/UBlk9JEMXMM77EvfYHKFdCKSe0yXbrA+4jXpTxruNiZjk8pQAKN+qX8YXzbJabHqGfrpAkyxc8b5jHdYaVnGF9mWwlln6igHTFO+tP4RPjbM7Q72G0ZZ6W+u8PM1bsGHCi3lOHU0MlIYRRw0X6LSxzjtU+4+Eig4UB/2Ko5QPqvX7H/lEQKJhwEIBd+655sDral3HT2Yg21xV2QzqjKf32rHJumLeRu6YPoGeYmahgExeOaEhXufPTrX77kH1462kMiA3h8Z/2klpUw9RToqmzOVmwO58eoQF8/OcJ3PDuRirqmjaR7xVmZu6NY3lrZSqrUooZ2yeSTYfLuOLU3iw7UEi11YHn1zKkVyiBJj07sirQ6wRJ0UFEhwSw+XCZzz5DAgwEmfSa1fuohHASIgNZcbAYg04w68qRPPDlDp9tgkx66u1O7VgGncDhUhiVEE5ZrY2F908j3KtVw9dbspg5fwGTV3MAABWfSURBVA+RQSbKam38eWo/nvjDMG75YDM7syuYfeVI7v58O+cO7UG93cm61NJmP/feEYGMTgxnd04lOeX1mI06ooMDyK3w7T8YYNAxfVAsw+PDeX35IRwudbKJUYEM6RXG8oNFON3LeoWZKaiyEGo2aL3dEiIDGZ0QAQLqbU42ppdSZ3MCMKhnCAN7hvJw1n3E2w5TaYhB57LjctjQK3YChBOd4sSkc2FQnOhw+szNrugxmMyI0F5qunFYb/fPeNUFuedwKWwlkuOMjjallzTQGdfmY8WB/CreXpWmPfz2Zv3Ms5kye/kxmNXJTRAW6gjA2/W5MQHYeNP4Kl86z2apaxx6nNoN/WTdPr40zeJO24O8Y3qFfpbPUNBhxMEk3X6iqOJy/Xput/9NEyUfGZ/nTP2uo3F6Ei9ec/yRBww/dsq+sl2xJOqKmyy/xvoE3wY8rb1Pd/Wiv65rHdkftN3DK6Y3O2VfC50TCKWeDa7hRIpq7jC03aegNW63PcIjhm8YqstudaxFMXKv/QHeN73od71d0TPQ+ikvXD2Ka8cnHvHcOnptPumFLkBJjZWYEP8pB1szynh12SHuOfMUqix2iqosDOgRwpQB/t2Y9+RUYjLoGNwrlHqbk9WHiimqsmjrhRCcM7QHceGBVNbZuf+rHeSW13HB8F78/YLBWvRRURQW7M7n7VVp2Bwubp7cl4TIQP63OAWHy8VVYxOYPCCajNI6+scE89bKNJwuhYn9o6i1OliRXExFnY1BPUN5/NJhxEcEMmfxQUqqbTx60RBqLA6eWbif3hGB/O2CwWSX1fHcbwc5f1hPbprUt9nPas2hYr7anA0CXr52DCaDjoySWmqsDobHh/H+2sO8sDgZk17Hw+cN4sIRvdiZXUGp1xN5m1NhQ1oJh0tqiQgy8Zdp/Vm8r4Bqi53JA2Iw6RsuZmnFtSzaW0BBlYWxfSL46M8T2JdbxZzFB6mstzOhXxS3TEli7aESluwv5MzBPZgxsQ+L9uZTbXGwPq2UzFI1bUKvE4xPiuKiEb2osTh4c2UawlrJDzU38aXpKj4MuAkhBKMTIrh+QiJjEiN4e2Uary0/hN2pMDohnEtHxHLzlP7c9L4alV/xtzPb/D2TSCTdHyl0j5zjWeh6uP3jLSw9UATAPy8aQqjZyIyJfdplfiU5MYigmp3mOwEYZvmAM3S7edv0CudY59Bf5DNGl0qyqw+vmd4AYIfrFJ53XM9jhs+Yab+D14yvM0CXr+1vnOUtLtJv5lnjh60e2yPO28NeVxIjdBnt2qYxNkWPSThbHyjpEJ3xOzoeGGT5mFlXj+MaKXRVhBAXAq8CeuA9RVFmtzT+RLiYnojkVdRj1OuIDe2cegWXS+FAQRUDYkM6v6/xwV/hqxvglgXQb5rfIQWVFlyKQnxEQy12tcVOjdXRpD5bIpEc35xsQre1664QIgD4BBgHlALXKYqS0dI+T5Rr87kvrSK1qIalD0/Xym8KKi28vSpNK9+RSNqDDpffFNhYyrFiapL22RSFybr9fGmaxamWt/nQNIe7bA+6TbhUB+oar1psgYvpul2sdo3mDPfPhuMrgHDPSRCAnSRRwIOG77nb/iC36X/lcePnDLF8iIUAdLgwY8OKESd6BohcSpUw+opCfgp4gsGWj9ypwQpX6tbwnPE9rrT9h31KEgKFeEopIhKBwgTdQRJEMbON7wHwV9s9vOqOeC5ynsZd9ocw4OB03R5WuUbzlelZ+opCeolyFjvHM995Og70VCrBbFUGE0o9SaJAO9bOgDsIFfXcZXuQ8/TbeNdxCdlKLPvMtwHwg3MqcaKMVx1X8rjhM4bpMhlleZfL9Ot51vgh65zDudP+EAtMj3GpbRaPGL7l/wyL+dU5gVQlnom6gwwUOZxlfYlKghkqsvgt4J88b7+et5yX+fzG/NWoJ1m+wICDUOrY4TZlm2CZy2bzvTxpv4XvnGfwgGE+sx03oKBDh4vBIpuhIpOXTG/77OtR+1/41jmdQSKHIiWCfiIfk3DwrvFFrrM9QTD13GFYwMP2exAozDR8ySrXaBa5JrhTu9Uq1oWmfzJclwnAfbb7yVR68ou7VGCK5TViRCUKgmwllo9NzzNalw7AYud41rpGEIyFaFHFGbrdPGi/l2QlERc6Zl85kusnHLl3z3EvdIUQeiAFOA/IAbYANyiKsr+5bU6Ui6nkGPLr32HHZ/BoBhikkYBEcrJzMgndtlx3hRD3AKMURblLCHE9cIWiKNe1tN8T5dqcXlzDpxszefySYU06L+zNreTm9zdR7i5P+uGeKVRZHNzyweZjMVWJRCLpljzzxxHc3EKmaFvp6LW5O5lRTQBSFUVJBxBCfAVcDjQrdDuFvJ3ww51deghJN6YiC/pOkSJXIpGcjLTluns58JT79XfAG0IIoXSXp+RdSP/YEJ78w3C/60b0DmfHE+ezL6+S7LI6Tu2jmt9kzL4Ep0vh802ZTBkQzScbMrlr+gDiIwLZmV3Bw9/sRADldXZ+vm8qV7+1gQKv8iaJRCI5kYgJ9m8AdrToTkK3N+Bd/ZwDTGw8SAhxB3AHQJ8+ndDGxhgEsYOPfD+S45PYITDhL8d6FhKJRHIsaMt1VxujKIpDCFEJRAM+fXg6/dp8nDA8Ppzh8eE+y/Q6obUWfPryEdryMYkRLH/kTJ+xG/91TrP7rqy3Y9LrCDSpJTsOp4vDJbXERwTicCmEBxqpsTpwuhRCAwxNos4AhwqrCTUb6RVups7mIKusjvBAY5OymxqrA6dTIbu8jqhgE8kF1WzPKufMwT3ILK1lQr8oooJNuBRYnVLM3txKbprUl1CzgbwKCzoBLgVKa6yU1to4f3hPssvqSCms4ewhPdieWc7oxAiCTHr251cRZjZSUGVhzaESgkx69uRW8tyVI6mss1NWayPAqCO1qIbyOjuVdTa2ZZYzsGcofxgVz/q0EmptTqwOJ++sSuf8YT0ZnRjB8PgwBvYMZWdWBXvzKkmKDmLawFgcTgUh1M+zvM7G9swK4iLM7M+rYuopMezJqSC5sJqiaiv/uGAI5XU2Nh8uY2tmGY9eOITM0jq2ZJTx0848TukRwv1nn0JqUQ2KAj3DAvhxZx5nD+nBhH5RvLMqDavDxZQBMSzYnceUAdFcd1oiK5OLWbK/kDvO6M+nGzNJL64lq6yO4fFh7Mur8vld3DSpD59tzAJgbJ8IgkwG1qaWcMcZ/Zm3Ot3vd+WUHiGkFtVo77+5czLXvrPBZ4zHqPPKsb35cUcuSdHBpHu1fokKVo1GPQzqGUJKYQ2NEQLOHBTLiuSmRk8hAQZunZLE5sNlbM4oa7J+QGwwacXHtt2M5OgxbWAMF42MO6Zz6E6py9cAFyiKcrv7/c3ABEVR/DdG48RJj5JIJBJJ9+AkS11u9borhNjnHpPjfp/mHtOsrb68NkskEomkM+notblpRfyxIwfwtuVKAJr6+0skEolEIukM2nLd1cYIIQxAONA0VCORSCQSSTejOwndLcBAIUQ/IYQJuB5oe5duiUQikUgk7aEt192fgVvcr68Glp8M9bkSiUQiOf7pNjW67tqf+4DFqG0OPlAUZd8xnpZEIpFIJCckzV13hRBPA1sVRfkZeB/4VAiRihrJvf7YzVgikUgkkrbTbYQugKIovwK/Hut5SCQSiURyMuDvuqsoyhNery3ANUd7XhKJRCKRHCndKXVZIpFIJBKJRCKRSCSSI0YKXYlEIpFIJBKJRCKRnFBIoSuRSCQSiUQikUgkkhMKKXQlEolEIpFIJBKJRHJCIY7nLgFCiGIgs5N2FwOUdNK+jkdO9vMH+RmA/AxAfgZwcn8GfRVFiT3WkziekdfmTkOe+8mJPPeTl5P5/Fs79w5dm49roduZCCG2Kooy/ljP41hxsp8/yM8A5GcA8jMA+RlIug8n83dRnrs895ONk/nc4eQ+/646d5m6LJFIJBKJRCKRSCSSEwopdCUSiUQikUgkEolEckIhhW4D8471BI4xJ/v5g/wMQH4GID8DkJ+BpPtwMn8X5bmfnMhzP3k5mc+/S85d1uhKJBKJRCKRSCQSieSEQkZ0JRKJRCKRSCQSiURyQiGFrkQikUgkEolEIpFITihOeqErhLhQCJEshEgVQsw81vM5WgghMoQQe4QQO4UQW93LooQQS4QQh9w/I4/1PDsTIcQHQogiIcRer2V+z1movOb+XuwWQow9djPvPJr5DJ4SQuT+f3v3HitHWcZx/Puz0FIoQgtiuFQFxARUKIVgCYSImAI1oV4wYDTUS2JAjBJDTBHk8pdBUPwDI8R4ASFcRSQStASqKIEWhdNSIJSCqKQNTeUejCI8/jHPgc1xZ5vtmXNmZ+b3SSY7+87s7vu88+48++6+u5t9YUzSkp5t52QbPC7p+HpqXS1J8yWtlPSYpEckfT3LO9MXBrRBp/qCja625uZhcu+gc4+kZbn/E5KW1RXPIFXl3LJYJR2Wbbkhb6vpjbBcVbm27HkgaV9Jq7JNbpA0c/qiG6zKHNvQY19Zfm3a8Ze0g6TVktZk7Bdled/6SpqV1zfk9vf03NdQbVIqIjq7ADOAJ4H9gJnAGuCguus1TbE/Dew+oey7wPJcXw5cXHc9K475GGAhsG5rMQNLgDsAAYuAVXXXfwrb4ELg7D77HpTPiVnAvvlcmVF3DBW0wZ7AwlzfGVifsXamLwxog071BS+jubQ5Nw+Te8vOPcA84Km8nJvrc+uOrU+sk865g2IFVgNH5m3uAE6sO+atxD7U+XXQ8wC4ETg1168Azqg75p54KsmxDT72leTXJh7/PB5zcn17YFUe0771Bb4CXJHrpwI3bGublC1d/0T3CGBDRDwVEf8BrgeW1lynOi0Frsr1q4CP11iXykXEPcBzE4rLYl4KXB2F+4FdJe05PTWdOiVtUGYpcH1E/Dsi/gpsoHjONFpEbIqIB3P9ZeAxYG861BcGtEGZVvYFG1ldy83DnnuOB+6MiOci4nngTuCE6a701lSUc/vGmtveHhH3RfHK+GpG6DVLRbm27/MgP738CHBz3n6kXq9VmGObeuyryq+NO/55DF/Jq9vnEpTXt7dP3Awcl/EN1SaD6tT1ge7ewD96rj/D4M7YJgGskPQXSV/OsndGxCYonqjAHrXVbvqUxdy1vvHVnDL0U701Zb31bZDTZA6leNexk31hQhtAR/uCjZQ297dhcm9ZOzS5faqKde9cn1g+6oY5v5aV7wa8EBH/nVA+ciaZYxt/7CeZXxt5/CXNkDQGbKZ4c+JJyuv7Zoy5/UWK+Co793V9oNtvTn9X/m/pqIhYCJwInCnpmLorNGK61Dd+BOwPLAA2Ad/L8la3gaQ5wC+BsyLipUG79ilrRTv0aYNO9gUbOW3ub8Pk3rJ2aGP7DBtrE9tg2PNro2OvIMe2Lf5OHP+IeD0iFgD7UHwCe2C/3fJyymPv+kD3GWB+z/V9gI011WVaRcTGvNwM/IqiMz47PiUzLzfXV8NpUxZzZ/pGRDybJ6Y3gB/z1pTU1raBpO0pEtC1EXFLFneqL/Rrgy72BRtJre1vQ+besnZocvtUFeszuT6xfGRtw/m1rHwLxfTe7SaUj4yKcmxjj31F+bWxxx8gIl4Afk/xHd2y+r4ZY27fhWLKf2Xnvq4PdB8ADshfA5tJ8UXo22qu05STtJOkncfXgcXAOorYx3/Vbhnw63pqOK3KYr4NOE2FRcCL41Nu2mbC900/QdEXoGiDU/NX8fYFDqD4AYhGy+9//AR4LCK+37OpM32hrA261hdsZLUyN29D7i079/wOWCxpbk5/XJxlTVBJrLntZUmL8nx2GiP+mmUbzq99nwf5vdSVwMl5+5F6vVZhjm3ksa8wvzbu+Et6h6Rdc3028FGK7yiX1be3T5wM3J3xDdUmAysVI/ArXXUuFL/2tp5iDvm5dddnmmLej+KXytYAj4zHTTEv/i7gibycV3ddK477OorpIq9RvCv0pbKYKaZH/DD7xcPA4XXXfwrb4BcZ49o8YezZs/+52QaPM0K/ajjJNjiaYqrLWmAslyVd6gsD2qBTfcHL6C5tzM3D5t5B5x7gixQ/0LIB+ELdsZXEW0nOLYsVOJxisPAkcDmgumPeSuxDn1/LngfZl1Znm9wEzKo75p66VZZjG3rsK8uvTTv+wMHAQxnjOuD8QfUFdsjrG3L7ftvaJmWL8kZmZmZmZmZmrdD1qctmZmZmZmbWMh7ompmZmZmZWat4oGtmZmZmZmat4oGumZmZmZmZtYoHumZmZmZmZtYqHuiajShJr0sa61mWb2X/0yWdVsHjPi1p98nej5mZWZdJOkvSjnXXw6yr/PdCZiNK0isRMaeGx32a4n/stkz3Y5uZmbWF86lZvfyJrlnD5CeuF0tanct7s/xCSWfn+tckPSppraTrs2yepFuz7H5JB2f5bpJWSHpI0pUUf94+/lify8cYk3SlpBk1hGxmZjbSJO0k6XZJayStk3QBsBewUtLK3GexpPskPSjpJklzsrwsr38672uNpHvqi86smTzQNRtdsydMXT6lZ9tLEXEEcDnwgz63XQ4cGhEHA6dn2UXAQ1n2LeDqLL8A+FNEHArcBrwLQNKBwCnAURGxAHgd+Gy1IZqZmbXCCcDGiDgkIj5AkZs3AsdGxLH5laDzgI9GxELgz8A3em7fL6+fDxwfEYcAJ01XIGZtsV3dFTCzUv/KAWY/1/VcXtZn+1rgWkm3Ardm2dHApwAi4u78JHcX4Bjgk1l+u6Tnc//jgMOAByQBzAY2Ty4kMzOzVnoYuFTSxcBvIuKPmTvHLQIOAu7N8pnAfT3b++X1e4GfS7oRuGUK627WSh7omjVTlKyP+xjFAPYk4NuS3k/PlOQ+t+13HwKuiohzJlNRMzOztouI9ZIOA5YA35G0YsIuAu6MiM+U3cXE9Yg4XdKHKHL6mKQFEfHPqutu1laeumzWTKf0XPa+I4yktwHzI2Il8E1gV2AOcA859VjSh4EtEfHShPITgbl5V3cBJ0vaI7fNk/TuKYzJzMyskSTtBbwaEdcAlwILgZeBnXOX+4Gjer5/u6Ok9/Xcxf/ldUn7R8SqiDgf2ALMn/pIzNrDn+iaja7ZksZ6rv82Isb/YmiWpFUUb1ZNfHd4BnBNTksWcFlEvCDpQuBnktYCrwLLcv+LgOskPQj8Afg7QEQ8Kuk8YEUOnl8DzgT+VnWgZmZmDfdB4BJJb1DkyzOAI4E7JG3K7+l+niLfzsrbnAesz/V+ef0SSQdQ5PK7gDXTE4pZO/jvhcwaxn9XYGZm1h7O62ZTw1OXzczMzMzMrFX8ia6ZmZmZmZm1ij/RNTMzMzMzs1bxQNfMzMzMzMxaxQNdMzMzMzMzaxUPdM3MzMzMzKxVPNA1MzMzMzOzVvkfIBO1unJR9GwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4])\n"
     ]
    }
   ],
   "source": [
    "num_episodes = 50000\n",
    "for i_episode in range(num_episodes):\n",
    "    # Initialize the environment and state\n",
    "    s = env.reset()\n",
    "    screen = get_screen()[:,2:3,:,:] # get the blue channel\n",
    "    screens = [screen]*past\n",
    "    state = torch.from_numpy(s).float().to(device) # torch.cat(screens, dim=1)\n",
    "    print(state.shape)\n",
    "    for t in count():\n",
    "        # Select and perform an action\n",
    "        action = select_action(state)\n",
    "        s, reward, done, _ = env.step(action.item())\n",
    "        reward = torch.tensor([reward], device=device)\n",
    "\n",
    "        # Observe new state\n",
    "        screen = get_screen()[:,2:3,:,:]\n",
    "        del screens[0]\n",
    "        screens.append(screen)\n",
    "        if not done:\n",
    "            next_state = torch.from_numpy(s).float().to(device)# torch.cat(screens, dim=1)\n",
    "            #print(next_state.shape)\n",
    "        else:\n",
    "            next_state = None\n",
    "\n",
    "        # Store the transition in memory\n",
    "        memory.push(state, action, next_state, reward)\n",
    "\n",
    "        # Move to the next state\n",
    "        state = next_state\n",
    "\n",
    "        # Perform one step of the optimization (on the target network)\n",
    "        optimize_model()\n",
    "        if done:\n",
    "            episode_durations.append(t + 1)\n",
    "            plot_stuff()\n",
    "            break\n",
    "    # Update the target network, copying all weights and biases in DQN\n",
    "    \n",
    "    if i_episode % TARGET_UPDATE == 0:\n",
    "        target_net.load_state_dict(policy_net.state_dict())\n",
    "    #target_net.update_params(policy_net.state_dict(), tau)\n",
    "\n",
    "print('Complete')\n",
    "env.render()\n",
    "env.close()\n",
    "plot_durations()\n",
    "plt.ioff()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "env.render()\n",
    "print(len(memory))\n",
    "done = False\n",
    "screen = get_screen()[:,2:3,:,:] - 1# get the blue channel\n",
    "screens = [screen]*past\n",
    "rgb_screen = get_screen()\n",
    "rgb_screens = [rgb_screen]*past\n",
    "print(torch.max(screen))\n",
    "while not done:\n",
    "    screen = get_screen()[:,2:3,:,:] - 1\n",
    "    del screens[0]\n",
    "    screens.append(screen)\n",
    "    \n",
    "    rgb_screen = get_screen()\n",
    "    del rgb_screens[0]\n",
    "    rgb_screens.append(rgb_screen)\n",
    "    \n",
    "    state = torch.cat(screens, dim=1)\n",
    "    print(policy_net(state))\n",
    "    action = select_action(state, eps_max=0)\n",
    "    _, reward, done, _ = env.step(action.item())\n",
    "    \n",
    "env.close()   \n",
    "\n",
    "for screen in rgb_screens:\n",
    "    plt.figure()\n",
    "    plt.imshow(screen.cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "    plt.title('Screen')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, param in policy_net.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, torch.mean(param.data), torch.std(param.data), torch.max(param.data),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy_net(state).max(0)[1].view(1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}