ddpg_pendulum.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **INVERTED PENDULUM CONTROL USING DEEP DETERMINISTIC POLICY GRADIENT**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing necessary modules required for solving this problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-17 21:40:10.058336: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of State Space -> 3\n",
      "Size of Action Space -> 1\n",
      "Max Value of Action -> 2.0\n",
      "Min Value of Action -> -2.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arch/miniconda3/envs/python_ai/lib/python3.8/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"Pendulum-v0\")       # create environment\n",
    "\n",
    "num_states = env.observation_space.shape[0]\n",
    "print(\"Size of State Space -> {}\".format(num_states))\n",
    "\n",
    "num_actions = env.action_space.shape[0]\n",
    "print(\"Size of Action Space -> {}\".format(num_actions))\n",
    "\n",
    "upper_bound = env.action_space.high[0]      # to scale our actions later\n",
    "lower_bound = env.action_space.low[0]\n",
    "print(\"Max Value of Action -> {}\".format(upper_bound))\n",
    "print(\"Min Value of Action -> {}\".format(lower_bound))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using noisy perturbations for better exploration by the Actor Network, like the **Ornstein-Uhlenbeck process**. This will sample noise from correlated normal distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OUActionNoise:\n",
    "    def __init__(self, mean, std_deviation, theta=0.15, dt=1e-2, x_initial=None):\n",
    "        self.theta = theta\n",
    "        self.mean = mean\n",
    "        self.std_dev = std_deviation\n",
    "        self.dt = dt\n",
    "        self.x_initial = x_initial\n",
    "        self.reset()\n",
    "\n",
    "    def __call__(self):\n",
    "        # mathematical formula for ornstein-uhlenbeck process\n",
    "        x = (\n",
    "            self.x_prev\n",
    "            + self.theta * (self.mean - self.x_prev) * self.dt\n",
    "            + self.std_dev * np.sqrt(self.dt) *\n",
    "            np.random.normal(size=self.mean.shape)\n",
    "        )\n",
    "        self.x_prev = x     # store x into x_prev to make noise dependant on current one\n",
    "        return x\n",
    "\n",
    "    def reset(self):\n",
    "        if self.x_initial is not None:\n",
    "            self.x_prev = self.x_initial\n",
    "        else:\n",
    "            self.x_prev = np.zeros_like(self.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implementing Experience Replay\n",
    "class Buffer:\n",
    "    def __init__(self, buffer_capacity=100000, batch_size=64):\n",
    "        self.buffer_capacity = buffer_capacity      # maximum number of \"experiences\" to store\n",
    "        self.batch_size = batch_size        # number of tuples to train on\n",
    "\n",
    "        self.buffer_counter = 0     # number of times record() was called\n",
    "\n",
    "        # different numpy arrays for each tuple element\n",
    "        self.state_buffer = np.zeros((self.buffer_capacity, num_states))\n",
    "        self.action_buffer = np.zeros((self.buffer_capacity, num_actions))\n",
    "        self.reward_buffer = np.zeros((self.buffer_capacity, 1))\n",
    "        self.next_state_buffer = np.zeros((self.buffer_capacity, num_states))\n",
    "\n",
    "    def record(self, obs_tuple):\n",
    "        \"\"\"sets index=0 if buffer_capacity exceeded, replace old records\"\"\"\n",
    "        index = self.buffer_counter % self.buffer_capacity\n",
    "\n",
    "        self.state_buffer[index] = obs_tuple[0]\n",
    "        self.action_buffer[index] = obs_tuple[1]\n",
    "        self.reward_buffer[index] = obs_tuple[2]\n",
    "        self.next_state_buffer[index] = obs_tuple[3]\n",
    "\n",
    "        self.buffer_counter += 1\n",
    "\n",
    "    # build a static graph out of the logic and computations in our function to speed up code execution\n",
    "    @tf.function        \n",
    "    def update(self, state_batch, action_batch, reward_batch, next_state_batch):\n",
    "        # training & updating Actor & Critic Networks\n",
    "        with tf.GradientTape() as tape:\n",
    "            target_actions = target_actor(next_state_batch, training=True)\n",
    "            y = reward_batch + gamma * target_critic(\n",
    "                [next_state_batch, target_actions], training=True\n",
    "            )\n",
    "            critic_value = critic_model([state_batch, action_batch], training=True)\n",
    "            critic_loss = tf.math.reduce_mean(tf.math.square(y - critic_value))\n",
    "\n",
    "        critic_grad = tape.gradient(critic_loss, critic_model.trainable_variables)\n",
    "        critic_optimizer.apply_gradients(\n",
    "            zip(critic_grad, critic_model.trainable_variables)\n",
    "        )\n",
    "\n",
    "        with tf.GradientTape() as tape:\n",
    "            actions = actor_model(state_batch, training=True)\n",
    "            critic_value = critic_model([state_batch, actions], training=True)\n",
    "            # maximize value given by critic for our actions\n",
    "            actor_loss = -tf.math.reduce_mean(critic_value)\n",
    "\n",
    "        actor_grad = tape.gradient(actor_loss, actor_model.trainable_variables)\n",
    "        actor_optimizer.apply_gradients(\n",
    "            zip(actor_grad, actor_model.trainable_variables)\n",
    "        )\n",
    "\n",
    "    # compute loss & update parameters\n",
    "    def learn(self):\n",
    "        record_range = min(self.buffer_counter, self.buffer_capacity)       # sampling range\n",
    "        batch_indices = np.random.choice(record_range, self.batch_size)     # random sample indices\n",
    "\n",
    "        # conversion to tensors\n",
    "        state_batch = tf.convert_to_tensor(self.state_buffer[batch_indices])\n",
    "        action_batch = tf.convert_to_tensor(self.action_buffer[batch_indices])\n",
    "        reward_batch = tf.convert_to_tensor(self.reward_buffer[batch_indices])\n",
    "        reward_batch = tf.cast(reward_batch, dtype=tf.float32)\n",
    "        next_state_batch = tf.convert_to_tensor(self.next_state_buffer[batch_indices])\n",
    "\n",
    "        self.update(state_batch, action_batch, reward_batch, next_state_batch)\n",
    "\n",
    "\n",
    "# update parameters slowly depending upon tau, given tau << 1 \n",
    "@tf.function\n",
    "def update_target(target_weights, weights, tau):\n",
    "    for (a, b) in zip(target_weights, weights):\n",
    "        a.assign(b * tau + a * (1 - tau))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor():\n",
    "    # initialize weights \n",
    "    last_init = tf.random_uniform_initializer(minval=-0.003, maxval=0.003)\n",
    "\n",
    "    inputs = layers.Input(shape=(num_states,))\n",
    "    out = layers.Dense(256, activation=\"relu\")(inputs)\n",
    "    out = layers.Dense(256, activation=\"relu\")(out)\n",
    "    outputs = layers.Dense(1, activation=\"tanh\", kernel_initializer=last_init)(out)\n",
    "\n",
    "    outputs = outputs * upper_bound     # upper_bound=2.0 for pendulum\n",
    "    model = tf.keras.Model(inputs, outputs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def get_critic():\n",
    "    state_input = layers.Input(shape=(num_states))      # input State\n",
    "    state_out = layers.Dense(16, activation=\"relu\")(state_input)\n",
    "    state_out = layers.Dense(32, activation=\"relu\")(state_out)\n",
    "\n",
    "    action_input = layers.Input(shape=(num_actions))        # input Action\n",
    "    action_out = layers.Dense(32, activation=\"relu\")(action_input)\n",
    "\n",
    "    # pass through separate layers before concatenating\n",
    "    concat = layers.Concatenate()([state_out, action_out])\n",
    "\n",
    "    out = layers.Dense(256, activation=\"relu\")(concat)\n",
    "    out = layers.Dense(256, activation=\"relu\")(out)\n",
    "    outputs = layers.Dense(1)(out)\n",
    "\n",
    "    # single value for given state-action\n",
    "    model = tf.keras.Model([state_input, action_input], outputs)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns an action sampled from Actor network plus some noise for exploration\n",
    "def policy(state, noise_object):\n",
    "    sampled_actions = tf.squeeze(actor_model(state))\n",
    "    noise = noise_object()\n",
    "    sampled_actions = sampled_actions.numpy() + noise       # adding noise to action\n",
    "\n",
    "    legal_action = np.clip(sampled_actions, lower_bound, upper_bound)       # check if action within bounds\n",
    "\n",
    "    return [np.squeeze(legal_action)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TRAINING HYPERPARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-17 21:40:12.809730: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcuda.so.1\n",
      "2022-07-17 21:40:12.824757: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.825198: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1733] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1660 Ti with Max-Q Design computeCapability: 7.5\n",
      "coreClock: 1.335GHz coreCount: 24 deviceMemorySize: 5.80GiB deviceMemoryBandwidth: 268.26GiB/s\n",
      "2022-07-17 21:40:12.825230: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
      "2022-07-17 21:40:12.843733: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcublas.so.11\n",
      "2022-07-17 21:40:12.843821: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcublasLt.so.11\n",
      "2022-07-17 21:40:12.888463: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcufft.so.10\n",
      "2022-07-17 21:40:12.891980: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcurand.so.10\n",
      "2022-07-17 21:40:12.895112: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcusolver.so.11\n",
      "2022-07-17 21:40:12.901194: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcusparse.so.11\n",
      "2022-07-17 21:40:12.902409: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudnn.so.8\n",
      "2022-07-17 21:40:12.902682: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.903092: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.903691: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1871] Adding visible gpu devices: 0\n",
      "2022-07-17 21:40:12.904330: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-07-17 21:40:12.905393: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.905663: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1733] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: NVIDIA GeForce GTX 1660 Ti with Max-Q Design computeCapability: 7.5\n",
      "coreClock: 1.335GHz coreCount: 24 deviceMemorySize: 5.80GiB deviceMemoryBandwidth: 268.26GiB/s\n",
      "2022-07-17 21:40:12.905784: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.906064: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:12.906263: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1871] Adding visible gpu devices: 0\n",
      "2022-07-17 21:40:12.906623: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
      "2022-07-17 21:40:13.765922: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1258] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2022-07-17 21:40:13.765953: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1264]      0 \n",
      "2022-07-17 21:40:13.765961: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1277] 0:   N \n",
      "2022-07-17 21:40:13.766138: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:13.766332: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:13.766490: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-17 21:40:13.766626: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1418] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 4665 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce GTX 1660 Ti with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 7.5)\n"
     ]
    }
   ],
   "source": [
    "std_dev = 0.2\n",
    "ou_noise = OUActionNoise(mean=np.zeros(\n",
    "    1), std_deviation=float(std_dev) * np.ones(1))\n",
    "\n",
    "actor_model = get_actor()\n",
    "critic_model = get_critic()\n",
    "\n",
    "target_actor = get_actor()\n",
    "target_critic = get_critic()\n",
    "\n",
    "# equal weights initially\n",
    "target_actor.set_weights(actor_model.get_weights())\n",
    "target_critic.set_weights(critic_model.get_weights())\n",
    "\n",
    "# learning rates\n",
    "critic_lr = 0.002\n",
    "actor_lr = 0.001\n",
    "\n",
    "critic_optimizer = tf.keras.optimizers.Adam(critic_lr)\n",
    "actor_optimizer = tf.keras.optimizers.Adam(actor_lr)\n",
    "\n",
    "total_episodes = 100\n",
    "gamma = 0.99        # discount factor for future rewards\n",
    "tau = 0.005     # update target networks\n",
    "\n",
    "buffer = Buffer(50000, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-17 21:40:14.004693: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcublas.so.11\n",
      "2022-07-17 21:40:15.099391: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcublasLt.so.11\n",
      "2022-07-17 21:40:15.762999: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2022-07-17 21:40:15.785631: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2601325000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode * 0 * Avg Reward is ==> -1456.093668843105\n",
      "Episode * 1 * Avg Reward is ==> -1396.5707653006539\n",
      "Episode * 2 * Avg Reward is ==> -1307.2849762693315\n",
      "Episode * 3 * Avg Reward is ==> -1392.428489013574\n",
      "Episode * 4 * Avg Reward is ==> -1415.1402784801671\n",
      "Episode * 5 * Avg Reward is ==> -1442.2287262827297\n",
      "Episode * 6 * Avg Reward is ==> -1436.8933958718158\n",
      "Episode * 7 * Avg Reward is ==> -1441.4043775555695\n",
      "Episode * 8 * Avg Reward is ==> -1416.6387637490452\n",
      "Episode * 9 * Avg Reward is ==> -1367.154150652183\n",
      "Episode * 10 * Avg Reward is ==> -1329.596883812418\n",
      "Episode * 11 * Avg Reward is ==> -1294.6398739211143\n",
      "Episode * 12 * Avg Reward is ==> -1256.9636579673747\n",
      "Episode * 13 * Avg Reward is ==> -1223.6737423188401\n",
      "Episode * 14 * Avg Reward is ==> -1189.6693980523619\n",
      "Episode * 15 * Avg Reward is ==> -1148.0126354780245\n",
      "Episode * 16 * Avg Reward is ==> -1111.2773969020966\n",
      "Episode * 17 * Avg Reward is ==> -1078.5461143862826\n",
      "Episode * 18 * Avg Reward is ==> -1048.7370742215737\n",
      "Episode * 19 * Avg Reward is ==> -1015.1065730177079\n",
      "Episode * 20 * Avg Reward is ==> -989.6101051084191\n",
      "Episode * 21 * Avg Reward is ==> -956.5690388877639\n",
      "Episode * 22 * Avg Reward is ==> -920.523230905389\n",
      "Episode * 23 * Avg Reward is ==> -896.8795354447266\n",
      "Episode * 24 * Avg Reward is ==> -875.4464364321254\n",
      "Episode * 25 * Avg Reward is ==> -846.6899581319917\n",
      "Episode * 26 * Avg Reward is ==> -820.0961536453118\n",
      "Episode * 27 * Avg Reward is ==> -804.090100702103\n",
      "Episode * 28 * Avg Reward is ==> -785.1392340715978\n",
      "Episode * 29 * Avg Reward is ==> -771.4030008080483\n",
      "Episode * 30 * Avg Reward is ==> -750.5859943746373\n",
      "Episode * 31 * Avg Reward is ==> -734.913561408621\n",
      "Episode * 32 * Avg Reward is ==> -716.5453694395211\n",
      "Episode * 33 * Avg Reward is ==> -699.2374228042872\n",
      "Episode * 34 * Avg Reward is ==> -682.9818083712668\n",
      "Episode * 35 * Avg Reward is ==> -667.4158081639657\n",
      "Episode * 36 * Avg Reward is ==> -656.4516836209432\n",
      "Episode * 37 * Avg Reward is ==> -639.2298978788405\n",
      "Episode * 38 * Avg Reward is ==> -638.8128117407527\n",
      "Episode * 39 * Avg Reward is ==> -641.5477758327238\n",
      "Episode * 40 * Avg Reward is ==> -611.307885587145\n",
      "Episode * 41 * Avg Reward is ==> -584.0739582300081\n",
      "Episode * 42 * Avg Reward is ==> -562.1962751704472\n",
      "Episode * 43 * Avg Reward is ==> -529.247266382858\n",
      "Episode * 44 * Avg Reward is ==> -494.8336314025761\n",
      "Episode * 45 * Avg Reward is ==> -458.289297880723\n",
      "Episode * 46 * Avg Reward is ==> -426.25206961053027\n",
      "Episode * 47 * Avg Reward is ==> -392.49038136397803\n",
      "Episode * 48 * Avg Reward is ==> -368.03520387763353\n",
      "Episode * 49 * Avg Reward is ==> -348.12100693494426\n",
      "Episode * 50 * Avg Reward is ==> -327.3648652538247\n",
      "Episode * 51 * Avg Reward is ==> -307.76968837302184\n",
      "Episode * 52 * Avg Reward is ==> -290.66252364278694\n",
      "Episode * 53 * Avg Reward is ==> -273.786224181922\n",
      "Episode * 54 * Avg Reward is ==> -264.74486573525417\n",
      "Episode * 55 * Avg Reward is ==> -251.72836277406685\n",
      "Episode * 56 * Avg Reward is ==> -247.71817221485418\n",
      "Episode * 57 * Avg Reward is ==> -237.7736887304853\n",
      "Episode * 58 * Avg Reward is ==> -224.9900405509462\n",
      "Episode * 59 * Avg Reward is ==> -218.62006899147053\n",
      "Episode * 60 * Avg Reward is ==> -213.40224908936366\n",
      "Episode * 61 * Avg Reward is ==> -206.88648676956618\n",
      "Episode * 62 * Avg Reward is ==> -206.81371826544105\n",
      "Episode * 63 * Avg Reward is ==> -200.88604134549186\n",
      "Episode * 64 * Avg Reward is ==> -194.95923570994788\n",
      "Episode * 65 * Avg Reward is ==> -194.8205187969451\n",
      "Episode * 66 * Avg Reward is ==> -203.1132804268479\n",
      "Episode * 67 * Avg Reward is ==> -201.51268603669897\n",
      "Episode * 68 * Avg Reward is ==> -198.3078960531384\n",
      "Episode * 69 * Avg Reward is ==> -192.10688621505602\n",
      "Episode * 70 * Avg Reward is ==> -192.15194755148102\n",
      "Episode * 71 * Avg Reward is ==> -189.0574098158799\n",
      "Episode * 72 * Avg Reward is ==> -188.935324754786\n",
      "Episode * 73 * Avg Reward is ==> -188.59989310406965\n",
      "Episode * 74 * Avg Reward is ==> -188.3308868925596\n",
      "Episode * 75 * Avg Reward is ==> -188.2619427690053\n",
      "Episode * 76 * Avg Reward is ==> -187.90882069518054\n",
      "Episode * 77 * Avg Reward is ==> -190.79630172986865\n",
      "Episode * 78 * Avg Reward is ==> -175.26661146271726\n",
      "Episode * 79 * Avg Reward is ==> -159.72647736605717\n",
      "Episode * 80 * Avg Reward is ==> -156.6945944236572\n",
      "Episode * 81 * Avg Reward is ==> -156.61333048902873\n",
      "Episode * 82 * Avg Reward is ==> -153.41715167143838\n",
      "Episode * 83 * Avg Reward is ==> -145.2377176539615\n",
      "Episode * 84 * Avg Reward is ==> -149.42369436414054\n",
      "Episode * 85 * Avg Reward is ==> -149.6592382807716\n",
      "Episode * 86 * Avg Reward is ==> -146.64203770966523\n",
      "Episode * 87 * Avg Reward is ==> -146.56323498891098\n",
      "Episode * 88 * Avg Reward is ==> -143.5434994881126\n",
      "Episode * 89 * Avg Reward is ==> -146.21957988525816\n",
      "Episode * 90 * Avg Reward is ==> -149.17567116812262\n",
      "Episode * 91 * Avg Reward is ==> -151.75596526742675\n",
      "Episode * 92 * Avg Reward is ==> -148.8476147501637\n",
      "Episode * 93 * Avg Reward is ==> -145.98492873652833\n",
      "Episode * 94 * Avg Reward is ==> -140.30893141811612\n",
      "Episode * 95 * Avg Reward is ==> -146.3202536857944\n",
      "Episode * 96 * Avg Reward is ==> -140.17631880706796\n",
      "Episode * 97 * Avg Reward is ==> -140.20388818508255\n",
      "Episode * 98 * Avg Reward is ==> -143.24470407442956\n",
      "Episode * 99 * Avg Reward is ==> -143.21860905925936\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEGCAYAAACgt3iRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuT0lEQVR4nO3deXhU5fn/8fedhEAIS0ggyL6DoAhIEK2touCuxX1vqbVqXWo3d22rtfpTv63WrSpVrNYF0SqiohSVihsCAZRVCfsmBAhbAtnm/v0xB42YhIHJZCbJ53Vdc82c55yZcw8n5M6znOcxd0dERCQaSfEOQERE6j4lExERiZqSiYiIRE3JREREoqZkIiIiUUuJdwDx0rp1a+/atWu8wxARqVNyc3M3unubPcsbbDLp2rUrM2fOjHcYIiJ1ipmtqKxczVwiIhI1JRMREYmakomIiERNyURERKKmZCIiIlFTMhERkagpmYiISNQa7H0mIiJ1zYZtu/hw8UbK3RnWuw3ZLZoAsG1XKR98mc/i9du/ObZJajJds9Lp1jqd9hlpJCfZN/vSGiV/Z7smKJmIiOzFqs1FjJ2xkq/W76BfuxYc0rElvbKb0yjFSE4ySspCrNxUxNKNhazZspPykOPulJY7KzcXsWxjIWsKdpKZnkrnzKZ0zEyja1Y6XVun0y0rnc6ZTWmRloJZ+Bd8cVk5qzbvZMWmQpZvKmL5xkJmLN/Moq+3fyeu/h1a0qxxCjOWb6YsFF6bKvgIqluq6t3fHU3P7GY1+m+kZCIiDZ67U1wW+mZ7c2EJeRt2sCR/B+8v2sBHeRsxoHNmU95buJ5QNb+oU5KMlGTDMFKSjA6t0ujbrjkj+mazubCUVQVFfJK3iVdnrfnO+5o3TqFDqzS27ypj7dad30kGzZukcFD7FtxwYh+O7t0Gw5jy5QbeX7SBgqISfvGj7ozom82gzq2+qXEUFpexfFMhyzYWsm7Lru+cq3Wz1Kj/zfZkDXWlxZycHNd0KiINk7szdsYqPlycz9L8QlZsKmJnaXmlx7Zv2YRzh3Ti3JxOtM9Io7C4jAXrtrEsv5CykBNyJznJ6JLZlG5t0mnbvAlJETQh7SwpZ8XmQpZvLGR1wc7gUUR645Sg1tI0/JyVTkbTRt/UWuLNzHLdPed75UomItKQbN1ZynUvf87kBevplJlGr+zmdGudTmZ6KmZgGC3SUujRphk92jSjdbPUhPlFngiqSiZq5hKRBmPu6q1c9UIu67bs4g+n9uPnR3ZVoqghSiYiUu+5O899tpI731hAVrNUXrriCAZ3aRXvsOqVhEsmZvZ/wGlACbAEuMTdtwT7bgYuBcqBa919UlB+IvAgkAw86e73xCF0Eakh23eV8vXWXZS74w4hd8pDTlko/Fzx4YSThQPl5f5NP0Z288Z0yUonLTWZW16dy4TP1zKsTxvuP3cgmek13wHd0CVcMgEmAze7e5mZ3QvcDNxoZv2A84GDgPbAu2bWO3jPo8BxwGpghplNcPcFcYhdpEEpDznz1mxlc1EJBL/0+7ZrQfuMtL2+b9bKAt5duJ6vguGuZkZhcRlLNxaSv724xmJMTjLcnetP6MOVR/eIqHNc9l3CJRN3/2+FzWnA2cHrkcBYdy8GlplZHnBYsC/P3ZcCmNnY4FglE5EYKCgsYerifKYs2sAHX+VTUFT6nf1JBsf0yebCoZ05uENLNheWUFBYwuotO1mSv4MlGwrJXbGZgqJSGiUbvbKbfzOctXFKEkf3bkP3Nul0yEijUXIS4V1Go+TwPR27HylJ4X27+zzMgmG5SUmYwfptu1ixqYg1W3Yyom9bDuuWWcv/Ug1LwiWTPfwceCl43YFwctltdVAGsGqP8qGVfZiZXQ5cDtC5c+caDVSkPnF3Fn29nQ8X57OzJIQTvg/js6WbmLNqCyGHrPRUjjkwm2F9sunYKo0kM0LuTFm0gbEzVvHeM98fLZmanETX1k0Z1iebEX3bclTv1jRv0igm36FvuxYx+VypXFySiZm9CxxQya5b3f314JhbgTLg+Zo6r7uPBkZDeGhwTX2uSH1QVFLGjOUFfLQ4n0nz17Nyc9F39pvBIR0zuHZ4L4b1yeaQDi0rbTI6tHMrrh3ei/cXbWDD9mKy0lNp1TSVA1o2oVOrNFKSNSVgfRSXZOLuI6rbb2Y/A04Fhvu3N8KsATpVOKxjUEY15SJSBXdnSf4OJs1fz5RFG5izagtlIadRsvGDHq25clgPhvfNJiu9MUY4mUQ6jLZRchInHFTZ34tSXyVcM1cwMusG4Gh3r/in0QTgBTO7n3AHfC9gOmBALzPrRjiJnA9cWLtRi9Qd7s6rs9bw6JQ8lm4sBMJzPF12VHeO6J5FTtdWNE1NuF8NkuAS8SfmEaAxMDn4K2iau//S3eeb2TjCHetlwNXuXg5gZtcAkwgPDR7j7vPjE7pIYluSv4PbXpvHp0s3MaBjS+4ceRAj+rWlXcvqR1+J7I2mUxFpANZv28XoqUv596craNwoiZtOOpALhnTWMFnZZ5pORaSBKSwuY+G6bbw+Zy0vzVxFecg5Y1AHbjixD9nNm8Q7PKlnlExE6pGSshCjpy7htdlrWLqxEHdolGycdWhHrhzWgy5Z6fEOUeopJROReiJ3xWZu+s9cFm/YwZE9szhtQHsObt+SgZ0zaN2scbzDk3pOyUSkjlu4bhv//HApr81eQ7sWTRjzsxyOPbBtvMOSBkbJRKSO+mzpJh56fzEf520irVEyv/hhN349ojfNGuu/tdQ+/dSJ1DHuzpiPl3PXWwvIbt7km5FZLZvGZloSkUgomYjUIcVl5fxh/DzGzVzNCQe15f5zB5KumogkAP0UitQRm3YU88vncpmxvIBrj+3Jb0b01n0ikjCUTETqgC+/3s6lz8wgf3sxD18wiNMGtI93SCLfoWQikuDeX7SeX70wm/TGKYy74ggGdMqId0gi36NkIpLAxs9ew+9f/py+7Zrz5E+HcEBL3bkuiUnJRCRBvTh9Jbe8NpfDu2Xx5KgcdbRLQtNPp0gCevrjZdzxxgKG9WnD4xcPpkmj5HiHJFItJRORBDPmo2X8+c0FnHBQWx66YBCNU5RIJPEpmYgkkGc/Xc6f31zAiQcdwMMXDqKRlriVOkI/qSIJ4rlpK/jj6/M5rl+4RqJEInWJflpFEsCzny7ntvHzGNE3m0cvPJTUFP3XlLpFzVwicfbkh0v5y1sLOa5fWx65cJASidRJSiYicfSP/+Vx3ztfcnL/A3jwfDVtSd2lZCISJ2Onr+S+d75k5MD2/O2cAaQokUgdpp9ekTj4dMkmbhs/j6N6t1EikXohYX+Czez3ZuZm1jrYNjN7yMzyzOwLMzu0wrGjzGxx8BgVv6hF9m75xkKufD6Xrq3TeeTCQUokUi8kZDOXmXUCjgdWVig+CegVPIYCjwFDzSwT+BOQAziQa2YT3L2gdqMW2bsdxWVc+swMDHhqVA4tmmhBK6kfEvVPogeAGwgnh91GAs962DQgw8zaAScAk919c5BAJgMn1nrEIhF4YPJXLN1YyD8uGkyXrPR4hyNSYxIumZjZSGCNu3++x64OwKoK26uDsqrKK/vsy81sppnNzM/Pr8GoRfZu4bpt/OuT5VxwWGeO6JEV73BEalRcmrnM7F3ggEp23QrcQriJq8a5+2hgNEBOTo7v5XCRGhMKOX8YP4+WaY244YQ+8Q5HpMbFJZm4+4jKys2sP9AN+NzMADoCs8zsMGAN0KnC4R2DsjXAsD3K/1fjQYtE4T+zVjNzRQH3nXUIGU1T4x2OSI1LqGYud5/r7tnu3tXduxJusjrU3b8GJgA/DUZ1HQ5sdfd1wCTgeDNrZWatCNdqJsXrO4jsaWtRKfe8vYhDO2dw9uCO8Q5HJCYScjRXFSYCJwN5QBFwCYC7bzazO4EZwXF/dvfN8QlR5PvunriQLTtLefb0g0lKsniHIxITCZ1MgtrJ7tcOXF3FcWOAMbUUlkjEPlmykZdmruKKo7tzUPuW8Q5HJGYSqplLpD7ZVVrOLa/OpUtWU34zvHe8wxGJqYSumYjUZQ++t5jlm4p4/hdDSUvVaolSv6lmIhIDC9dtY/TUpZwzuCNH9mwd73BEYq7KmomZPcx370D/Dne/NiYRidRx7s6dby6gRZMUbj2lb7zDEakV1dVMZgK5QBPgUGBx8BgIaKC8SBXeX7SBT5Zs4tfDe+meEmkwqqyZuPszAGZ2JfBDdy8Lth8HPqyd8ETqltLyEHdPXEj31ulcdHiXeIcjUmsi6TNpBbSosN0sKBORPYydvpIl+YXcfHJfrZooDUoko7nuAWab2RTAgKOA22MZlEhdtG1XKQ+8u5jDu2cyom92vMMRqVXVJhMzSwK+JLx+yNCg+MZgehMRqeDh9xZTUFTCbaf0I5hbTqTBqDaZuHvIzB5190HA67UUk0ids3j9dp7+eDnnD+nEwR10p7s0PJE06r5nZmeZ/tQSqZS7c/sb80lvnML1JxwY73BE4iKSZHIF8DJQbGbbzGy7mW2LcVwidcbb877m47xNXHd8bzLTNRRYGqa9dsC7e/PaCESkLioqKeMvby6gX7sWXDhUQ4Gl4Ypobq5gnZBehG9gBMDdp8YqKJG64uH381i7dRcPXTCIZE0vLw3YXpOJmf0C+DXhFQznAIcDnwLHxjQykQT35dfb+efUpZw9uCM5XTPjHY5IXEXSZ/JrYAiwwt2PAQYBW2IZlEiiC4WcW1+bS/MmKdxysubfEokkmexy910AZtbY3RcBfWIblkhiGzdzFTNXFHDzyX3V6S5CZH0mq80sAxgPTDazAmBFLIMSSWQbdxTz/95exGHdMjlHa7qLAJGN5jojeHl7MKVKS+CdmEYlkqBKykL8btznFJWUcfcZB+tOd5FAJB3wdwJTgU/c/YPYhySSmMpDzm/HzWHqV/ncc2Z/emZr1LzIbpH0mSwFLgBmmtl0M/ubmY2McVwiCcU93OH+1hfruOXkAzn/sM7xDkkkoew1mbj70+7+c+AY4DngnOA5ZszsV2a2yMzmm9l9FcpvNrM8M/vSzE6oUH5iUJZnZjfFMjZpmO6f/BVjZ6zimmN6cvlRPeIdjkjCiaSZ60mgH7Ce8KJYZwOzYhWQmR0DjAQGuHuxmWUH5f2A84GDgPbAu2bWO3jbo8BxwGpghplNcPcFsYpRGpYpizbw8Pt5nJvTkd8f33vvbxBpgCIZzZUFJBO+t2QzsHH3qosxciVwj7sXA7j7hqB8JDA2KF9mZnnAYcG+PHdfCmBmY4NjlUwkamu37OS34+bQt10L/jxSHe4iVYmkmesMdx8K3AdkAFPMbHUMY+oN/MjMPjOzD8xsSFDeAVhV4bjVQVlV5d9jZpeb2Uwzm5mfnx+D0KU+KS0P8asXZ1NaFuLRCwfRpFFyvEMSSViRNHOdCvyI8AqLGcD7RLkGvJm9CxxQya5bg5gyCU/bMgQYZ2bdoznfbu4+GhgNkJOT4zXxmVJ//e2/X5G7ooAHzx9I9zbN4h2OSEKLpJnrRMLJ40F3X1sTJ3X3EVXtM7MrgVfd3YHpZhYCWgNrgE4VDu0YlFFNuch+mbF8M09MXcL5QzoxcmClFV0RqSCSZq5rgGmEO+ExszQzi+UA+/GER44RdLCnAhuBCcD5ZtbYzLoRnsV4OjAD6GVm3cwslXAn/YQYxif1XFFJGde9/DkdMtK47dR+8Q5HpE6IpJnrMuBywk1PPQj/5f84MDxGMY0BxpjZPKAEGBXUUuab2TjCHetlwNXuXh7EeA0wifBAgTHuPj9GsUkDcO/bi1ixqYgXLzucZo0jWqVBpMGL5H/K1YRHTX0G4O6Ldw/XjQV3LwEurmLfXcBdlZRPBCbGKiapX0Ih54XpK1maX0hZKERpuZPRtBGdM5sScueZT1fwsx905YgeWfEOVaTOiCSZFLt7ye4hkWaWAqjzWuqkopIyfvvSHCbNX096ajKNUpJISTK27iyltDz8Y92tdTo3nqi13EX2RSTJ5AMzuwVIM7PjgKuAN2IblkjN+3rrLi59ZgYL123jj6f245Iju35z30h5yFm3dScrNxfRo00z0lI1DFhkX0SSTG4CLgXmAlcAE939nzGNSqQGlZaHeGnGKv7+7lfsKg3x1KghHHPgd1tqk5OMjq2a0rFV0zhFKVK3RTIFfQj4Z/DAzI43s8nuflysgxOJhrvz9ryv+b9JX7JsYyE5XVpx1xn96XOAZvsVqWlVJhMzO5bwqK32hIfr3gs8DRiVdIKLJJKikjJue20er85eQ++2zXjypzkM75ut6VBEYqS6msnfCA8J/hQ4KXi+yd0fqY3ARPZX3oYdXPV8Los37OA3I3rxq2N7kZykJCISS9UlE3f3/wWvx5vZGiUSSXQT567jupc/J61RMs/+/DB+1KtNvEMSaRCqSyYZZnZmxWMrbrv7q7ELS2TflIec/5v0JY9/sIRDO2fwj4sGc0DLJvEOS6TBqC6ZfACcVmF7aoVtB5RMJCFsKSrhVy/O5sPFG7loaGf+dNpBpKZEsoioiNSUKpOJu19Sm4GI7I9lGwu59F8zWF2wk3vP6s95Q7Scrkg8aOIhqbM+XbKJXz6XS3KS8fxlQxnSNTPeIYk0WEomUie9Oms1N7zyBV2ymvL0zw6jc5ZuNhSJJyUTqVPcnSemLuWetxdxRPcsHv/JYFqmNYp3WCIN3l57Kc3sajPLqLDdysyuimlUIpUIhZw/v7mAe95exGkD2vOvnw9RIhFJEJEMebnM3bfs3nD3AuCymEUkUomCwhJ+/swMnv54OZf+sBsPnjeQximajFEkUUTSzJVsZhYsUIWZJRNe/VCkVsxZtYWrn59F/vZi7jz9YH5yeJd4hyQie4gkmbwDvGRmTwTbVwRlIjE3fvYarn/lc7KbN+HlXx7BgE4Z8Q5JRCoRSTK5kXACuTLYngw8GbOIRAKvz1nD78bN4bBumTx+8WAymqpCLJKoIp2C/rHgIVIr3vxiLb99aQ5DumYy5mdDaJqqgYciiay6KejHufu5ZjaXSpbpdfdDYhqZNEjuzriZq7jltXkM7tJKiUSkjqjuf+mvg+dTayMQkfXbdnHzq3N5f9EGjuiexT9H5ZDeWIlEpC6obm6udcHzitoLB8xsIOFFuZoAZcBV7j7dwqsaPQicDBQBP3P3WcF7RgG3BR/xF3d/pjZjluhNmv8117/8OSXlIf54aj9+9oOuJGkNEpE6o7pmru1U0ry1m7u3iElEcB9wh7u/bWYnB9vDCC/Q1St4DCXchzPUzDKBPwE5Qby5ZjYhuB9G6oAXp6/kltfmckiHljxw3kC6t2kW75BEZB9VVzNpDmBmdwLrgH8TXrL3IqBdDGNyYHeiagmsDV6PBJ4N7neZZmYZZtaOcKKZ7O6bg3gnAycCL8YwRqkhj/1vCfe+s4hhfdrw2EWDSUvVjYgidVEkDdI/dvcBFbYfM7PPgT/GKKbfAJPM7K+E79D/QVDeAVhV4bjVQVlV5d9jZpcTXoqYzp01VXk8lZWHuGviQp7+eDk/HtCev54zQGuQiNRhkSSTQjO7CBhLuNZwAVAYzUnN7F3ggEp23QoMB37r7v8xs3OBp4AR0ZxvN3cfDYwGyMnJqbIJT2Irf3sx17wwi8+WbeaSI7vyh1P6qX9EpI6LJJlcSLjj+8Fg+6OgbL+5e5XJwcye5duRZC/z7Q2Sa4BOFQ7tGJStIdzUVbH8f9HEJ7Eza2UBVz6Xy9adpTxw3gDOGNQx3iGJSA3Ya7uCuy9395Hu3jp4nO7uy2MY01rg6OD1scDi4PUE4KcWdjiwNRhxNgk4PpjNuBVwfFAmCWblpiJGjZlO45RkXr3ySCUSkXpkrzUTM+sIPAwcGRR9CPza3VfHKKbLgAfNLAXYRdDHAUwkPCw4j/DQ4EsA3H1zMEhgRnDcn3d3xkvi2FVazpXP52LA878YSqdMLWYlUp9E0sz1NPACcE6wfXFQdlwsAnL3j4DBlZQ7cHUV7xkDjIlFPFIzbp8wn/lrt/HUqBwlEpF6KJLhM23c/Wl3Lwse/wLaxDguqUdenrmKsTNWcfUxPRjet228wxGRGIgkmWwys4vNLDl4XAxsinVgUj/kbdjBH16fxxHds/jdcX3iHY6IxEgkyeTnwLnA18HjbIL+CpHqlJSF+M1Ls0lrlMzfzx9Isob/itRbkUxBvwL4cS3EIvXM/ZO/Yt6abYz+yWDatmgS73BEJIb2WjMxs/vMrIWZNTKz98wsP2jqEqnSJ0s28sTUJVw4tDPHH1TZ/akiUp9E0sx1vLtvIzwV/XKgJ3B9LIOSum3lpiJ+M3YO3Vqnc9spfeMdjojUgkiSye6msFOAl919awzjkTpu7ZadXPjkNErKQzx+8WAtbCXSQETyP/1NM1sE7ASuNLM2hG8mFPmO/O3FXPzkZ2wtKuWFyw6nd9vm8Q5JRGpJJNOp3ER45t4cdy8lPMnjyFgHJnXLtl2l/OSpz1i3dRdPXzKE/h1bxjskEalF1S2Oday7v29mZ1Yoq3jIq7EMTOqO0vIQVz03i7wNO3j6kiHkdM2Md0giUsuqa+Y6GngfOK2SfY6SiQDuzq2vzeWjvI3839mH8KNemhxBpCGqbqXFPwXPukFRqvTolDzGzVzNtcf25JycTnt/g4jUS5HcZ5JlZg+Z2SwzyzWzB80sqzaCk8Q2Z9UW/vrfrzh9YHt+e1zveIcjInEUydDgsUA+cBbhqVTygZdiGZQkPnfnzjcX0LpZY/5yRv89+9NEpIGJJJm0c/c73X1Z8PgLoKlfG7g3v1hH7ooCrj+hN80a614SkYYukmTyXzM738ySgse5aCXDBm1XaTn3vL2Ifu1acPZg9ZOISGTJ5DLCi2MVB4+xwBVmtt3MtsUyOElMT320jDVbdvKHU/tpJmARASKbNVi3Mcs31m/bxT+m5HF8v7Yc0UPjMEQkrMqaScWZgc3syD32XRPLoCRx/fmNBZSFnFs1gaOIVFBdM9fvKrx+eI99P49BLJLgpny5gbfmruNXx/akS1Z6vMMRkQRSXTKxKl5Xti313M6Scv74+jx6tEnnsqO6xzscEUkw1SUTr+J1Zdv7xMzOMbP5ZhYys5w99t1sZnlm9qWZnVCh/MSgLM/MbqpQ3s3MPgvKXzKz1Ghik8o9MmUxqzbv5K4z+tM4JTne4YhIgqkumRxoZl+Y2dwKr3dv94nyvPOAM4GpFQvNrB9wPnAQcCLwDzNLNrNk4FHgJKAfcEFwLMC9wAPu3hMoAC6NMjbZw7w1Wxk9dSlnHdqRw7ur011Evq+60Vwx62F194XwvVmIITy1/Vh3LwaWmVkecFiwL8/dlwbvGwuMNLOFwLHAhcExzwC3A4/FKvaGZtOOYq74dy5tmjVWp7uIVKm6iR5X1GYggQ7AtArbq4MygFV7lA8FsoAt7l5WyfHfY2aXA5cDdO7cuYZCrr9Ky0Nc/cIsNu4o5pVf/oDMdLUgikjlYjYPhpm9CxxQya5b3f31WJ23Ou4+GhgNkJOTE1W/T0Nw11sLmbZ0Mw+cN0CLXYlItWKWTNx9xH68bQ1QcX6OjkEZVZRvAjLMLCWonVQ8XqIwaf7X/OuT5Vz6w26cMahjvMMRkQQXyXQqtWkCcL6ZNTazbkAvYDowA+gVjNxKJdxJP8HdHZhCeDZjgFFAXGo99UlhcRl3TJjPgQc056aTDox3OCJSB+xXMjGz26M5qZmdYWargSOAt8xsEoC7zwfGAQuAd4Cr3b08qHVcQ3iCyYXAuOBYgBuB3wWd9VnAU9HEJvDQ+4tZu3UXd51xMI2SE+3vDRFJRPvbzJUbzUnd/TXgtSr23QXcVUn5RGBiJeVL+XbEl0Tpq/XbeerDZZyb05HBXbSWu4hEZr/+7HT3N2o6EIk/d+e28fNo1iSFm07SMGARidxeayZm9lAlxVuBmfEalSWx8XLuaqYv28z/O7O/hgGLyD6JpGbSBBgILA4ehxAeNXWpmf09ZpFJrVpdUMSf31jAYd0yOS9HC16JyL6JpM/kEOBIdy8HMLPHgA+BHwJzYxib1JJQyLn+5S9wd/52zgCStOCViOyjSGomrYBmFbbTgcwguRTHJCqpVf/6ZDmfLt3En047iE6ZTeMdjojUQZHUTO4D5pjZ/whPPX8UcLeZpQPvxjA2qQV5G3Zw7zuLGNE3m3NydHOiiOyfSJbtfcrMJvLt8Ntb3H1t8Pr6mEUmMRcKOTe/+gVpqcncfWb/yibeFBGJSCSjud4AXiB8x3lh7EOS2jJu5ipmLC/gvrMPIbt5k3iHIyJ1WCR9Jn8FfgQsMLNXzOxsM9Nvnjouf3sxd09cyGHdMjlnsJq3RCQ6kTRzfQB8ECxQdSxwGTAGaBHj2CSG7nprATtLy7n7DDVviUj0IppOxczSgNOA84BDCS9CJXXUR4s3Mn7OWq4d3oue2c32/gYRkb2IpM9kHOHO93eAR4AP3D0U68AkNsrKQ9zxxny6ZDXlqmE94h2OiNQTkdRMngIuqHDT4g/N7AJ3vzq2oUksjJu5msUbdvD4xYfSpFFyvMMRkXoikj6TSWY2yMwuAM4FlgGvxjwyqXE7isu4f/JXDOnaihMOqmwRTBGR/VNlMjGz3sAFwWMj8BJg7n5MLcUmNeyJD5awcUcxT47KUae7iNSo6momiwjPwXWqu+cBmNlvayUqqXHrtu7knx8u5ccD2jOwU0a8wxGReqa6+0zOBNYBU8zsn2Y2nPB0KlIH/X3yYkIhuP6EPvEORUTqoSqTibuPd/fzgQMJr7P+GyDbzB4zs+NrKT6pAas2F/GfWau5cGhnTeQoIjGx1zvg3b3Q3V9w99MIr2Mym/C661JHPPbBEpLMuOLo7vEORUTqqX1attfdC9x9tLsPj1VAUrPWbtnJyzNXce6QjrRrmRbvcESkntqvNeCl7njigyUAXDmsZ5wjEZH6LC7JxMzOMbP5ZhYys5wK5ceZWa6ZzQ2ej62wb3BQnmdmD1kwttXMMs1sspktDp5bxeM7JaL123bx4oxVnD24Ix0yVCsRkdiJV81kHuHRYlP3KN8InObu/YFRwL8r7HuM8CSTvYLHiUH5TcB77t4LeC/YFuCx/y2hPORcpVqJiMRYXJKJuy909y8rKZ9dYeGt+UCamTU2s3ZAC3ef5u4OPAucHhw3km8nnnymQnmDtujrbfx72grOG9JJI7hEJOYSuc/kLGCWuxcDHYDVFfatDsoA2rr7uuD110Dbqj7QzC43s5lmNjM/Pz8WMScEd+eP4+fTokkK1x+v+0pEJPYimoJ+f5jZu0BlE0Dd6u6v7+W9BwH3Avt0P4u7u5l5NftHA6MBcnJyqjyurhs/Zw3Tl2/mnjP70yo9Nd7hiEgDELNk4u4j9ud9ZtYReA34qbsvCYrXEL7HZbeOQRnAejNr5+7rguawDfsbc32wbVcpd721iAGdMjg3p1O8wxGRBiKhmrnMLAN4C7jJ3T/eXR40Y20zs8ODUVw/BXbXbiYQ7qwneK621lPfPTD5KzYVFvOXkQeTlKTZb0SkdsRraPAZZrYaOAJ4y8wmBbuuAXoCfzSzOcEjO9h3FfAkkAcsAd4Oyu8BjjOzxcCIYLtBmrdmK898spyLhnamf8eW8Q5HRBoQCw+OanhycnJ85syZ8Q6jxpSHnDMf+4Q1BUW89/thtExrFO+QRKQeMrNcd8/Zszyhmrlk/704fSWfr9rCbaf0UyIRkVqnZFIP5G8v5t53FvGDHlmMHNg+3uGISAOkZFIP/L+JCykuDXHn6QdrBUURiQslkzpu7uqtvDp7DZf+qBs92jSLdzgi0kApmdRh7s497yykVdNGXDmsR7zDEZEGTMmkDpu6eCMf523iV8f2okUTdbqLSPwomdRR5SHnnrcX0SkzjYsO7xzvcESkgVMyqaPGz17DwnXbuO74PjROSY53OCLSwCmZ1EE7S8r523+/pH+Hlpx2iIYCi0j8KZnUkJKyEFuLSmvlXI9/sIS1W3dx2yl9Nf+WiCQEJZMa8vd3v+K4Bz6gPBTb6WlWbS7i8Q+WcOoh7RjaPSum5xIRiZSSSQ35cPFGNmwvZtHX22J6nrsnLsQMbjm5b0zPIyKyL5RMakBRSRkL1oWTyPRlm2N2no/zNvL2vK+5elhP2mekxew8IiL7SsmkBsxZteWb5q0Zy2OTTMrKQ9zxxnw6ZaZx2VHdY3IOEZH9pWRSA3KXFwAw/MBspi/bTCym9X8ldzVfrd/BzSf1pUkjDQUWkcSiZFIDclcW0Cu7GSP6tWXjjhKWbSys0c/fWVLOA+9+xaDOGZx08AE1+tkiIjVBySRKoZAza0UBOV1bMaRrJlDz/SZjPl7G+m3F3HJyX80KLCIJSckkSos37GDbrjIGd8mkR5t0stJTmV6D/SabdhTz2P+WcFy/tt8kKxGRRKNkEqWZK8KJI6dLK8yMIV0za7Rm8vD7eewsLefGEw+ssc8UEalpSiZRyl1RQFZ6Kl2ymgJwWLdMVhfsZO2WnVF/9vptu3j+sxWcm9OJntlaq0REEpeSSZRyVxQwOKiVQDiZQM0MER47fRWl5c4VGgosIgkuLsnEzM4xs/lmFjKznEr2dzazHWZ2XYWyE83sSzPLM7ObKpR3M7PPgvKXzCy1tr5H/vZiVmwqIqdrq2/K+rZrQbPGKVE3dZWWh3hh+gqO6t2Grq3Tow1VRCSm4lUzmQecCUytYv/9wNu7N8wsGXgUOAnoB1xgZv2C3fcCD7h7T6AAuDRWQe8pd0X4/pLBXb7tGE9OMgZ3aRV1Mnlv4XrWbyvmJ4d3iepzRERqQ1ySibsvdPcvK9tnZqcDy4D5FYoPA/Lcfam7lwBjgZEWbls6FnglOO4Z4PRYxb2nmcs3k5qSxMEdWnyn/KjebVi8YQfPTVux35/972kr6JCRxrEHZkcbpohIzCVUn4mZNQNuBO7YY1cHYFWF7dVBWRawxd3L9iiv6vMvN7OZZjYzPz8/qljdnckL13NY18zvLU416oguDD8wmz+8Po935q3b589ekr+Dj/M2ccFhnUjWFPMiUgfELJmY2btmNq+Sx8hq3nY74SarHbGIyd1Hu3uOu+e0adMmqs+au2YrKzYVcdqAdt/bl5KcxCMXHsqgThlcO3YO05Zu2qfPfn7aSholG+cO6RRVjCIitSUlVh/s7iP2421DgbPN7D4gAwiZ2S4gF6j4m7UjsAbYBGSYWUpQO9ldHnNvfL6WRsnGCQdVPr1JWmoyT40awjlPfMplz87kjWt+GFFHekFhCS/nruLEg9uR3bxJTYctIhITCdXM5e4/cveu7t4V+Dtwt7s/AswAegUjt1KB84EJHp5RcQpwdvARo4DXYx1nKOS8+cU6jurVhoymVQ8ea5WeytM/G0KSGVc9P4tdpeXVfm5peYgrn8+luDSk4cAiUqfEa2jwGWa2GjgCeMvMJlV3fFDruAaYBCwExrn77g76G4HfmVke4T6Up2IXeVjuygLWbd3FaQP2vv56p8ym3H/uABas28adby6o9tg73pjPtKWbuees/hzcoWVNhSsiEnMxa+aqjru/Bry2l2Nu32N7IjCxkuOWEh7tVWve+HwtTRolcVy/thEdP7xvW644ujtPfLCUQzu3oktWUz5ZsokFa7fRM7sZgzpnsDS/kOemreSKo7tz5qEdY/wNRERqVlySSV1WVh5i4tx1DD+wLemNI//nu+74PuQuL+D3L38OgBl0bJXG5IXrv1lYa/iB2dxwgubgEpG6R8lkH01bupmNO0oqHcVVnUbJSfzjokP51yfL6d+hJYd3z6JVeipFJWXMXb2VvPwdjBzYQUOBRaROUjLZR298vpZmjVMY1mffbybMbtGEG/aY/bdpagpDu2cxtHtWTYUoIlLrEmo0V13QpXVTfnJEFy2dKyJSgWom++iqYT3jHYKISMJRzURERKKmZCIiIlFTMhERkagpmYiISNSUTEREJGpKJiIiEjUlExERiZqSiYiIRM3CS4I0PGaWD+zvIu2tgY01GE5d0RC/d0P8ztAwv7e+c2S6uPv3lqptsMkkGmY2091z4h1HbWuI37shfmdomN9b3zk6auYSEZGoKZmIiEjUlEz2z+h4BxAnDfF7N8TvDA3ze+s7R0F9JiIiEjXVTEREJGpKJiIiEjUlk31kZiea2ZdmlmdmN8U7nlgws05mNsXMFpjZfDP7dVCeaWaTzWxx8Nwq3rHWNDNLNrPZZvZmsN3NzD4LrvdLZpYa7xhrmpllmNkrZrbIzBaa2RH1/Vqb2W+Dn+15ZvaimTWpj9fazMaY2QYzm1ehrNJra2EPBd//CzM7dF/OpWSyD8wsGXgUOAnoB1xgZv3iG1VMlAG/d/d+wOHA1cH3vAl4z917Ae8F2/XNr4GFFbbvBR5w955AAXBpXKKKrQeBd9z9QGAA4e9fb6+1mXUArgVy3P1gIBk4n/p5rf8FnLhHWVXX9iSgV/C4HHhsX06kZLJvDgPy3H2pu5cAY4GRcY6pxrn7OnefFbzeTviXSwfC3/WZ4LBngNPjEmCMmFlH4BTgyWDbgGOBV4JD6uN3bgkcBTwF4O4l7r6Fen6tCS9ZnmZmKUBTYB318Fq7+1Rg8x7FVV3bkcCzHjYNyDCzdpGeS8lk33QAVlXYXh2U1Vtm1hUYBHwGtHX3dcGur4G28YorRv4O3ACEgu0sYIu7lwXb9fF6dwPygaeD5r0nzSydenyt3X0N8FdgJeEkshXIpf5f692qurZR/X5TMpEqmVkz4D/Ab9x9W8V9Hh5TXm/GlZvZqcAGd8+Ndyy1LAU4FHjM3QcBhezRpFUPr3Urwn+FdwPaA+l8vymoQajJa6tksm/WAJ0qbHcMyuodM2tEOJE87+6vBsXrd1d7g+cN8YovBo4Efmxmywk3Xx5LuC8hI2gKgfp5vVcDq939s2D7FcLJpT5f6xHAMnfPd/dS4FXC17++X+vdqrq2Uf1+UzLZNzOAXsGoj1TCnXYT4hxTjQv6Cp4CFrr7/RV2TQBGBa9HAa/Xdmyx4u43u3tHd+9K+Lq+7+4XAVOAs4PD6tV3BnD3r4FVZtYnKBoOLKAeX2vCzVuHm1nT4Gd993eu19e6gqqu7QTgp8GorsOBrRWaw/ZKd8DvIzM7mXDbejIwxt3vim9ENc/Mfgh8CMzl2/6DWwj3m4wDOhOevv9cd9+zc6/OM7NhwHXufqqZdSdcU8kEZgMXu3txHMOrcWY2kPCgg1RgKXAJ4T806+21NrM7gPMIj1ycDfyCcP9AvbrWZvYiMIzwVPPrgT8B46nk2gaJ9RHCTX5FwCXuPjPicymZiIhItNTMJSIiUVMyERGRqCmZiIhI1JRMREQkakomIiISNSUTkRpiZuVmNqfCo9rJEc3sl2b20xo473Izax3t54hEQ0ODRWqIme1w92ZxOO9ywjPgbqztc4vsppqJSIwFNYf7zGyumU03s55B+e1mdl3w+tpg/ZgvzGxsUJZpZuODsmlmdkhQnmVm/w3W43gSsArnujg4xxwzeyJYNkEk5pRMRGpO2h7NXOdV2LfV3fsTvsP475W89yZgkLsfAvwyKLsDmB2U3QI8G5T/CfjI3Q8CXiN8JzNm1pfwXd1HuvtAoBy4qCa/oEhVUvZ+iIhEaGfwS7wyL1Z4fqCS/V8Az5vZeMLTXQD8EDgLwN3fD2okLQivP3JmUP6WmRUExw8HBgMzwjNjkEb9mqBREpiSiUjt8Cpe73YK4SRxGnCrmfXfj3MY8Iy737wf7xWJipq5RGrHeRWeP624w8ySgE7uPgW4EWgJNCM82eZFwTHDgI3BujJTgQuD8pOA3euzvwecbWbZwb5MM+sSu68k8i3VTERqTpqZzamw/Y677x4e3MrMvgCKgQv2eF8y8FywhK4BD7n7FjO7HRgTvK+Ib6cNvwN40czmA58QnlIdd19gZrcB/w0SVClwNeGZYUViSkODRWJMQ3elIVAzl4iIRE01ExERiZpqJiIiEjUlExERiZqSiYiIRE3JREREoqZkIiIiUfv/3l1ggIzF/IIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ep_reward_list = []     # store reward history of each episode\n",
    "avg_reward_list = []        # store average reward history of last few episodes\n",
    "\n",
    "for ep in range(total_episodes):\n",
    "\n",
    "    prev_state = env.reset()\n",
    "    episodic_reward = 0\n",
    "\n",
    "    while True:\n",
    "        tf_prev_state = tf.expand_dims(tf.convert_to_tensor(prev_state), 0)\n",
    "\n",
    "        action = policy(tf_prev_state, ou_noise)\n",
    "        state, reward, done, info = env.step(action)        # receive state & reward\n",
    "\n",
    "        buffer.record((prev_state, action, reward, state))\n",
    "        episodic_reward += reward\n",
    "\n",
    "        buffer.learn()\n",
    "        update_target(target_actor.variables, actor_model.variables, tau)\n",
    "        update_target(target_critic.variables, critic_model.variables, tau)\n",
    "\n",
    "        # end episode\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "        prev_state = state\n",
    "\n",
    "    ep_reward_list.append(episodic_reward)\n",
    "\n",
    "    avg_reward = np.mean(ep_reward_list[-40:])      # mean of last 40 episodes\n",
    "    print(\"Episode * {} * Avg Reward is ==> {}\".format(ep, avg_reward))\n",
    "    avg_reward_list.append(avg_reward)\n",
    "\n",
    "plt.plot(avg_reward_list)       # graph for Episodes vs Avg.rewards\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Avg. Episodic Reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save weights\n",
    "actor_model.save_weights(\"pendulum_actor.h5\")\n",
    "critic_model.save_weights(\"pendulum_critic.h5\")\n",
    "\n",
    "target_actor.save_weights(\"pendulum_target_actor.h5\")\n",
    "target_critic.save_weights(\"pendulum_target_critic.h5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('python_ai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a5af77f05d3bdf18f34d2227fcd1c3aa7be868e46f5c9d85f13e352356a74cc6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}