Weighted_Qlearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "259164d8",
   "metadata": {
    "id": "259164d8"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1098686e",
   "metadata": {
    "id": "1098686e"
   },
   "outputs": [],
   "source": [
    "gamma = 0.95\n",
    "nA = 4\n",
    "nS = 9\n",
    "p = 0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "mNgB1fpI8olB",
   "metadata": {
    "id": "mNgB1fpI8olB"
   },
   "source": [
    "## Bias Game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1bLhjEei8sZ4",
   "metadata": {
    "id": "1bLhjEei8sZ4"
   },
   "outputs": [],
   "source": [
    "def rewardnstate(a, s):\n",
    "    r, state, terminal = P[s][a]\n",
    "    if s==1:\n",
    "        r = r + np.random.uniform(-1, 1)\n",
    "    return r, state, terminal\n",
    "  \n",
    "def act_functn(Q, state, state_visits):\n",
    "    #eps = 0.1\n",
    "    #eps = 1/np.power(state_visits[state], 0.2)\n",
    "    eps = 1/np.sqrt(state_visits[state])\n",
    "    state_visits[state] += 1\n",
    "    if np.random.choice([True, False], p=[1-eps, eps]):\n",
    "        action = np.argmax(Q[state, :nAfes[state]])\n",
    "    else:\n",
    "        action = np.random.choice(nAfes[state])\n",
    "    return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "tM3DbDqw85M-",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tM3DbDqw85M-",
    "outputId": "6582ea20-23f2-4197-845f-a379a7c5d7c5"
   },
   "outputs": [],
   "source": [
    "nAfes = [1, 8, 2] #num_Actions_for_each_state\n",
    "\n",
    "def train(epoch = 1000):\n",
    "    np.random.seed(20)\n",
    "    nS = 3\n",
    "    nA = 8\n",
    "    count_matrix_a = np.ones([nS, nA])\n",
    "    count_matrix_b = np.ones([nS, nA])\n",
    "    state_visits = np.ones(nS)\n",
    "    reward_list = []\n",
    "    maximal_value_a = []\n",
    "    maximal_value_b = []\n",
    "    maximal_value = []\n",
    "    pref_list = []\n",
    "    cumsum = 0\n",
    "    count = 0\n",
    "    Qa = np.random.rand(nS, nA) #Random between -0.5 and 0.5\n",
    "    Qb = np.random.rand(nS, nA)\n",
    "    Qa[0], Qb[0] = 0, 0\n",
    "    start_state = 2\n",
    "    for i in range(epoch):\n",
    "        state = start_state\n",
    "        terminal = True if state==0 else False\n",
    "        while(not terminal):\n",
    "            action = act_functn(Qa + Qb, state, state_visits)\n",
    "            reward, next_state, terminal = rewardnstate(action, state)\n",
    "            cumsum += reward\n",
    "            count += 1\n",
    "\n",
    "\n",
    "            if np.random.choice([True, False]):#Qa\n",
    "                alpha_a = (1/np.power(count_matrix_a[state, action], p))\n",
    "                action_star = np.argmax(Qa[next_state, :nAfes[next_state]])\n",
    "                action_l = np.argmin(Qa[next_state, :nAfes[next_state]])\n",
    "                diff = np.abs(Qb[next_state, action_star]-Qb[next_state, action_l])\n",
    "                beta_a = diff/(100+diff)\n",
    "                delta = reward + gamma*(beta_a*Qa[next_state, action_star]\n",
    "                                        + (1-beta_a)*Qb[next_state, action_star]) - Qa[state, action]\n",
    "                Qa[state, action] += alpha_a*(delta)\n",
    "                count_matrix_a[state, action] += 1\n",
    "            else:\n",
    "                alpha_b = (1/np.power(count_matrix_b[state, action], p))\n",
    "                action_star = np.argmax(Qb[next_state])#Qb\n",
    "                action_l = np.argmin(Qb[next_state])\n",
    "                diff = np.abs(Qa[next_state, action_star]-Qa[next_state, action_l])\n",
    "                beta_b = diff/(100+diff)\n",
    "                delta = reward + gamma*(beta_b*Qb[next_state, action_star]\n",
    "                                        +(1-beta_b)*Qa[next_state, action_star])-Qb[state, action]\n",
    "                Qb[state, action] += alpha_b*(delta)\n",
    "                count_matrix_b[state, action] += 1\n",
    "\n",
    "\n",
    "            state = next_state\n",
    "\n",
    "        reward_list.append(cumsum/count)\n",
    "        maximal_value_a.append(np.max(Qa[start_state, :nAfes[start_state]]))\n",
    "        maximal_value_b.append(np.max(Qb[start_state, :nAfes[start_state]]))\n",
    "        pref = .0\n",
    "        pref += Qa[2][0] - Qa[2][1] #preference of left over right\n",
    "        pref += Qb[2][0] - Qb[2][1] #preference of left over right\n",
    "        pref /= float(2)\n",
    "        pref_list.append(pref)\n",
    "    return pref_list, maximal_value_a, maximal_value_b, Qa, Qb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "VyQ7qWxIAaa8",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 282
    },
    "id": "VyQ7qWxIAaa8",
    "outputId": "11300ea4-4317-4241-ca07-a0637e1735af",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAABE+klEQVR4nO3deXhU5cE28PvMZJZMkpkkJJnshH3fkTTgWqJRKZZucimfUKr4qvgVm1YFUdD2raFWKbZFcCku71vEpWr9CqKIAqJRJOz7FkiATBaSzEy2mcnM8/0xmZMMJJCBJCfh3L/rmiuZM88588wRkzvPKgkhBIiIiIgUolG6AkRERKRuDCNERESkKIYRIiIiUhTDCBERESmKYYSIiIgUxTBCREREimIYISIiIkUxjBAREZGiwpSuQHv4fD6cPXsWUVFRkCRJ6eoQERFROwgh4HQ6kZycDI2m7faPHhFGzp49i7S0NKWrQURERJehuLgYqampbb7eI8JIVFQUAP+HMZvNCteGiIiI2sPhcCAtLU3+Pd6WHhFGAl0zZrOZYYSIiKiHudQQi5AHsG7ZsgVTp05FcnIyJEnCRx99dNHyH3zwAW6++WbEx8fDbDYjKysLn376aahvS0RERFepkMNIbW0tRo0aheXLl7er/JYtW3DzzTdj3bp1KCgowE033YSpU6di586dIVeWiIiIrj6SEEJc9smShA8//BDTpk0L6bxhw4Zh+vTpWLRoUbvKOxwOWCwW2O12dtMQERH1EO39/d3lY0Z8Ph+cTidiY2PbLONyueByueTnDoejK6pGREQqJYRAY2MjvF6v0lXpUbRaLcLCwq542Y0uDyPPP/88ampqcOedd7ZZJi8vD88880wX1oqIiNTK7XajpKQEdXV1SlelRzKZTEhKSoJer7/sa3RpGFm9ejWeeeYZ/Pvf/0ZCQkKb5RYsWIDc3Fz5eWBqEBERUUfy+XwoLCyEVqtFcnIy9Ho9F9dsJyEE3G43ysvLUVhYiAEDBlx0YbOL6bIwsmbNGtx333147733kJ2dfdGyBoMBBoOhi2pGRERq5Xa74fP5kJaWBpPJpHR1epzw8HDodDqcOnUKbrcbRqPxsq7TJXvTvP3225g9ezbefvttTJkypSvekoiIqN0u9y966ph7F3LLSE1NDY4dOyY/LywsxK5duxAbG4v09HQsWLAAZ86cwVtvvQXA3zUza9YsvPjii8jMzITNZgPgT1MWi+WKPwARERH1bCHHme3bt2PMmDEYM2YMACA3NxdjxoyRp+mWlJSgqKhILv/KK6+gsbERc+fORVJSkvyYN29eB30EIiIi6slCbhm58cYbcbGlSd54442g55s2bQr1LYiIiEhF2ElGREREilJ9GKl1NeLlzcdRdI7zy4mIiNridrs77dqqDyP/vfYA8j45hNv/+pXSVSEiIoUJIVDnblTkEeruLC6XC7/+9a+RkJAAo9GIa6+9Ft9//z18Ph9SU1OxYsWKoPI7d+6ERqPBqVOnAADV1dW477775I1sf/jDH2L37t1y+aeffhqjR4/Ga6+9hj59+lz2tN326PIVWLubr45WAABqXI0K14SIiJRW7/Fi6CJldpY/8PscmPTt/7X82GOP4V//+hfefPNN9O7dG8899xxycnJw7Ngx3HXXXVi9ejUefPBBufw///lPTJo0Cb179wYA/OIXv0B4eDg++eQTWCwWvPzyy5g8eTKOHDkib9ly7Ngx/Otf/8IHH3wArVbbsR+4BdW3jDR6L3ufQCIiIkXU1tZixYoV+POf/4zbbrsNQ4cOxauvvorw8HD84x//wIwZM/D111/Ls1t9Ph/WrFmDGTNmAAC2bt2Kbdu24b333sP48eMxYMAAPP/884iOjsb7778vv4/b7cZbb72FMWPGYOTIkZ32eVTfMuLx+pSuAhERdRPhOi0O/D5Hsfdur+PHj8Pj8WDSpEnyMZ1OhwkTJuDgwYN49NFHMWTIEKxevRrz58/H5s2bUVZWhl/84hcAgN27d6Ompga9evUKum59fT2OHz8uP+/duzfi4+Ov8JNdmurDiJthhIiImkiSFFJXSXc2Y8YMOYysXr0at956qxw+ampqkJSU1OryG9HR0fL3ERERXVJXdtOwm4aIiHqYfv36Qa/X4+uvv5aPeTwefP/99xg6dCgA4O6778a+fftQUFCA999/X+6iAYCxY8fCZrMhLCwM/fv3D3rExcV1+edRfRhhNw0REfU0ERERePDBB/Hoo49i/fr1OHDgAObMmYO6ujrce++9AICMjAxMnDgR9957L7xeL+644w75/OzsbGRlZWHatGn47LPPcPLkSXzzzTdYuHAhtm/f3uWf5+poi7oCjT62jBARUc+zZMkS+Hw+3HPPPXA6nRg/fjw+/fRTxMTEyGVmzJiBhx56CDNnzkR4eLh8XJIkrFu3DgsXLsTs2bNRXl6OxMREXH/99bBarV3+WSQR6sRmBTgcDlgsFtjtdpjN5g69dsb8tfL3J5dwR2EiIjVpaGhAYWFhp6+jcTW72D1s7+9v1XfTEBERkbIYRoiIiEhRDCNERESkKIYRIiIiUhTDCBERqV4PmMvRbXXEvWMYISIi1dLpdACAuro6hWvScwXuXeBeXg7VrzNCRETqpdVqER0djbKyMgCAyWSCJEkK16pnEEKgrq4OZWVliI6OvqJdfRlGiIhI1RITEwFADiQUmujoaPkeXi6GESIiUjVJkpCUlISEhAR4PB6lq9Oj6HS6K2oRCWAYISIigr/LpiN+sVLoOICViIiIFMUwQkRERIpiGCEiIiJFMYwQERGRohhGiIiISFEMI0RERKQohhEiIiJSFMMIERERKYphhIiIiBTFMEJERESKYhghIiIiRTGMEBERkaIYRoiIiEhRDCNERESkKIaRFt75vkjpKhAREakOw0gLj/9rr9JVICIiUh3VhxGr2aB0FYiIiFRN9WFEI0lKV4GIiEjVGEZahJEoY5iCNSEiIlIn1YeRlqJNOqWrQEREpDqqDyNCCPn76HC9gjUhIiJSJ9WHEaerUf7eHM5uGiIioq6m+jBS7/bK33997JyCNSEiIlIn1YeR8RkxSleBiIhI1VQfRkx6ds0QEREpKeQwsmXLFkydOhXJycmQJAkfffTRJc/ZtGkTxo4dC4PBgP79++ONN964jKp2jpYDWNNiwxWsCRERkTqFHEZqa2sxatQoLF++vF3lCwsLMWXKFNx0003YtWsXHnnkEdx333349NNPQ65sZxAtvvf5FKsGERGRaoXcR3Hbbbfhtttua3f5lStXok+fPnjhhRcAAEOGDMHWrVvxl7/8BTk5OaG+fYdr0TCC6jq3chUhIiJSqU4fM5Kfn4/s7OygYzk5OcjPz+/stw5ZrduLUkeD0tUgIiJSlU4PIzabDVarNeiY1WqFw+FAfX19q+e4XC44HI6gR2cR5z3/9gSn9xIREXWlbjmbJi8vDxaLRX6kpaV12nu1HMAKAHUt1h0hIiKiztfpYSQxMRGlpaVBx0pLS2E2mxEe3vrslQULFsBut8uP4uLizq6mrNHLUaxERERdqdMX2cjKysK6deuCjm3YsAFZWVltnmMwGGAwGDq7aq1ye8/vuCEiIqLOFHLLSE1NDXbt2oVdu3YB8E/d3bVrF4qKigD4WzVmzpwpl3/ggQdw4sQJPPbYYzh06BBeeuklvPvuu/jNb37TMZ/gCgV6acI0EgDAw5YRIiKiLhVyGNm+fTvGjBmDMWPGAAByc3MxZswYLFq0CABQUlIiBxMA6NOnD9auXYsNGzZg1KhReOGFF/Daa691i2m9ACCahrDqw/y3gt00REREXSvkbpobb7zxgkGfLbW2uuqNN96InTt3hvpWXUqn1QDwspuGiIioi3XL2TRdKZCr/GGE3TRERERdjWGkKYwYmrppthwpV7A2RERE6sMw0jRmJNLg77Ey6bVKVoeIiEh1VB9GAib0iQUAnKvl/jRERERdSfVh5PxuGncjx4wQERF1JYaRpq9Gnb975nRVPcqdLuUqREREpDKqDyMBgZYRAHjzm5PKVYSIiEhlGEYC3TS65luhD+NtISIi6iqq/60bmE1jCGueRcMZNURERF2HYaSpZSTK2LwYbWD8CBEREXU+1YeRAJO+OYwkmo0K1oSIiEhdVB9GWu5EMzot+oJjRERE1LkYRpr6aSTJ/2h5jIiIiDqf6sNIgARA05RGGEWIiIi6jurDSMvgIQWOsWWEiIioyzCMNOUOSZJadNMoVx8iIiK1YRhp+ioBkMBuGiIioq6m+jASEDyAVdm6EBERqQnDSIvkEQgjPqYRIiKiLqP6MCJ300jspiEiIlKC6sNIgASJ64wQEREpQPVhpGXuCKwzQkRERF2HYSTQKSNxzAgREZESGEaas8gFx4iIiKjzqT6MBPgXPWsawMowQkRE1GVUH0aCx4w0HVOmKkRERKrEMNL0VUJzVw3HjBAREXUdhpGm4OFfgZVNI0RERF1N9WGkpeZumu6VRoor67DvjF3pahAREXWKMKUr0F1ILTpqfN0oiwghcNPzm9DoE9j0uxuRERehdJWuekII7Cyuxse7zmLL0XLUubyINukwMtWCazJiMSLVgv7xkQjTMssTEXUE1YcReWpvN90or7iyHo1N6Sj33V344KFJCtfo6iSEwO7TdnxxsBQf7jqD4sr6oNdtjgYcsjnx7vbTAIBwnRZ94yPQPyESNw1KwMhUCzJ6RUCj4cJ5REShUn0YCWg5gLU7ddMcKHHI3+8oqlauIlcpZ4MH//yuCO8XnMaxshr5eIRei8lDrLh9RBKSo404U1WPglNV2HfWjh1F1aj3eLH/rAP7zzrw711nAQBxkQYMSoxEarQJo9Oj0ScuAiNSLIgw8H8zIqKLUf1PyZbBQ9MN1xk5WuoMeu5s8CDKqFOoNleXLw+XYeEHe3HW3gAAMOm1uGlQAiYPScCtwxNh0jf/7zEyNRq3jUgCAPh8AicqanCyog4FRVX45lgFDtqcqKhxoeKYC8A5vLO9GIB/HFJydDhGp0VjQp9YJFvC0buXCVFGHaJNOhh12i7/3ERE3Q3DSIu5vd1xo7z9Zx1Bz1/dcgK5twxSqDZXh0avD8/8vwP4n29PAQASzUb8evIATBmZBEv4pYOeRiOhf0IU+idEIXuoFQDgbvRhV3E1iivrcLSsBvvP2nG8rAZn7Q04XVWP01X1+M+ekguuFWkIQ5QxDDqtBjEmHfRhGri9Au5GH9yNXri9PngaBbxC4JqMGNw4MAE5wxJhMTGQEtHVg2Gk6WvQrr2K1eZCx8trgp7/9YtjDCNXwF7nwW/f243PD5YCAGZkpuN3twxCTIT+iq6rD9NgQp9YTOgTKx8TQqC8xoXjZbXYVliJ3aerUeZswKlzdah3e9HoE6hxNaLG1QgAKKq8+Hus22vDur02PP3/9uOmQQn4+bhUjE6LhiVcx7EqRNSjqT6MBEhSYEZN9+mm8fkETlXWAQD+Mn0UfvPObmg1Eho8XjbvX4aNB0vx67d3otbthT5Mgxenj5a7XjqDJElIiDIiIcqIrH69gl4TQsDpakS504U6lxeuRi8qa91o9AnotRrowzTQNX01hGngqPfg6+MV2HCgFEdKa7B2bwnW7vW3tETotbh2QBymjkrGDwcnBHUvERH1BKr/qdWyS6a7ddOUOBrgbvRBp5Vwx6gUPP3xAdjrPThRXouhyWalq9ejfHviHB743wJ4vAJWswEvzRiLcb1jL31iJ5EkCWajDuYQxv9M7B+H390yCDuLq/Hu98XYdLgcNkcDat1efLq/FJ/uL0WYRsLE/nHIGWbF2PQYDLJGsdWEiLo9hpGmrxKaV2DtLuuMHGwaL5IWa4JWI2GgNRLfn6zCV0fLGUZCsOVIuRxEsvr2wpu/mgB9WM9cI0SSJIxNj8HY9BgIIVDv8eJ4WS3+s+csPtlnQ1FlHbYcKceWI+UAgP4Jkbj/ur748ZhkGMLYmkZE3ZPqw0iAJEktpvZ2DwVFVQCACRn+v+DHZ8Ti+5NV+P5kFf7rBiVr1v2dKK/Bur0l+M+eEhyy+WckDUs24x+/HN9jg8j5JEmCSR+GEakWjEi1YP5tg3G8vBaf7rdh85Fy7D1tx7GyGjz2rz147tNDuGtCOu7OTEeSJVzpqhMRBWEYaZE8uls3TWDdi8GJUQCA6wfEY8Wm4zhc6rjYaaolhMBnB0rx2lcn8P3JKvm4XqvBz8al4LGcwVf1eApJktA/IRL9E/pj7k394Wzw4O1tRXj965MosTfgb18cw0ubjuPWYYm4J6s3MvvENu/HRESkoKv3J3M7yd00EvD5Af8Mi/X7bLjvur7KVarJ8aYw0j/BH0YCoaS4sh41rkZEcjEt2b4zdiz+eD8KTvlDiFYjYVL/OEwZkYhbhiZe8WyZnijKqMP91/fD7El9sOFAKd785iS+K6yUB78OTozCzKwM/HRsCgdEE5GiVP/bTN61F0Ct2wsA2H6q6iJndA1Xoxcnz9UCAAZYIwEg6BfqF4fKcMeoZEXqBvjv2183HkNlrQuDk8z4Qd9e6B1rUmSw5LvfF2PhR3vh8QqEaST86to+uPfaPrCajV1el+5Ip9Xg9hFJuH1EEg7ZHHgr/xQ+3HEGh2xOPPHhXry85TgeyxmM20cksqWEiBSh+jAS0N1+Bp86Vwef8C+KlRBlkI/HRepRUePGq1tOdGoY8foECk5VYXBSFITABYuBvVdwGn/5/EjQscA01AEJkegVacCMzHTcOCih0+oIAKu2FuL3/zkAABjfOwZ/mT4aabGmTn3PnmxwohnP/mQEHr91MN4vOI1XthzHqXN1mLt6ByZkxOKZHw/DkCQOjiairqX6MNJydMjdmelY/V0R7hyfqlh9Ak40LXbWNz4i6K/Vn41LxcubT2DvGXunvO83xyrwP9+ewif7bEHHR6ZaMDotGnGRBiz7/EjQjKMx6dHYf9bRtGqoT95D58tDZSh46uZ2rWoaKq9PYPmXx7B0gz8QzZ6UgSenDIWW01jbxRKuw73X9sGd41Px2leFeGXLCWw7WYnb//oVfjwqGQtuH8KWJSLqMgwj8i9VCfGR/hYIXTfYGv54ub+Lpl98ZNDxeyf1wcubTwAANh0uwy9f/x65Nw/ErycPaNd1nQ0erNlWjE/2leChG/vjqX/vQ0nT3iwXs+e0HXtOXxiA1v36OgxNNqPR60OJvQF1bi/2nrHjd+/tRqNP4J3vi3D/9f3aVbf2KqyoxWPv75YHqd7zg95YPHVYh76HWkQZdfjNzQPxi/GpyPvkENbuKcFHu85iw4FS3H99P/zq2gzuhUREnU71YSRAkiD/Ve3rBrNpvj1xDoB/OmpLCWYjUmPCcbqqHr98/XsAwNINR3D/9X1bHYRYUePCifJavJV/8oK9Ue57a/vF67BgMsqcDaiq86DgZCVsDv/3DR4vDtmc2PjbG+RFu8K0Grl7ZFBiFEqq6/HChiNYs624XWGkweOFViPB4/UhvOlz1Lm9qHU1orLOjZLqBtS6G3GktAarthaixtWICL0Wc3/YHw/e0LFhR41SY0xYfvdY/Nf11Vj88X7sLKrGXz4/gte2nsD/+UFv/GpSH8S36C4kIupIqg8jLXftlcOIT6naNNtdXA0AFywjDgANngsrePtfv8IXv71Rft7o9aH/wk8u670/fGgixqTHAAASLf6m+hsGxod0jZ+OS8XSz4/gREUtXt58HCNTo1HqaMCZ6npoNRIO25w4ZHOius6N6joP6j1e+VyjTgN3o++ii89NyIjFn38xEr17RYT+AalNI1Oj8a8HJuI/e0vw4udHcLy8Fis2HcfrXxfiF+PScP/1fTkmh4g63GWFkeXLl+PPf/4zbDYbRo0ahb/97W+YMGFCm+WXLVuGFStWoKioCHFxcfj5z3+OvLw8GI3K9kkLIVBcWQ/Avw+MpmlshlfhlhFHgweOBv/maRmt/LJ9bdZ4TFv+ddCxworaoOervi5s8/oFT2YjNkKP3aftkACMSosG4L8fDR4fwvVXPs0zJTockwcn4PODZcj75FBI57YMW5IERIfrkGgJR6RBiwSzEVl9e2H6NWndojvtaqTRSLhjVDJ+NCIJnx8sxfIvj2H3aTv+59tT+Od3p3Dr8ETc84MMZPaJ5VLzRNQhQg4j77zzDnJzc7Fy5UpkZmZi2bJlyMnJweHDh5GQcOHMidWrV2P+/PlYtWoVJk6ciCNHjuCXv/wlJEnC0qVLO+RDXK7AbqkAUFRZh8DvNp/C68GfbgpIMSYdIlpZS2R0U3hoSQjAZm9AosWIihoXnl3XHADm3tQP08enIy02PGgw7PnXkSSpQ4JIwMIpQ1FYUYuKGjci9FokRYcjyWKEEP6BucNTLEg0G2EO1yHWpIeAgFYj4VyNG+F6LSINYTDptZxuqhCNRsItwxJx81Ar8o+fw4rNx/HV0Qp59+DUmHDMyOyNn41LQUIUB7sS0eULOYwsXboUc+bMwezZswEAK1euxNq1a7Fq1SrMnz//gvLffPMNJk2ahLvvvhsAkJGRgbvuugvffffdFVb9yrWMHJOHWPHe9mIAyreMBNYXuVgXxI6nbsYD/1uAvJ+OwH1vbkdhRS1++fo2rH/kejy3vjmIHPz9rR0aMELRJy4CG1t0HbUXB0x2L5Lk33xvYv84HLI58OY3p/Cf3Wdxuqoef1p/CM9/dhjZQxJw5/g0XD8wni1WRBSykH5quN1uFBQUIDs7u/kCGg2ys7ORn5/f6jkTJ05EQUEBtm3bBgA4ceIE1q1bh9tvv73N93G5XHA4HEGPzhau08pjRrwKt4wEulz6xLUdRmIj9Hj3v7LQLz5SHldyyOZExvy1eHf7aQDAazPHKxZE6Oo0ONGMvJ+OwLaF2Xju5yMxNj0aXp/Ap/tLce+b2zFxyRd4/tPDOGRzdJttFYio+wupZaSiogJerxdWqzXouNVqxaFDrY8LuPvuu1FRUYFrr70WQgg0NjbigQcewBNPPNHm++Tl5eGZZ54JpWqX5fyfld1lNk0gjLQ2XqQ1i340FKu/Kwo6dteEdGQPtbZxBtGVCddrcef4NNw5Pg1HSp1Ys60YH+8+g3KnC3//8hj+/uUxDEiIxE/HpuK24YnIuEiwpquLvd6D/WftOGJzYu8ZBw6XOlDqcMFqNiA91oS0GBNSY00Ylx6D3r1MrXZFk/p0+r+CTZs24dlnn8VLL72EzMxMHDt2DPPmzcMf/vAHPPXUU62es2DBAuTm5srPHQ4H0tLSOrWekoTmAayXaBmpdTXC4/Uh2tQ5+50ENsgLLAN/KUadFt8vzMY1f/wcAJAQZcAzd3DdDeoaA61RWDR1KObfNhifHbDho51nsOVoBY6W1eBP6w/hT+sPYUiSGbcNT8TtIxLlvZao52v0+nCktAb5J85h/xk79pyxyz+/zlfudGHfmQtbuXv3MiHRbITVbES4TosoYxhSY8IxJj0G/RIiuQeXSoT0XzkuLg5arRalpaVBx0tLS5GYmNjqOU899RTuuece3HfffQCAESNGoLa2Fvfffz8WLlwIjebCniKDwQCDoQvWNDgvczSHkYufduuLW1BS3YCdi27u8PENQojmMJLQvjACAPFRBmx/Mhu1rkakx5o46JO6nD5Mgx+NTMaPRibDXu/B2j0lWLe3BPknzuFgiQMHSxxYuuEI+sZF4PqB8bhhUDyy+vbiJn0dTAgBe70H5U4Xyp0unKt1o7reA3udG46GRqTFhCMjLgIaSYJOq8EgaxQspvb9HLPXebCjuAo7T1Xhq2MV2H/GAXcrPzBTY8IxODEKw5ItGJpsRqLZiHKnC6cq63C6qg7Hymqwq7gazoZGnDpXh1Pn6tp8z7hIA1KijUiODkeSJRy9IvWINIQhvZcJ1igjUmPD5fWOqOcKKYzo9XqMGzcOGzduxLRp0wAAPp8PGzduxMMPP9zqOXV1dRcEDq3W/8OnO/UpS0DzbJqL1KvW1ShPB95z2o5J/eM6tB4l9gbUuBoRppFCXkMjLtKAuEguTEXKs4TrcHdmOu7OTEdlrRufHyjFJ/tKsPVYBU5U1OJERS3e+OYkTHotsvr2wsT+ccjsE4shSWYu6d+GBo8X5U4XypwNKHO4UNYUNipq/F/La5qfe7yh/WyNMemQHB0u7yuVZPG3VMSY9CivacCR0hp8eagMh2zOC86N0GtxTZ9YjEmLwfAUM0Y1bRvRHudqXDhSWoMyZwPKnS40eLxwNDTiSKkTu4qrUV3nQUWN/zPtbmUF6IBokw5RxjAkmo1IizUh0hCGGJMeCWYDYkx6RIfrYDHpYDbqkGgxcpB1NxRy+1dubi5mzZqF8ePHY8KECVi2bBlqa2vl2TUzZ85ESkoK8vLyAABTp07F0qVLMWbMGLmb5qmnnsLUqVPlUKIUcV7TSHu6aQ6WNDczljtdHV6nLUfKAfgHr+rD+D8M9XyxEXrceU0a7rwmDfZ6D/KPV2DL0Qp8cbAMNkcDNh4qw8ZDZQD8G0OO6x2DzL6xuCYjFiNSLKpoOfH5BCpqXDhTXY+z1Q04W13f9H09ztr9xypr3SFd0xKuQ1ykHr0iDYgx6RAdrofJoEVhRS1s9gYI4V/e4Ex1ParqPKiq82D/2UtPFugbF4GRqRZM7BeHzL6xSIu5/N26e0UakHWR4OJs8OBkRV3TPfA/7PUe2Os9OHWurrnlp86D6joPiivr5W0i2qLVSEiNCUevCD1iI/SIizQg0hCGSGMYIvRhsJh0iA7XISZCj3CdFkadFr0i9LCE6y7rc9rrPPjmeAUMOg0GJETBajbyZ3srQg4j06dPR3l5ORYtWgSbzYbRo0dj/fr18qDWoqKioJaQJ598EpIk4cknn8SZM2cQHx+PqVOn4o9//GPHfYoOIEkSwrSXDiMnWiwutu1kJaaNSenQelTU+AOO51J9RUQ9kCVch1uHJ+HW4UkQ0wT2n3Xgq6MV+K7wHApOVsHpasTmI+XY3BTKdVoJw5ItGNc7BiNTLRiRYkFGr4geudhanbsRx8tqcaKiBkXn6lBcVYfiSn/oKLHXt6s1Qx+mgdVsQEKUEQlRBsRHGRAf6f8a1/Q1PsqAXpF6GMLaF+JqXY04ea4WZ6sbcKysBlV1bpyprkeZowH2eg9iI/ToHRuBrH69cN2AOPTqwtbXKKMOI1ItGJFqabOMvd6DUkcDnA0enKluwOmqOtS7vaiocaOixiWv8lxV54GjwQN3o++SXUOtkSQg0WxERq8IxEcZEBuhxwBrJBKijIiPMiA1JhyxJj00GgmuRi8+21+K9ftt2HCgFO7G5p/nGsm/bEN8lAEDrZEYaI1CeqwJ/RMikWQJV23LoCS6U19JGxwOBywWC+x2O8zmjtvevKrWjTF/2AAAOP7s7fhsvw0P/nMHxveOwfsPTmz1nCc+3CvPXBlkjcKnv7m+w+oDAE9/vB9vfHMSD93YD4/dOrhDr03UnXl9AodsDmwrrMR3Jyqx/VSVHM5bijSEYVBiFIYnmzE8xYJhyRYMsEZ2m6b3WlcjjpXV4EipE8fKanC06fvTVfUXPU/T9MsuOTpcfgTGSiQ3LRhoCddxPNgVEEKgzOnCyYpaVNW5ca7WjcoaN2pcjXC6GlHnakR1vT+42OvcqPd4Uef2wtnQeOmLAwjTSLCajXDUe+Bssahm36aW7uPlNRcNnUadBn3jIpFkMaJfQiQSogxIiQ5HSoz/30CvCH2P++/f3t/fHKbcRALkKWYtV2a9mDPV9fD6RIcm2XNNTbFd+dcHUXeg1fhbQYYlWzB7Uh95u4aCokrsOFWNvWfsOFjiQI2rEQWnqlBwqrk5XqeV0D8hCkOSojA82f+X9KDEqE4d2Ohs8PjDRmkNjpY5cbTp+zPVbYeOXhF69IuPRHov/xTXtNhwpMaYkBITDmuUAWHdJFBdrSTJHxas5tBWDHY3+lBd78aJ8lqUOhpQ6mhARY0bR0qdqKp1w+ZoQJnThUafkP/7J0QZ8NOxqbh1eCJGpVogSRJ8Pn8YOlFRg3KnCwfOOlDYNIbq1LlaNHh8OFDiwIESh9x12VKUIQzJ0eEwh4fBbNTBHK5DtEmHuEgDEs1Gudsp2qRDr0g9TPqe8yu+59S0E5yfTyMMzbvFtqWyprnftsbViG2Fla1uZne5zjX9JRgX2TnThol6CkmSkN7LhPReJvxkTCoA/1TSExW12H/Wjn1nHHJAcTY0yjN2PthxRr5GfJQBfeMi0Dc+Ev3iIzDAGuVvDjcb29XV4/UJVNW5cepcHY6WNgWOshocLXWixN7Q5nlxkQYMSIjEQGsk+lujMCAhUh4cSj2PPkzT1DXWdojxeH2oqHHhbHUDjDoN+idEXtBVptFISLQY5Q1Ifzy6uZvf6xMoqqxDYUUNzlTV43h5Lc7VulFUWQebvR5lThecrkYcLr1wEHFbwnVaWMJ1MBn8X3tFGGA1G/yBxhiGCEOY/3OZDYgwhCE+0qDYeBZVh5GWJAlyiqxzt94y4vUJrN9vCzr28pbjHRxGmlpGIvhDi+h8YVoNBlqjMNAahZ+M8R8TQuB0VT0ONv1Fue+MHXvP2FHqcMnTW78rrAy6TrhOiwSzAS6PDzWuRsRE6ORFBu31HjiaBklW13suWByxpYQoAwZYIzEgIar5a0IkYiL4x4Ta6LQaJFn8048vh1YjoU9cRJsrbzd4vCiqrEOZw+X/N9rg/3daVedBmaMB5TUuVNb6x8nY6z1o8PhQ7/EG7Yh+Ke/+VxYm9Im9rPpfKVWHkfOHy0ToL95N03I0+y8nZuCNb05i0+HyDq3TuVp/y0gvtowQtYskSUiLNSEt1oRbhjWvd+Ro8KCw3D9g9HhZLY6X+1s1TlbUot7jDRrAWNNiyn5rrGYDBlqjWoQOf/Bo7/ocRFfKqNPKQbw9alyNOFfjgqO+ETWuRtjr/dOkSx0NKLE3oNbVCGdDI2yOBpyrcaHW7YVJwe1DVB1GWpIkSe6mafD4Wh0Lcry8eWXBWU1hBPA3z3XE4Ll6t1ceMxJqnyYRBTMbdRiVFo1R5+1O7fH6UFxZhxJ7A3RaDYQQ8AngdFUdJElCjMnfFx9lDEOvCP+0WI7loJ4m0hAW8uq1Ss5nUXUYuXDMSPPtqHM3XrC6asvpWWkxzU1xzoZGxHZAs2xFjQtC+EdUd8T1iOhCOq0GfeMj0Tf+/BWOO667lagnUnKmDuN+C4YwDQKNIbWuC/vZAgNbx6ZHI0yrQURTk5aj3tMh7y/PpOF4ESIiUhFVh5HzW6T8XTX+1pHaVgaxNjQNBAoMdLWE+1tOHA0dE0Yqm8aLxESwH5qIiNRD1WEkoGXLVGBxm/e2n76gXKBlJLypRcQcCCP17VuX5FIqa/2hJpYtI0REpCKqDiPn703T0srNxy84FpjyG960V0ZgQaWObhmJ5Qh9IiJSEVWHkYD2Dtlp7qYJtIz4u2tC3eOgLWwZISIiNVJ3GAlxFlOgmyawi2hgfMnRsvaviHcxcssIx4wQEZGKqDuMNGk5nenxi2xOV1Xnb7kItIykxZgAAN+dqGzznFCwZYSIiNRI1WGktYaRH41MAtA8LqSlwKJnMSb/GiCjmxZT6qiFYtgyQkREaqTqMBLQcsxIoNWj3uNFzl+24GiLTYkCm9gNsPoXSxqS7N8OubzG1SGBJLDcPFtGiIhITVQdRlrLDy1XYT1c6sTSDUfk54E9a+Kadt4M7Kzr8QpU1135jJrmMMLVV4mISD1UHUYCWq4zYgjTBO1Jc6yseT+awBokgfX+DWFaRDdNwy1vajW5XB6vD46m6zOMEBGRmqg6jLS2zogkSdBpm8OIq2k/GmeDR55NE2Vsbj2Jb2olKXNcWRipqvO3imik5pVdiYiI1EDVYSRAOm+lkQZP84Z4zqYFzd7KPwXA30XTMiwEdtctdTRcUR0CXTTRJv0FuwUTERFdzVQdRtoz5lSr8d+ib0+cAwDc84PeQduJJ5j9LSOlzisMIzUcL0JEROqk6jAiu0hDxLlaFzxeH7YV+tcSuXZAXNDrgZaR9nTTHCxx4NP9tlZn3lQ2ddPEmhhGiIhIXcIuXeTq1Z7JuEIAp6vq5bEjgWm9AdYof8uIzX7xlhGvT+C2F78CADw5ZQjuu65v0OucSUNERGrFlhFc2DDyee4N6BMXIT8/0rTWiCQBkfrg/NYn3h9O9p21X/Q9Wi4Z/+y6g1i/rwRlLcaZBMJIDMMIERGpjKrDSFsLlfVPiMSXv7sR43vHAIC88FmkIQya8waXjusdA61Gwumqepyprm/zvQ6WOOTvfQJ44H93YMKzG+U6BMJIL4YRIiJSGZWHEf9XqY0xI1aLfzzI0aa1RszGC6fcRhrCMMgaBQA4cNZxwesBL3x2pNXje8/Ysau4Wp6tw24aIiJSG1WPGbmUwBoih23+lpGW64u0lB5rwoESB85U1bV5rbCmFpUpI5Owdk+JfPyOv38dVC6G+9IQEZHKqLplJOD8dUYCAl0mh5rCSO9eplbLpcSEA/APdG1No9eHk+f8QeXhm/rLG+y1JtkS3q46ExERXS3YMnIRgaXeA2ZNzGi13MCmGTavbS2E2+vD7388XH7tP3vO4uHVO+XnfeIisHpOJuz1Htz35nbsb+raGZFiwU/GpGBCn9gO/hRERETdm6rDyKXGjJyvX3xkq8dHtWjpeCv/FO67ti/Sm1pRWnbJAIBR598V2KQPw//em4nyGhcGNo05ISIiUiN201zE+XNtYtpYkKx3bETQ88f/tQc+n//sT/bZ5OO/njwg+HoRegYRIiJSPVWHkcBGeW01jJw/tkMf1vrtCtdr8dubB2JokhmGMA3yT5zDH9cdDCrz+uxrkHvzwCutMhER0VVH1WHkUkamRsuBZNro5IuW/b+TB2DdvOuweOowAMA/thZiySeH5NevyeBYECIiotZwzAgA6SKDRv73vkx8fawCPxyc0K5r3p2ZjlJHA17ceBQrNx+Xj0caVH2riYiI2sSWkUuINIQhZ1gidNr236pHsgfgRyOT5Od3TUjvjKoRERFdFVT953pggGo7J9O0myRJWHrnaJjDdcg/fg7zbx3cwe9ARER09VB1GOlM+jANnv3JCKWrQURE1O2puptG3iivo5tGiIiIqN1UHUaIiIhIeaoOI501ZoSIiIjaT9VhhIiIiJSn6jDSnnVGiIiIqHOpOowQERGR8lQeRpr2pmHDCBERkWJUHkaIiIhIaaoOI1xmhIiISHmqDiNERESkvMsKI8uXL0dGRgaMRiMyMzOxbdu2i5avrq7G3LlzkZSUBIPBgIEDB2LdunWXVeGOJK8zwkEjREREigl5b5p33nkHubm5WLlyJTIzM7Fs2TLk5OTg8OHDSEhIuKC82+3GzTffjISEBLz//vtISUnBqVOnEB0d3RH1JyIioh4u5DCydOlSzJkzB7NnzwYArFy5EmvXrsWqVaswf/78C8qvWrUKlZWV+Oabb6DT6QAAGRkZV1brDsIxI0RERMoLqZvG7XajoKAA2dnZzRfQaJCdnY38/PxWz/n444+RlZWFuXPnwmq1Yvjw4Xj22Wfh9XrbfB+XywWHwxH0ICIioqtTSGGkoqICXq8XVqs16LjVaoXNZmv1nBMnTuD999+H1+vFunXr8NRTT+GFF17Af//3f7f5Pnl5ebBYLPIjLS0tlGq2m+A6I0RERIrr9Nk0Pp8PCQkJeOWVVzBu3DhMnz4dCxcuxMqVK9s8Z8GCBbDb7fKjuLi4s6tJRERECglpzEhcXBy0Wi1KS0uDjpeWliIxMbHVc5KSkqDT6aDVauVjQ4YMgc1mg9vthl6vv+Acg8EAg8EQStUuS2DMCEeNEBERKSeklhG9Xo9x48Zh48aN8jGfz4eNGzciKyur1XMmTZqEY8eOwefzyceOHDmCpKSkVoMIERERqUvI3TS5ubl49dVX8eabb+LgwYN48MEHUVtbK8+umTlzJhYsWCCXf/DBB1FZWYl58+bhyJEjWLt2LZ599lnMnTu34z7FZWretVfZehAREalZyFN7p0+fjvLycixatAg2mw2jR4/G+vXr5UGtRUVF0GiaM05aWho+/fRT/OY3v8HIkSORkpKCefPm4fHHH++4T0FEREQ9liRE88iJ7srhcMBiscBut8NsNnfYdfeftWPKX7ciIcqAbQuzL30CERERtVt7f39zbxoiIiJSlKrDCMeMEBERKU/VYYSIiIiUxzACQOI6I0RERIpRdRjp/kN3iYiIrn6qDiMBHDNCRESkHFWHkcBGeURERKQcVYeRADaMEBERKUfVYYRjRoiIiJSn6jASIHHQCBERkWJUHUbYMEJERKQ8VYcRIiIiUp6qw0gP2COQiIjoqqfqMBLAISNERETKUXUYYbsIERGR8lQdRgLYMkJERKQcVYeRvaftAIDiynqFa0JERKReqg4jiz/er3QViIiIVE/VYYSIiIiUxzBCREREimIYISIiIkUxjBAREZGiGEaIiIhIUQwjREREpChVh5FrMmKUrgIREZHqqTqM/KBvLwDA5MEJCteEiIhIvVQdRgKb9qbFmpStCBERkYqpOowQERGR8hhGiIiISFEMI0RERKQohhEiIiJSlKrDiIBQugpERESqp+owEiBJSteAiIhIvRhGiIiISFEMI0RERKQoVYcRwSEjREREilN1GAmQwEEjRERESmEYISIiIkUxjBAREZGiGEaIiIhIUaoOIxy/SkREpDxVh5EALnpGRESkHIYRIiIiUhTDCBERESmKYYSIiIgUpeowwhVYiYiIlHdZYWT58uXIyMiA0WhEZmYmtm3b1q7z1qxZA0mSMG3atMt5207D8atERETKCTmMvPPOO8jNzcXixYuxY8cOjBo1Cjk5OSgrK7voeSdPnsTvfvc7XHfddZddWSIiIrr6hBxGli5dijlz5mD27NkYOnQoVq5cCZPJhFWrVrV5jtfrxYwZM/DMM8+gb9++V1RhIiIiurqEFEbcbjcKCgqQnZ3dfAGNBtnZ2cjPz2/zvN///vdISEjAvffee/k17QSCy54REREpLiyUwhUVFfB6vbBarUHHrVYrDh061Oo5W7duxT/+8Q/s2rWr3e/jcrngcrnk5w6HI5RqhoyLnhERESmnU2fTOJ1O3HPPPXj11VcRFxfX7vPy8vJgsVjkR1paWifWkoiIiJQUUstIXFwctFotSktLg46XlpYiMTHxgvLHjx/HyZMnMXXqVPmYz+fzv3FYGA4fPox+/fpdcN6CBQuQm5srP3c4HAwkREREV6mQwoher8e4ceOwceNGeXquz+fDxo0b8fDDD19QfvDgwdi7d2/QsSeffBJOpxMvvvhimwHDYDDAYDCEUjUiIiLqoUIKIwCQm5uLWbNmYfz48ZgwYQKWLVuG2tpazJ49GwAwc+ZMpKSkIC8vD0ajEcOHDw86Pzo6GgAuOK4Ijl8lIiJSXMhhZPr06SgvL8eiRYtgs9kwevRorF+/Xh7UWlRUBI2mZy3sKnEEKxERkWJCDiMA8PDDD7faLQMAmzZtuui5b7zxxuW8JREREV2lelYTBhEREV11GEaIiIhIUaoOIxy/SkREpDxVh5EADl8lIiJSDsMIERERKYphhIiIiBSl6jAiBEeNEBERKU3VYUTGQSNERESKYRghIiIiRTGMEBERkaIYRoiIiEhRqg4jHL9KRESkPFWHkQCJI1iJiIgUwzBCREREimIYISIiIkUxjBAREZGiVB1GOH6ViIhIeaoOIwESx68SEREphmGEiIiIFMUwQkRERIpiGCEiIiJFqTqMcAVWIiIi5ak6jARw/CoREZFyGEaIiIhIUQwjREREpChVhxHBZc+IiIgUp+owEsBFz4iIiJTDMEJERESKYhghIiIiRTGMEBERkaJUHUa46BkREZHyVB1GAiQue0ZERKQYhhEiIiJSFMMIERERKYphhIiIiBTFMEJERESKYhgBV2AlIiJSEsMIERERKYphhIiIiBSl6jAiuOoZERGR4lQdRgI4ZISIiEg5DCNERESkKIYRIiIiUhTDCBERESlK1WGEw1eJiIiUp+owIuOqZ0RERIphGCEiIiJFXVYYWb58OTIyMmA0GpGZmYlt27a1WfbVV1/Fddddh5iYGMTExCA7O/ui5YmIiEhdQg4j77zzDnJzc7F48WLs2LEDo0aNQk5ODsrKylotv2nTJtx111348ssvkZ+fj7S0NNxyyy04c+bMFVeeiIiIer6Qw8jSpUsxZ84czJ49G0OHDsXKlSthMpmwatWqVsv/85//xEMPPYTRo0dj8ODBeO211+Dz+bBx48YrrvyV4gKsREREygspjLjdbhQUFCA7O7v5AhoNsrOzkZ+f365r1NXVwePxIDY2ts0yLpcLDocj6NGZOHyViIhIOSGFkYqKCni9Xlit1qDjVqsVNputXdd4/PHHkZycHBRozpeXlweLxSI/0tLSQqkmERER9SBdOptmyZIlWLNmDT788EMYjcY2yy1YsAB2u11+FBcXd0p9/ufbUwCAr49VdMr1iYiI6NLCQikcFxcHrVaL0tLSoOOlpaVITEy86LnPP/88lixZgs8//xwjR468aFmDwQCDwRBK1a7I9lNVXfZeREREFCyklhG9Xo9x48YFDT4NDEbNyspq87znnnsOf/jDH7B+/XqMHz/+8mtLREREV52QWkYAIDc3F7NmzcL48eMxYcIELFu2DLW1tZg9ezYAYObMmUhJSUFeXh4A4E9/+hMWLVqE1atXIyMjQx5bEhkZicjIyA78KERERNQThRxGpk+fjvLycixatAg2mw2jR4/G+vXr5UGtRUVF0GiaG1xWrFgBt9uNn//850HXWbx4MZ5++ukrqz0RERH1eJIQ3X+1DYfDAYvFArvdDrPZ3GHXzZi/Vv7+5JIpHXZdIiIiav/vb+5NQ0RERIpiGCEiIiJFMYwQERGRohhGiIiISFEMI0RERKQohhEiIiJSFMMIERERKYphhIiIiBTFMEJERESKYhghIiIiRak6jIRpJADALydmKFsRIiIiFVN1GJnUPw4AMDzFonBNiIiI1EvVYYSIiIiUxzACQFK6AkRERCqm6jAilK4AERERqTuMBEhsGiEiIlKMqsOIEGwbISIiUpqqwwgREREpj2EE7KYhIiJSEsMIERERKYphBIDEyb1ERESKUXUY4fhVIiIi5ak6jBAREZHyVB1GRNOyZxzASkREpBxVhxEiIiJSHsMIERERKUrVYYQDWImIiJSn6jBCREREylN1GAm0jEgcwUpERKQYVYcRIiIiUp6qw4g8tVfhehAREamZqsMIERERKY9hhIiIiBSl6jDSPIBV2XoQERGpmarDCBERESlP1WEksOaZxCGsREREilF1GCEiIiLlMYwQERGRotQdRjiAlYiISHHqDiNERESkOFWHEa7ASkREpDxVhxEiIiJSnqrDSGDRMyIiIlKOqsNIAAewEhERKYdhhIiIiBSl6jDCXhoiIiLlXVYYWb58OTIyMmA0GpGZmYlt27ZdtPx7772HwYMHw2g0YsSIEVi3bt1lVbbzsJ+GiIhIKSGHkXfeeQe5ublYvHgxduzYgVGjRiEnJwdlZWWtlv/mm29w11134d5778XOnTsxbdo0TJs2Dfv27bviyl8pwRGsREREigs5jCxduhRz5szB7NmzMXToUKxcuRImkwmrVq1qtfyLL76IW2+9FY8++iiGDBmCP/zhDxg7diz+/ve/X3HlOwoHsBIRESknpDDidrtRUFCA7Ozs5gtoNMjOzkZ+fn6r5+Tn5weVB4CcnJw2ywOAy+WCw+EIehAREdHVKaQwUlFRAa/XC6vVGnTcarXCZrO1eo7NZgupPADk5eXBYrHIj7S0tFCq2W7spCEiIlJet5xNs2DBAtjtdvlRXFzcqe/HXhoiIiLlhIVSOC4uDlqtFqWlpUHHS0tLkZiY2Oo5iYmJIZUHAIPBAIPBEErVLsvPxqZiYr9e6BMX0envRURERK0LqWVEr9dj3Lhx2Lhxo3zM5/Nh48aNyMrKavWcrKysoPIAsGHDhjbLd6X/84PeeDRnMAZYo5SuChERkWqF1DICALm5uZg1axbGjx+PCRMmYNmyZaitrcXs2bMBADNnzkRKSgry8vIAAPPmzcMNN9yAF154AVOmTMGaNWuwfft2vPLKKx37SYiIiKhHCjmMTJ8+HeXl5Vi0aBFsNhtGjx6N9evXy4NUi4qKoNE0N7hMnDgRq1evxpNPPoknnngCAwYMwEcffYThw4d33KcgIiKiHksSPWDlL4fDAYvFArvdDrPZrHR1iIiIqB3a+/u7W86mISIiIvVgGCEiIiJFMYwQERGRohhGiIiISFEMI0RERKQohhEiIiJSFMMIERERKYphhIiIiBTFMEJERESKYhghIiIiRYW8N40SAivWOxwOhWtCRERE7RX4vX2pnWd6RBhxOp0AgLS0NIVrQkRERKFyOp2wWCxtvt4jNsrz+Xw4e/YsoqKiIElSh13X4XAgLS0NxcXF3ICvE/E+dx3e667B+9w1eJ+7RmfeZyEEnE4nkpOTodG0PTKkR7SMaDQapKamdtr1zWYz/6F3Ad7nrsN73TV4n7sG73PX6Kz7fLEWkQAOYCUiIiJFMYwQERGRolQdRgwGAxYvXgyDwaB0Va5qvM9dh/e6a/A+dw3e567RHe5zjxjASkRERFcvVbeMEBERkfIYRoiIiEhRDCNERESkKIYRIiIiUpSqw8jy5cuRkZEBo9GIzMxMbNu2TekqdVtbtmzB1KlTkZycDEmS8NFHHwW9LoTAokWLkJSUhPDwcGRnZ+Po0aNBZSorKzFjxgyYzWZER0fj3nvvRU1NTVCZPXv24LrrroPRaERaWhqee+65zv5o3UpeXh6uueYaREVFISEhAdOmTcPhw4eDyjQ0NGDu3Lno1asXIiMj8bOf/QylpaVBZYqKijBlyhSYTCYkJCTg0UcfRWNjY1CZTZs2YezYsTAYDOjfvz/eeOONzv543caKFSswcuRIeZGnrKwsfPLJJ/LrvMedY8mSJZAkCY888oh8jPe6Yzz99NOQJCnoMXjwYPn1bn+fhUqtWbNG6PV6sWrVKrF//34xZ84cER0dLUpLS5WuWre0bt06sXDhQvHBBx8IAOLDDz8Men3JkiXCYrGIjz76SOzevVvccccdok+fPqK+vl4uc+utt4pRo0aJb7/9Vnz11Veif//+4q677pJft9vtwmq1ihkzZoh9+/aJt99+W4SHh4uXX365qz6m4nJycsTrr78u9u3bJ3bt2iVuv/12kZ6eLmpqauQyDzzwgEhLSxMbN24U27dvFz/4wQ/ExIkT5dcbGxvF8OHDRXZ2tti5c6dYt26diIuLEwsWLJDLnDhxQphMJpGbmysOHDgg/va3vwmtVivWr1/fpZ9XKR9//LFYu3atOHLkiDh8+LB44oknhE6nE/v27RNC8B53hm3btomMjAwxcuRIMW/ePPk473XHWLx4sRg2bJgoKSmRH+Xl5fLr3f0+qzaMTJgwQcydO1d+7vV6RXJyssjLy1OwVj3D+WHE5/OJxMRE8ec//1k+Vl1dLQwGg3j77beFEEIcOHBAABDff/+9XOaTTz4RkiSJM2fOCCGEeOmll0RMTIxwuVxymccff1wMGjSokz9R91VWViYAiM2bNwsh/PdVp9OJ9957Ty5z8OBBAUDk5+cLIfzBUaPRCJvNJpdZsWKFMJvN8r197LHHxLBhw4Lea/r06SInJ6ezP1K3FRMTI1577TXe407gdDrFgAEDxIYNG8QNN9wghxHe646zePFiMWrUqFZf6wn3WZXdNG63GwUFBcjOzpaPaTQaZGdnIz8/X8Ga9UyFhYWw2WxB99NisSAzM1O+n/n5+YiOjsb48ePlMtnZ2dBoNPjuu+/kMtdffz30er1cJicnB4cPH0ZVVVUXfZruxW63AwBiY2MBAAUFBfB4PEH3evDgwUhPTw+61yNGjIDVapXL5OTkwOFwYP/+/XKZltcIlFHjv3+v14s1a9agtrYWWVlZvMedYO7cuZgyZcoF94P3umMdPXoUycnJ6Nu3L2bMmIGioiIAPeM+qzKMVFRUwOv1Bt10ALBarbDZbArVqucK3LOL3U+bzYaEhISg18PCwhAbGxtUprVrtHwPNfH5fHjkkUcwadIkDB8+HID/Puj1ekRHRweVPf9eX+o+tlXG4XCgvr6+Mz5Ot7N3715ERkbCYDDggQcewIcffoihQ4fyHnewNWvWYMeOHcjLy7vgNd7rjpOZmYk33ngD69evx4oVK1BYWIjrrrsOTqezR9znHrFrL5EazZ07F/v27cPWrVuVrspVadCgQdi1axfsdjvef/99zJo1C5s3b1a6WleV4uJizJs3Dxs2bIDRaFS6Ole12267Tf5+5MiRyMzMRO/evfHuu+8iPDxcwZq1jypbRuLi4qDVai8YSVxaWorExESFatVzBe7Zxe5nYmIiysrKgl5vbGxEZWVlUJnWrtHyPdTi4Ycfxn/+8x98+eWXSE1NlY8nJibC7Xajuro6qPz59/pS97GtMmazuUf84OoIer0e/fv3x7hx45CXl4dRo0bhxRdf5D3uQAUFBSgrK8PYsWMRFhaGsLAwbN68GX/9618RFhYGq9XKe91JoqOjMXDgQBw7dqxH/JtWZRjR6/UYN24cNm7cKB/z+XzYuHEjsrKyFKxZz9SnTx8kJiYG3U+Hw4HvvvtOvp9ZWVmorq5GQUGBXOaLL76Az+dDZmamXGbLli3weDxymQ0bNmDQoEGIiYnpok+jLCEEHn74YXz44Yf44osv0KdPn6DXx40bB51OF3SvDx8+jKKioqB7vXfv3qDwt2HDBpjNZgwdOlQu0/IagTJq/vfv8/ngcrl4jzvQ5MmTsXfvXuzatUt+jB8/HjNmzJC/573uHDU1NTh+/DiSkpJ6xr/pKx4C20OtWbNGGAwG8cYbb4gDBw6I+++/X0RHRweNJKZmTqdT7Ny5U+zcuVMAEEuXLhU7d+4Up06dEkL4p/ZGR0eLf//732LPnj3ixz/+catTe8eMGSO+++47sXXrVjFgwICgqb3V1dXCarWKe+65R+zbt0+sWbNGmEwmVU3tffDBB4XFYhGbNm0KmqJXV1cnl3nggQdEenq6+OKLL8T27dtFVlaWyMrKkl8PTNG75ZZbxK5du8T69etFfHx8q1P0Hn30UXHw4EGxfPlyVU2FnD9/vti8ebMoLCwUe/bsEfPnzxeSJInPPvtMCMF73JlazqYRgve6o/z2t78VmzZtEoWFheLrr78W2dnZIi4uTpSVlQkhuv99Vm0YEUKIv/3tbyI9PV3o9XoxYcIE8e233ypdpW7ryy+/FAAueMyaNUsI4Z/e+9RTTwmr1SoMBoOYPHmyOHz4cNA1zp07J+666y4RGRkpzGazmD17tnA6nUFldu/eLa699lphMBhESkqKWLJkSVd9xG6htXsMQLz++utymfr6evHQQw+JmJgYYTKZxE9+8hNRUlISdJ2TJ0+K2267TYSHh4u4uDjx29/+Vng8nqAyX375pRg9erTQ6/Wib9++Qe9xtfvVr34levfuLfR6vYiPjxeTJ0+Wg4gQvMed6fwwwnvdMaZPny6SkpKEXq8XKSkpYvr06eLYsWPy6939PktCCHHl7StEREREl0eVY0aIiIio+2AYISIiIkUxjBAREZGiGEaIiIhIUQwjREREpCiGESIiIlIUwwgREREpimGEiIiIFMUwQkRERIpiGCEiIiJFMYwQERGRohhGiIiISFH/Hy02mV3ghcuCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7652876995079114\n"
     ]
    }
   ],
   "source": [
    "mu = 1\n",
    "gamma = 1\n",
    "P = {\n",
    "    0:{0: (0, 0, True)}, #state:{action: (reward, next state, terminal)}\n",
    "    1:{0: (mu, 0, True),\n",
    "       1: (mu, 0, True),\n",
    "       2: (mu, 0, True),\n",
    "       3: (mu, 0, True),\n",
    "       4: (mu, 0, True),\n",
    "       5: (mu, 0, True),\n",
    "       6: (mu, 0, True),\n",
    "       7: (mu, 0, True)}, \n",
    "    2:{0: (0, 1, False),\n",
    "       1: (0, 0, True)}\n",
    "}\n",
    "pref_list_over, maximal_value_a_over, maximal_value_b_over, Qa_over, Qb_over = train(5000)\n",
    "plt.plot(pref_list_over, label = \"over\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(pref_list_over[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c0c80fed",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxVElEQVR4nO3de3zU1Z3/8ffMJJMLMEkwIRcJdySgAgIaQ72TNXjZirVda9lyWReWVh6VhZ8KrsUqtrHa+rNaV9ptEbvVYruraKvlV4qItwiKhLsREASEJNxyI5DbnN8fSYYMBJiZZOY73+T1fDzmwcx3zsx8coLOm/M953wdxhgjAAAAm3BaXQAAAEAwCC8AAMBWCC8AAMBWCC8AAMBWCC8AAMBWCC8AAMBWCC8AAMBWCC8AAMBWYqwuoLN5vV4dOHBAvXr1ksPhsLocAAAQAGOMqqurlZWVJafz3GMrXS68HDhwQNnZ2VaXAQAAQrBv3z717dv3nG26XHjp1auXpOYf3uPxWFwNAAAIRFVVlbKzs33f4+fS5cJL66kij8dDeAEAwGYCmfLBhF0AAGArhBcAAGArhBcAAGArXW7OCwAAoTLGqLGxUU1NTVaX0iXFxsbK5XJ1+H0ILwAASKqvr9fBgwdVW1trdSldlsPhUN++fdWzZ88OvQ/hBQDQ7Xm9Xu3evVsul0tZWVlyu91sdNrJjDE6dOiQ9u/fr6FDh3ZoBIbwAgDo9urr6+X1epWdna3ExESry+my0tLStGfPHjU0NHQovDBhFwCAFufblh4d01mjWfyWAACArRBeAACArRBeAACAzzvvvCOHw6GKigqrSzkrwgsAALAVwksQth6o1DOrdqiitt7qUgAAiFr19eH9niS8BOGWZ97XUys/12/f3211KQCAMDLGqLa+0ZKbMSbgOgcMGKCnn37a79jo0aP1ox/9SFLz6p7f/OY3uv3225WYmKihQ4fqjTfe8Gv/1ltv6aKLLlJCQoKuv/567dmz54zPef/993X11VcrISFB2dnZ+sEPfqDjx4/71bFo0SJNmTJFHo9HM2fODPhnCAX7vIRgR1mN1SUAAMLoREOTRiz8f5Z89rZHC5To7ryv50ceeURPPPGEnnzyST377LOaPHmyvvzyS/Xu3Vv79u3TN77xDd1zzz2aOXOmPvnkE82bN8/v9bt27dLEiRP12GOPacmSJTp06JBmz56t2bNn64UXXvC1+9nPfqaFCxfq4Ycf7rTaz4aRFwAAurBp06bprrvu0pAhQ/STn/xENTU1WrdunSTp+eef1+DBg/Xzn/9cw4YN0+TJkzVt2jS/1xcWFmry5MmaM2eOhg4dqvHjx+uZZ57R7373O508edLX7oYbbtC8efM0ePBgDR48OKw/EyMvAACcJiHWpW2PFlj22Z1p5MiRvvs9evSQx+NReXm5JGn79u3Kzc31a5+Xl+f3eOPGjdq0aZNeeukl3zFjjO+SCsOHD5ckjRs3rlPrPhfCSwi43AUAdG0Oh6NTT92Ei9PpPGOOTENDg9/j2NhYv8cOh0Nerzfgz6ipqdG//du/6Qc/+MEZz/Xr1893v0ePHgG/Z0dF/28GAAC0Ky0tTQcPHvQ9rqqq0u7dgS8qGT58+BkTeD/66CO/x2PGjNG2bds0ZMiQjhXbiZjzAgCATd1www367//+b7333nvavHmzpk6dGtQFD2fNmqUdO3bovvvuU0lJiV5++WUtXbrUr80DDzygDz/8ULNnz1ZxcbF27Nih119/XbNnz+7knyZwEQkvzz33nAYMGKD4+Hjl5ub6Jgq1Z+nSpXI4HH63+Pj4SJQJAICtLFiwQNdee61uvfVW3XLLLZo0aVJQk2X79eun//3f/9Xy5cs1atQoLV68WD/5yU/82owcOVJr1qzR559/rquvvlqXXXaZFi5cqKysrM7+cQIW9tNGr7zyiubOnavFixcrNzdXTz/9tAoKClRSUqI+ffq0+xqPx6OSkhLf4866CmVnibJyAADdlMfj0bJly/yOTZ061Xe/vT1jTt/2/9Zbb9Wtt97qd2z69Ol+jy+//HL97W9/O2sd7e0NE05hH3l56qmnNGPGDE2fPl0jRozQ4sWLlZiYqCVLlpz1NQ6HQxkZGb5benp6uMsEAAA2EdbwUl9fr/Xr1ys/P//UBzqdys/PV1FR0VlfV1NTo/79+ys7O1u33Xabtm7dGs4yAQCAjYQ1vBw+fFhNTU1njJykp6ertLS03dcMGzZMS5Ys0euvv67f//738nq9Gj9+vPbv399u+7q6OlVVVfndAABA1xV1q43y8vI0ZcoUjR49Wtdee61effVVpaWl6Ve/+lW77QsLC5WUlOS7ZWdnh71Gh5j0AgCAVcIaXlJTU+VyuVRWVuZ3vKysTBkZGQG9R2xsrC677DLt3Lmz3ecXLFigyspK323fvn0drvu8yC4A0CUFc1FEBK+z+jes4cXtdmvs2LFatWqV75jX69WqVavO2H74bJqamrR582ZlZma2+3xcXJw8Ho/fDQCAYLTuQltbW2txJV1bfX29JAW1F017wr5Ueu7cuZo6darGjRunK664Qk8//bSOHz/uW4Y1ZcoUXXjhhSosLJQkPfroo7ryyis1ZMgQVVRU6Mknn9SXX36pf/3Xfw13qQCAbsrlcik5Odl3zZ/ExMSo26bD7rxerw4dOqTExETFxHQsfoQ9vNx55506dOiQFi5cqNLSUo0ePVorVqzwTeLdu3evnM5TA0DHjh3TjBkzVFpaqpSUFI0dO1YffvihRowYEe5SAQDdWOt0htYAg87ndDrVr1+/DgdDh+liJ/iqqqqUlJSkysrKTj+FNGD+m5KkW0dm6pffGdOp7w0AiA5NTU1nXNwQncPtdvsNWLQVzPc3F2YEAKANl8vV4TkZCK+oWyoNAABwLoQXAABgK4SXEDADHQAA6xBeAACArRBeAACArRBeAACArRBeQsCMFwAArEN4AQAAtkJ4AQAAtkJ4AQAAtkJ4CQHbvAAAYB3CCwAAsBXCCwAAsBXCCwAAsBXCSwiY8gIAgHUILwAAwFYILyHgqtIAAFiH8AIAAGyF8AIAAGyF8BICThoBAGAdwgsAALAVwgsAALAVwkuAjDFWlwAAAER4CZhfdmHSCwAAliG8AAAAWyG8BIiTRgAARAfCS4CY8wIAQHQgvATIf8oLk14AALAK4QUAANgK4SVAnDUCACA6EF4CZJiyCwBAVCC8BKjtyIuDKS8AAFiG8AIAAGyF8AIAAGyF8BIgJuwCABAdCC8BajthlykvAABYh/ACAABshfASIE4bAQAQHQgvASK7AAAQHQgvAWp7YUb2eQEAwDqEFwAAYCuElwBx2ggAgOhAeAmQ3+UBWCwNAIBlCC+BYugFAICoQHgBAAC2QngJkGHoBQCAqEB4CZDfnBemvAAAYBnCS4AYdwEAIDoQXgLEJnUAAEQHwgsAALAVwkuA2p424iKNAABYh/ASIAILAADRgfASoLZLpQkyAABYh/ACAABshfASKNP2LkMvAABYhfASIOIKAADRgfASoLbzXJjzAgCAdQgvAADAVggvAfJbbWRhHQAAdHeElwBxqggAgOhAeAkQO+wCABAdCC8AAMBWCC8BantVafZ5AQDAOoSXAHGqCACA6EB4AQAAtkJ4CQWjMAAAWIbwEiBOGwEAEB0ILwFikzoAAKJDRMLLc889pwEDBig+Pl65ublat27dOdv/6U9/Uk5OjuLj43XppZfqrbfeikSZ58TICwAA0SHs4eWVV17R3Llz9fDDD+vTTz/VqFGjVFBQoPLy8nbbf/jhh7rrrrt09913a8OGDZo0aZImTZqkLVu2hLvUgBmSDAAAlgl7eHnqqac0Y8YMTZ8+XSNGjNDixYuVmJioJUuWtNv+F7/4hSZOnKj77rtPw4cP16JFizRmzBj98pe/DHep50RcAQAgOoQ1vNTX12v9+vXKz88/9YFOp/Lz81VUVNTua4qKivzaS1JBQcFZ29fV1amqqsrvFg7+m9QBAACrhDW8HD58WE1NTUpPT/c7np6ertLS0nZfU1paGlT7wsJCJSUl+W7Z2dmdU/xpCCwAAEQH2682WrBggSorK323ffv2hf0zmfICAIB1whpeUlNT5XK5VFZW5ne8rKxMGRkZ7b4mIyMjqPZxcXHyeDx+t3Do3ztR/3btoLC8NwAACFxYw4vb7dbYsWO1atUq3zGv16tVq1YpLy+v3dfk5eX5tZeklStXnrV9pMS4nErvFS+JU0gAAFgpJtwfMHfuXE2dOlXjxo3TFVdcoaefflrHjx/X9OnTJUlTpkzRhRdeqMLCQknSvffeq2uvvVY///nPdcstt2jZsmX65JNP9Otf/zrcpZ6Xw9H85/o9R60tBACAbizs4eXOO+/UoUOHtHDhQpWWlmr06NFasWKFb1Lu3r175XSeGgAaP368Xn75ZT300EN68MEHNXToUC1fvlyXXHJJuEs9r7gYlyTpQOVJVZ9sUK/4WIsrAgCg+3GYLrbjWlVVlZKSklRZWdnp81+OHa/XZYtWSpKKFtygzKSETn1/AAC6q2C+v22/2iiSUnq45XY1d1nXinwAANgH4SVYLfNeyC4AAFiD8BKkluzC9Y0AALAI4SVIzpYlR2QXAACsQXgJUutyacILAADWILwEyXfaiFkvAABYgvASJAenjQAAsBThJUinRl4AAIAVCC/B8s15Ib4AAGAFwkuQGHkBAMBahJcgnZrzQnwBAMAKhJcgOVkqDQCApQgvQfKNvFhcBwAA3RXhJUinLg9gaRkAAHRbhJcg+XbYZewFAABLEF6CxiZ1AABYifASJK5tBACAtQgvQWqd8+IlvQAAYAnCS5BaR14AAIA1CC9BcnJhRgAALEV4CdKpywOQXgAAsALhJUgORl4AALAU4SVEZBcAAKxBeAlS64RdVhsBAGANwkuQ2OcFAABrEV6C5GgzZRcAAEQe4SVIjLwAAGAtwkuQfPu8WFwHAADdFeElSL6TRqQXAAAsQXgJlu+0EekFAAArEF6CdOrCjJaWAQBAt0V4CZJvh11mvQAAYAnCS5B8F5UmuwAAYAnCS5B8S6WtLQMAgG6L8BKk1k3qmK8LAIA1CC9BOjXyQnoBAMAKhJcg+Sbskl0AALAE4SVIp5ZKk14AALAC4SVIraeNdpbXWFsIAADdFOElSHWNXknSb9/fbXElAAB0T4SXIP3TuL6SpES3y+JKAADonggvQRrbP0WS1MT1AQAAsAThJUguZ3OXNRJeAACwBOElSDHO5hm7jLwAAGANwkuQXC3hhZEXAACsQXgJEiMvAABYi/ASJN/IS5PX4koAAOieCC9BimmZsMvICwAA1iC8BMnlYs4LAABWIrwEiTkvAABYi/ASpLarjbwEGAAAIo7wEqSkhFjf/fLqOgsrAQCgeyK8BCnW5VTL4IsaWHEEAEDEEV5C0CMuRhLhBQAAKxBeQhDrau62hibmvAAAEGmElxDEtiyXZuQFAIDII7yEoHWjOsILAACRR3gJgTumudvYqA4AgMgjvISgdaO6ldvKZAwBBgCASCK8hCCxZbXRr9/9QlsPVFlcDQAA3QvhJQT33TjMd//I8XoLKwEAoPshvITgqqGpGpHpkSROGwEAEGGElxC1LDgS0QUAgMgivITIoeZJu4y8AAAQWYSXEDlarm9EdgEAILIILyFqyS6EFwAAIozwEiJHy9AL2QUAgMgivISo9bSRl6EXAAAiivASIk4bAQBgjbCGl6NHj2ry5MnyeDxKTk7W3XffrZqamnO+5rrrrpPD4fC7zZo1K5xlhqT1tBEnjgAAiKyYcL755MmTdfDgQa1cuVINDQ2aPn26Zs6cqZdffvmcr5sxY4YeffRR3+PExMRwlhkSRl4AALBG2MLL9u3btWLFCn388ccaN26cJOnZZ5/VzTffrJ/97GfKyso662sTExOVkZERrtI6hZMJuwAAWCJsp42KioqUnJzsCy6SlJ+fL6fTqbVr157ztS+99JJSU1N1ySWXaMGCBaqtrT1r27q6OlVVVfndIoIJuwAAWCJsIy+lpaXq06eP/4fFxKh3794qLS096+u+853vqH///srKytKmTZv0wAMPqKSkRK+++mq77QsLC/XII490au2B4LQRAADWCDq8zJ8/Xz/96U/P2Wb79u0hFzRz5kzf/UsvvVSZmZmaMGGCdu3apcGDB5/RfsGCBZo7d67vcVVVlbKzs0P+/ED5dtgN+ycBAIC2gg4v8+bN07Rp087ZZtCgQcrIyFB5ebnf8cbGRh09ejSo+Sy5ubmSpJ07d7YbXuLi4hQXFxfw+3UW35wXhl4AAIiooMNLWlqa0tLSztsuLy9PFRUVWr9+vcaOHStJevvtt+X1en2BJBDFxcWSpMzMzGBLDSuubQQAgDXCNmF3+PDhmjhxombMmKF169bpgw8+0OzZs/Xtb3/bt9Loq6++Uk5OjtatWydJ2rVrlxYtWqT169drz549euONNzRlyhRdc801GjlyZLhKDYnvqtKcOAIAIKLCukndSy+9pJycHE2YMEE333yzrrrqKv3617/2Pd/Q0KCSkhLfaiK3262///3vuvHGG5WTk6N58+bpjjvu0J///OdwlhkSRl4AALBGWDep69279zk3pBswYIDfnJHs7GytWbMmnCV1Gt+FGQkvAABEFNc2ClHrUmn2eQEAILIILyFiqTQAANYgvISodeSF9AIAQGQRXkLkm/NCegEAIKIILyFystoIAABLEF5C1pxevIQXAAAiivASolMTdkkvAABEEuElRFxVGgAAaxBeQuS7MKPFdQAA0N0QXkJ06vIAxBcAACKJ8BIirm0EAIA1CC8h8l1VmvQCAEBEEV5CxOUBAACwBuElRK077LLPCwAAkUV4CdGppdKkFwAAIonwEiKH4/xtAABA5yO8hKg1uzz25nZL6wAAoLshvIRoaHovq0sAAKBbIryE6K4r+vnue5m1CwBAxBBeQuRqM+nFy6RdAAAihvASIkebnmsivAAAEDGElxD5jbx4LSwEAIBuhvASIpeT00YAAFiB8BKitvu8cNoIAIDIIbyEyP+0EeEFAIBIIbyEyP+0kYWFAADQzRBeQuRoM/LSRHoBACBiCC8d0Dr6woRdAAAih/DSAa3zXggvAABEDuGlA1rPHHHaCACAyCG8dIDvtBGb1AEAEDGElw5wctoIAICII7x0QOtqaTapAwAgcggvHdB62uh4XaPFlQAA0H0QXjrgWG2DJOmz0mqLKwEAoPsgvHRAdu8ESafmvgAAgPAjvHTAkLSekri2EQAAkUR46QCXs7n7mLALAEDkEF46wNXSe42MvAAAEDGElw6IaRl54bQRAACRQ3jpAGfLUmlGXgAAiBzCSwfE+C4PQHgBACBSCC8d0LpEmgm7AABEDuGlA1on7HJVaQAAIofw0gG+pdKEFwAAIobw0gEslQYAIPIILx3AUmkAACKP8NIBrRN2GXkBACByCC8dEONqWSrNaiMAACKG8NIBvqXSjLwAABAxhJcOYKk0AACRR3jpAJZKAwAQeYSXDnAxYRcAgIgjvHSAb8Iu4QUAgIghvHQA1zYCACDyCC8d0HpVaea8AAAQOYSXDnASXgAAiDjCSwe0THkhvAAAEEGElw5wuVgqDQBApBFeOsDFhF0AACKO8NIBrRN2d5RV66mVn2tHWbXFFQEA0PURXjqgZ3yMJGnPkVo9s2qHHn5jq8UVAQDQ9RFeOuCGnD66r2CYCi5OlyQdPV5vcUUAAHR9hJcOiI916Z7rh+juqwZJkuobvRZXBABA10d46QRxMc3dWEd4AQAg7AgvncBNeAEAIGIIL52gdeSlpq5BL6/dqyM1dRZXBABA10V46QQ945pXHZ1s8OrB1zbrpys+s7giAAC6LsJLJ+jjideDN+dodHayJOlg5UlrCwIAoAsLW3j58Y9/rPHjxysxMVHJyckBvcYYo4ULFyozM1MJCQnKz8/Xjh07wlVip5p5zWDNurZ51VFtfZPF1QAA0HWFLbzU19frW9/6lr73ve8F/JonnnhCzzzzjBYvXqy1a9eqR48eKigo0MmT9hjJSHA3nz4ivAAAED4x4XrjRx55RJK0dOnSgNobY/T000/roYce0m233SZJ+t3vfqf09HQtX75c3/72t8NVaqdJdLskSbX1jRZXAgBA1xU1c152796t0tJS5efn+44lJSUpNzdXRUVFZ31dXV2dqqqq/G5W6dVyuYCak4QXAADCJWrCS2lpqSQpPT3d73h6errvufYUFhYqKSnJd8vOzg5rnefiiY+VJFWdbJDhStMAAIRFUOFl/vz5cjgc57x99llklwkvWLBAlZWVvtu+ffsi+vlteRKaw0tDk9GJBua9AAAQDkHNeZk3b56mTZt2zjaDBg0KqZCMjAxJUllZmTIzM33Hy8rKNHr06LO+Li4uTnFxcSF9Zmfr4XbJ7XKqvsmrdbuP6rphfawuCQCALieo8JKWlqa0tLSwFDJw4EBlZGRo1apVvrBSVVWltWvXBrViyUoOh0MX9HTrYOVJTXvhY735g6t0cVaS1WUBANClhG3Oy969e1VcXKy9e/eqqalJxcXFKi4uVk1Nja9NTk6OXnvtNUnNX/xz5szRY489pjfeeEObN2/WlClTlJWVpUmTJoWrzE73/euH+O7vOVxrYSUAAHRNYVsqvXDhQr344ou+x5dddpkkafXq1bruuuskSSUlJaqsrPS1uf/++3X8+HHNnDlTFRUVuuqqq7RixQrFx8eHq8xO990r+2vV9jK9U3KIJdMAAISBw3SxZTFVVVVKSkpSZWWlPB6PJTV87/fr9dctpVp028X6bt4AS2oAAMBOgvn+jpql0l1JQmzzZnU/fH2r9hw+bnE1AAB0LYSXMMhIOnWa64UPdltYCQAAXQ/hJQxmXTdYI/s2rzIqrbLHdZkAALALwksYeOJj9f3rBkuSPtx5RN9a/KG+tfhDTVmyTiWl1RZXBwCAvYVttVF3N6RPL0lSdV2jPt5zzHf83c8P6cpBvSVJcTEu/Z8bh+nSvuwFAwBAoFhtFEab91dq/7HmvV7e/qxcf1q/v912yYmx53yfmy7JVOE3Lu30+gAAiBbBfH8TXiLEGKOiL47o6PF6SdLnpdV65u2dAb3W5XRo109uDmd5AABYKpjvb04bRYjD4dD4wamnDoyUpn1toC/MtKeitl7fXFykJq+RMUYOhyMClQIAEN0ILxbq3cOt3j3cZ33+WJtg4zWSi+wCAACrjaKZ03kqrTR5u9TZPQAAQkZ4iWKuNuHF27WmJgEAEDLCSxRztZnj0sjICwAAkggvUc3FaSMAAM5AeIlifqeNCC8AAEgivES1NtlFTcx5AQBAEuElqjkcDl+A4bQRAADNCC9RLsbZ/CsivAAA0IzwEuVasgvhBQCAFoSXKNe6XPpQTZ0qTzRYXA0AANYjvES51hVH3/jPDzX60b/pT5/ss7giAACsRXiJcjdfmqnWveqMkdZ/eczaggAAsBjhJco9fsdI7S68RfNvypHETrsAABBebCKm5fRRY5PX4koAALAW4cUmWsNLAyMvAIBujvBiEzGu5l8VIy8AgO6O8GITrSMv7PcCAOjuCC820Try0tBEeAEAdG+EF5uIdbVM2PVy2ggA0L0RXmyi9RpHnx2strgSAACsRXixiQR386/qyPF6VZ3kMgEAgO6L8GITeYNSffc/2nXEwkoAALAW4cUmEtwuZfdOkCSdbGTeCwCg+yK82MjA1J6S2OsFANC9EV5sJLZ1l13CCwCgGyO82Egse70AAEB4sZMYFyMvAAAQXmzE7Rt5IbwAALovwouNtJ422nf0hMWVAABgHcKLjRyvb5QkVZxgkzoAQPdFeLGRC5Ob93lpXXUEAEB3RHixkb4pzeGltr7J4koAALAO4cVGEtwxkqTaBsILAKD7IrzYSKLbJUk60TL3BQCA7ijG6gIQuISW8HKw8qRe27BfLqdT1w5NU1JirMWVAQAQOYQXG/HEN/+69h87oX9/ZaMk6daRmfrld8ZYWRYAABFFeLGRUX2T9c9X9tOXR2p19Hi9th6o0r5j7PkCAOheCC82EuNy6rFJl0qSinYd0V3/9ZFqTrLnCwCgeyG82FTPuOZf3Z4jtbrmidWSpNSebi3+7lj16RVvZWkAAIQVq41sql/vRCXEutTkNdp7tFZ7j9bq070Vevfzw1aXBgBAWDHyYlNJibF69/7rte9YrSTp/678XO/tOKwqLh0AAOjiGHmxsbRecRrTL0Vj+qWob0qiJOnRv2yTMcbiygAACB/CSxcxIsvju3+ous7CSgAACC9OG3UR/5zbTz9cvkWS9LdtZcpKPnPSbkqiW6Ozk+VwcGFHAIB9EV66CIfDoUGpPfTF4eN6qCXEtOfqoam6flifM473652o/BHp4SwRAIBOQXjpQr5//RD990dftjvnZdP+SknSezsO670d7a9IWjHnauVkeNp9DgCAaEF46UK+Obavvjm2b7vPfVVxQs+u2qHj9WdekfqDnYd19Hi9Pth5RI1NRn16xamPh71iAADRyWG62NKUqqoqJSUlqbKyUh4PowiB+JelH+vtz8p9j50O6f/NuUZD03tZWBUAoDsJ5vubkRfon8b11Y7yatU3enWstkH1jV79edNBXTP07HvGZPdOVDqjMwAACzDyAj///kqxXtvwVUBtF912sb6bNyC8BQEAuoVgvr8JL/Dz4a7DevTP21TX6D1rm92Hj/vuu5wOpfZ0d8pnO+TQFQN765qL0oJ+7Zh+yRqU1rNT6gAARB7hhfASVlu+qtStz75vdRlnSEmMDWkPG2OMvj4qSz3iTp1FvaBnnMYPviCg12f3TvRdKBMAEBrCC+El7BqbvNp5qEZN3s7563OivkmL1+xSY5DvZ4y05vNDnVJDR2QlxZ8RnPqmJGjABT3kdDp01xXZGtk32ZriAMAGCC+El27lZEOT9h2tDem1O8pr9OmXx9Q2M32w87CO1dYrkP8wAr0Uw+jsZC2/52sh1QgA3QHhhfCCCKmpa9QXh2r8jhkjrdt9VPVNXm07WKU3Nx1UQqwr5B2MrxiQwsRoAF0eS6WBCOkZF9Pu6aBR2c3HKk80aOXWMp1oaNKfNx4I6TP+vPGAbr40Uxf0jOtApQDQdRBegDBKSojVSzNytbnl8gzB+sWqHao80aDv/nad3rr36k6uDgDsifAChNnlA3rr8gG9Q3rtpv0VWl58IOQ5PQDQFRFegCj22O2XannxAVXXNWrh61vkDGEpeKv84em6amhqJ1YHANYgvABRrGdcjFJ7unW4pl6/K/qyQ++19MM9Wjr9crmcDg24oIfiY10drq93D7dcztADFQCEImyrjX784x/rzTffVHFxsdxutyoqKs77mmnTpunFF1/0O1ZQUKAVK1YE/LmsNkJX88meox3ay6a+yatfrfmiEyvy17tHcDssJ8S69NM7RjIKBMBPVKw2qq+v17e+9S3l5eXpt7/9bcCvmzhxol544QXf47g4Vligexs3oLfGhThnptXg1J76/dov1eQ1Olh5Usdq6ztcV+s/e44eD/69/vm3a9XD7T/yk+B2Kd0Tf85TY2P6JetHX784pJ2UAQTvjY0HtOgv21R/2iVjBqT20OsW7l0VtvDyyCOPSJKWLl0a1Ovi4uKUkZERhoqA7uufLs/WP12e3anv2eQ12n24RsGM3W45UKl/f2WjJOl4fZPfc8frm3S45txBaPNXlXqx6EulJMb6jo3pl6Ifff3iwIsIUM+4GKUEOaoEdDWvb/iq3c04q082WFDNKVE35+Wdd95Rnz59lJKSohtuuEGPPfaYLrjg7NeYqaurU13dqY6tqqqKRJlAt+dyOjSkT6+gXjM0vZduGJauyhP+/+Nr9Hp1oOKkGprOfkHQuX8s1rHa5te1/ilJqz4r16rPyoOqIxipPd1K98T7rl916YVJWnDzcDkdYgQIXc4fP9mnJe/vlrflXyX7jp6QJC2adInf9d7cLqcl9bWKqvAyceJEfeMb39DAgQO1a9cuPfjgg7rppptUVFQkl6v9yYWFhYW+UR4A0S8pMVZJbUZOWp3vquCfPPQP2n34uNpO0/uP17Zoy4HQ9tA5l0av8Q2TH66p9xsRWrv7qH7z/m7fY6dDcjoccjoccjgkx2mPW/+McTr09VEX6saL09U28rQGoNYc5PAd97Vo0/b0No7TWrRt4/++pz/fXptzvfZsdTkczfeyeycq1uIvNHTcf737hXaU++8a7nBI1wxNVf8LelhU1ZmCmrA7f/58/fSnPz1nm+3btysnJ8f3eOnSpZozZ05AE3ZP98UXX2jw4MH6+9//rgkTJrTbpr2Rl+zsbCbsAuiQqpMNqqxt0FcVJ3TseL28Rir863btP3bC6tKi1sxrBp1xLC7GqX++sr/SPfEWVNQ9LN/wlbZ81Tkh/uV1e1Vb36QnvjlSfVMSJEkXJidEJLiEbcLuvHnzNG3atHO2GTTozL+8oRo0aJBSU1O1c+fOs4aXuLg4JvUC6HSe+Fh54mOV3TvRd+yWkZmqPNEgr9fIa4y8RjLGyEi+x16vkTGtj5uPVZ6o1w+Xb1V9k9dv5Micdqf1cWubtv+yNL42xv9xO//8PP31p7/W/9jp72NOe3z2uoyRGpq8qm2Zv/Trd9tf1fbs2zv9ltQ7HdKIrCQVXJzuN8rj8P3paDMa5GhzvOVx2xEov+dPjXLdkNMn6i+pcbKhSZ/uPaYx/VJC3rqgrOqk5rxS3Kl1OR3SPwxPj+o5X0GFl7S0NKWlpYWrljPs379fR44cUWZmZsQ+EwDOJSnhzFNegejKl3dY/Vm5ir44csbxHWXVWl3SvMy/qc2l25skbdxXoY37KsJa10XpPX2BxuFwtMxTajmVp1NByNny3PjBqboo/dQ8rkS3S1cPTVVMmE6HPfLnrfrDun2+zwpFa7+mJMbqzsv7dUpdo7OTojq4SGGc87J3714dPXpUe/fuVVNTk4qLiyVJQ4YMUc+ezee2c3JyVFhYqNtvv101NTV65JFHdMcddygjI0O7du3S/fffryFDhqigoCBcZQIAOuj6nD66PqdPu8/V1jeq5mSj73Gj12jJ+7tVdbLBb/THmJaRIdP62LQ5fuqx2owiGXPqda3tyqtOamPLtcQ+L/Ofu3E+H+851u5xT3zzV6XD4dDwzF4antk5UxLeKD51sdba01bfBevGERmaf1PO+Rt2EWELLwsXLvTbcO6yyy6TJK1evVrXXXedJKmkpESVlc1/yVwulzZt2qQXX3xRFRUVysrK0o033qhFixZxWggAbCrRHaNEt/9XzUO3jgjrZ+47Wqv9x074BRxvm9N7an3c8md5dZ1eL/7Kb/XYnsPHVd6yRLiqTfj66Iuj+uiLo51a72+mjNOwjOBW7rXldDqUldS95hSFbYddq7DDLgCgM5RXnVR1XXNwaWjy6q1NB9XUyV+ZA1N76ptj+3bqe9pVVOywCwCAnfXxxKvtybCcDP5BHC1YlA8AAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyF8AIAAGyly11V2rRcrryqqsriSgAAQKBav7dbv8fPpcuFl+rqaklSdna2xZUAAIBgVVdXKykp6ZxtHCaQiGMjXq9XBw4cUK9eveRwODr1vauqqpSdna19+/bJ4/F06nvjFPo5MujnyKCfI4e+joxw9bMxRtXV1crKypLTee5ZLV1u5MXpdKpv375h/QyPx8N/GBFAP0cG/RwZ9HPk0NeREY5+Pt+ISysm7AIAAFshvAAAAFshvAQhLi5ODz/8sOLi4qwupUujnyODfo4M+jly6OvIiIZ+7nITdgEAQNfGyAsAALAVwgsAALAVwgsAALAVwgsAALAVwkuAnnvuOQ0YMEDx8fHKzc3VunXrrC4pqr377rv6x3/8R2VlZcnhcGj58uV+zxtjtHDhQmVmZiohIUH5+fnasWOHX5ujR49q8uTJ8ng8Sk5O1t13362amhq/Nps2bdLVV1+t+Ph4ZWdn64knngj3jxZVCgsLdfnll6tXr17q06ePJk2apJKSEr82J0+e1D333KMLLrhAPXv21B133KGysjK/Nnv37tUtt9yixMRE9enTR/fdd58aGxv92rzzzjsaM2aM4uLiNGTIEC1dujTcP17UeP755zVy5Ejfplx5eXn661//6nuePg6Pxx9/XA6HQ3PmzPEdo6877kc/+pEcDoffLScnx/e8LfrY4LyWLVtm3G63WbJkidm6dauZMWOGSU5ONmVlZVaXFrXeeust8x//8R/m1VdfNZLMa6+95vf8448/bpKSkszy5cvNxo0bzde//nUzcOBAc+LECV+biRMnmlGjRpmPPvrIvPfee2bIkCHmrrvu8j1fWVlp0tPTzeTJk82WLVvMH/7wB5OQkGB+9atfRerHtFxBQYF54YUXzJYtW0xxcbG5+eabTb9+/UxNTY2vzaxZs0x2drZZtWqV+eSTT8yVV15pxo8f73u+sbHRXHLJJSY/P99s2LDBvPXWWyY1NdUsWLDA1+aLL74wiYmJZu7cuWbbtm3m2WefNS6Xy6xYsSKiP69V3njjDfPmm2+azz//3JSUlJgHH3zQxMbGmi1bthhj6ONwWLdunRkwYIAZOXKkuffee33H6euOe/jhh83FF19sDh486LsdOnTI97wd+pjwEoArrrjC3HPPPb7HTU1NJisryxQWFlpYlX2cHl68Xq/JyMgwTz75pO9YRUWFiYuLM3/4wx+MMcZs27bNSDIff/yxr81f//pX43A4zFdffWWMMeY///M/TUpKiqmrq/O1eeCBB8ywYcPC/BNFr/LyciPJrFmzxhjT3K+xsbHmT3/6k6/N9u3bjSRTVFRkjGkOmk6n05SWlvraPP/888bj8fj69v777zcXX3yx32fdeeedpqCgINw/UtRKSUkxv/nNb+jjMKiurjZDhw41K1euNNdee60vvNDXnePhhx82o0aNavc5u/Qxp43Oo76+XuvXr1d+fr7vmNPpVH5+voqKiiyszL52796t0tJSvz5NSkpSbm6ur0+LioqUnJyscePG+drk5+fL6XRq7dq1vjbXXHON3G63r01BQYFKSkp07NixCP000aWyslKS1Lt3b0nS+vXr1dDQ4NfXOTk56tevn19fX3rppUpPT/e1KSgoUFVVlbZu3epr0/Y9Wtt0x/8GmpqatGzZMh0/flx5eXn0cRjcc889uuWWW87oD/q68+zYsUNZWVkaNGiQJk+erL1790qyTx8TXs7j8OHDampq8vslSVJ6erpKS0stqsreWvvtXH1aWlqqPn36+D0fExOj3r17+7Vp7z3afkZ34vV6NWfOHH3ta1/TJZdcIqm5H9xut5KTk/3ant7X5+vHs7WpqqrSiRMnwvHjRJ3NmzerZ8+eiouL06xZs/Taa69pxIgR9HEnW7ZsmT799FMVFhae8Rx93Tlyc3O1dOlSrVixQs8//7x2796tq6++WtXV1bbp4y53VWmgu7rnnnu0ZcsWvf/++1aX0iUNGzZMxcXFqqys1P/8z/9o6tSpWrNmjdVldSn79u3Tvffeq5UrVyo+Pt7qcrqsm266yXd/5MiRys3NVf/+/fXHP/5RCQkJFlYWOEZeziM1NVUul+uMmdZlZWXKyMiwqCp7a+23c/VpRkaGysvL/Z5vbGzU0aNH/dq09x5tP6O7mD17tv7yl79o9erV6tu3r+94RkaG6uvrVVFR4df+9L4+Xz+erY3H47HN/+w6yu12a8iQIRo7dqwKCws1atQo/eIXv6CPO9H69etVXl6uMWPGKCYmRjExMVqzZo2eeeYZxcTEKD09nb4Og+TkZF100UXauXOnbf4+E17Ow+12a+zYsVq1apXvmNfr1apVq5SXl2dhZfY1cOBAZWRk+PVpVVWV1q5d6+vTvLw8VVRUaP369b42b7/9trxer3Jzc31t3n33XTU0NPjarFy5UsOGDVNKSkqEfhprGWM0e/Zsvfbaa3r77bc1cOBAv+fHjh2r2NhYv74uKSnR3r17/fp68+bNfmFx5cqV8ng8GjFihK9N2/dobdOd/xvwer2qq6ujjzvRhAkTtHnzZhUXF/tu48aN0+TJk3336evOV1NTo127dikzM9M+f587ZdpvF7ds2TITFxdnli5darZt22ZmzpxpkpOT/WZaw191dbXZsGGD2bBhg5FknnrqKbNhwwbz5ZdfGmOal0onJyeb119/3WzatMncdttt7S6Vvuyyy8zatWvN+++/b4YOHeq3VLqiosKkp6eb7373u2bLli1m2bJlJjExsVstlf7e975nkpKSzDvvvOO37LG2ttbXZtasWaZfv37m7bffNp988onJy8szeXl5vudblz3eeOONpri42KxYscKkpaW1u+zxvvvuM9u3bzfPPfdct1paOn/+fLNmzRqze/dus2nTJjN//nzjcDjM3/72N2MMfRxObVcbGUNfd4Z58+aZd955x+zevdt88MEHJj8/36Smppry8nJjjD36mPASoGeffdb069fPuN1uc8UVV5iPPvrI6pKi2urVq42kM25Tp041xjQvl/7hD39o0tPTTVxcnJkwYYIpKSnxe48jR46Yu+66y/Ts2dN4PB4zffp0U11d7ddm48aN5qqrrjJxcXHmwgsvNI8//nikfsSo0F4fSzIvvPCCr82JEyfM97//fZOSkmISExPN7bffbg4ePOj3Pnv27DE33XSTSUhIMKmpqWbevHmmoaHBr83q1avN6NGjjdvtNoMGDfL7jK7uX/7lX0z//v2N2+02aWlpZsKECb7gYgx9HE6nhxf6uuPuvPNOk5mZadxut7nwwgvNnXfeaXbu3Ol73g597DDGmM4ZwwEAAAg/5rwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABbIbwAAABb+f/rhweuQiJmFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.4878953780352275\n"
     ]
    }
   ],
   "source": [
    "mu = -1\n",
    "gamma = 1\n",
    "P = {\n",
    "    0:{0: (0, 0, True)}, #state:{action: (reward, next state, terminal)}\n",
    "    1:{0: (mu, 0, True),\n",
    "       1: (mu, 0, True),\n",
    "       2: (mu, 0, True),\n",
    "       3: (mu, 0, True),\n",
    "       4: (mu, 0, True),\n",
    "       5: (mu, 0, True),\n",
    "       6: (mu, 0, True),\n",
    "       7: (mu, 0, True)}, \n",
    "    2:{0: (0, 1, False),\n",
    "       1: (0, 0, True)}\n",
    "}\n",
    "pref_list_under, maximal_value_a_under, maximal_value_b_under, Qa_under, Qb_under = train(5000)\n",
    "plt.plot(pref_list_under, label = \"under\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(pref_list_under[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89278fb9",
   "metadata": {},
   "source": [
    "# mu = 1 mu = -1\n",
    "# 0.8109077770750113 -1.422531091043924 epoch 1000\n",
    "# 0.7683828158670879 -1.506782806816227 epoch 3000\n",
    "# 0.7652876995079114 -1.4878953780352275 epoch 5000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "hr_LcGeRD9jf",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hr_LcGeRD9jf",
    "outputId": "674eb553-fbd4-4477-90ac-c7c3d07c60a2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.46981791223903413"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(pref_list[1000:])-mu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7867f77e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "Wlg7vfSx9lKz",
   "metadata": {
    "id": "Wlg7vfSx9lKz"
   },
   "source": [
    "## Grid Game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0a715d69",
   "metadata": {
    "id": "0a715d69"
   },
   "outputs": [],
   "source": [
    "#r1, r2, next_state, terminal: 0: up, 1: down, 2: right, 3: left\n",
    "# Map = [\n",
    "#         \"CCG\",012\n",
    "#         \"CCC\",345\n",
    "#         \"SCC\",678\n",
    "#     ]\n",
    "P = {\n",
    "    0:{\n",
    "        0:(-12, 10, 0, False),\n",
    "        1:(-12, 10, 3, False),\n",
    "        2:(-12, 10, 1, False),\n",
    "        3:(-12, 10, 0, False)\n",
    "    },\n",
    "    1:{\n",
    "        0:(-12, 10, 1, False),\n",
    "        1:(-12, 10, 4, False),\n",
    "        2:(-12, 10, 2, False),\n",
    "        3:(-12, 10, 0, False)\n",
    "    },\n",
    "    2:{                        #Goal\n",
    "        0:(5, 5, 9, True),\n",
    "        1:(5, 5, 9, True),\n",
    "        2:(5, 5, 9, True),\n",
    "        3:(5, 5, 9, True)\n",
    "    },\n",
    "    3:{\n",
    "        0:(-12, 10, 0, False),\n",
    "        1:(-12, 10, 6, False),\n",
    "        2:(-12, 10, 4, False),\n",
    "        3:(-12, 10, 3, False)\n",
    "    },\n",
    "    4:{\n",
    "        0:(-12, 10, 1, False),\n",
    "        1:(-12, 10, 7, False),\n",
    "        2:(-12, 10, 5, False),\n",
    "        3:(-12, 10, 3, False)\n",
    "    },\n",
    "    5:{\n",
    "        0:(-12, 10, 2, False),\n",
    "        1:(-12, 10, 8, False),\n",
    "        2:(-12, 10, 5, False),\n",
    "        3:(-12, 10, 4, False)\n",
    "    },\n",
    "    6:{                        #Starting state\n",
    "        0:(-12, 10, 3, False),\n",
    "        1:(-12, 10, 6, False),\n",
    "        2:(-12, 10, 7, False),\n",
    "        3:(-12, 10, 6, False)\n",
    "    },\n",
    "    7:{\n",
    "        0:(-12, 10, 4, False),\n",
    "        1:(-12, 10, 7, False),\n",
    "        2:(-12, 10, 8, False),\n",
    "        3:(-12, 10, 6, False)\n",
    "    },\n",
    "    8:{\n",
    "        0:(-12, 10, 5, False),\n",
    "        1:(-12, 10, 8, False),\n",
    "        2:(-12, 10, 8, False),\n",
    "        3:(-12, 10, 7, False)\n",
    "    },\n",
    "    9:{                       #End of episode state\n",
    "        0:(0, 0, 9, True),\n",
    "        1:(0, 0, 9, True),\n",
    "        2:(0, 0, 9, True),\n",
    "        3:(0, 0, 9, True)\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ccbc82d1",
   "metadata": {
    "id": "ccbc82d1"
   },
   "outputs": [],
   "source": [
    "def rewardnstate(a, s):\n",
    "    r1, r2, state, terminal = P[s][a]\n",
    "    reward = np.random.choice([r1, r2])\n",
    "    return reward, state, terminal\n",
    "\n",
    "def act_functn(Q, state, state_visits):\n",
    "    #eps = 0.1\n",
    "    #eps = 1/np.power(state_visits[state], 0.2)\n",
    "    eps = 1/np.sqrt(state_visits[state])\n",
    "    state_visits[state] += 1\n",
    "    if np.random.choice([True, False], p=[1-eps, eps]):\n",
    "        action = np.argmax(Q[state])\n",
    "    else:\n",
    "        action = np.random.choice(nA)\n",
    "    return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cf459f29",
   "metadata": {
    "id": "cf459f29"
   },
   "outputs": [],
   "source": [
    "np.random.seed(2)\n",
    "count_matrix_a = np.ones([nS, nA])\n",
    "count_matrix_b = np.ones([nS, nA])\n",
    "state_visits = np.ones(nS)\n",
    "reward_list = []\n",
    "maximal_value_a = []\n",
    "maximal_value_b = []\n",
    "cumsum = 0\n",
    "count = 0\n",
    "Qa = np.random.rand(nS+1, nA) #Random between -0.5 and 0.5\n",
    "Qb = np.random.rand(nS+1, nA)\n",
    "Qa[nS], Qb[nS] = 0, 0\n",
    "for i in range(10000):\n",
    "    state = 6\n",
    "    terminal = True if state==2 else False\n",
    "    while(not terminal):\n",
    "        action = act_functn((Qa + Qb)/2, state, state_visits)\n",
    "        reward, next_state, terminal = rewardnstate(action, state)\n",
    "        cumsum += reward\n",
    "        count += 1\n",
    "        \n",
    "        \n",
    "        if np.random.choice([True, False]):#Qa\n",
    "            alpha_a = (1/np.power(count_matrix_a[state, action], p))\n",
    "            action_star = np.argmax(Qa[next_state])\n",
    "            action_l = np.argmin(Qa[next_state])\n",
    "            diff = np.abs(Qb[next_state, action_star]-Qb[next_state, action_l])\n",
    "            beta_a = diff/(1000+diff)\n",
    "            delta = reward + gamma*(beta_a*Qa[next_state, action_star]+(1-beta_a)*Qb[next_state, action_star])-Qa[state, action]\n",
    "            Qa[state, action] += alpha_a*(delta)\n",
    "            count_matrix_a[state, action] += 1\n",
    "        else:\n",
    "            alpha_b = (1/np.power(count_matrix_b[state, action], p))\n",
    "            action_star = np.argmax(Qb[next_state])#Qb\n",
    "            action_l = np.argmin(Qb[next_state])\n",
    "            diff = np.abs(Qa[next_state, action_star]-Qa[next_state, action_l])\n",
    "            beta_b = diff/(1000+diff)\n",
    "            delta = reward + gamma*(beta_b*Qb[next_state, action_star]+(1-beta_b)*Qa[next_state, action_star])-Qb[state, action]\n",
    "            Qb[state, action] += alpha_b*(delta)\n",
    "            count_matrix_b[state, action] += 1\n",
    "\n",
    "        \n",
    "        state = next_state\n",
    "\n",
    "    reward_list.append(cumsum/count)\n",
    "    maximal_value_a.append(np.max(Qa[6]))\n",
    "    maximal_value_b.append(np.max(Qb[6]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50dfe479",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "id": "50dfe479",
    "outputId": "6b716d34-c8ee-4f3b-f288-b242463dc240"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD4CAYAAAAqw8chAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAY9ElEQVR4nO3deXRc5Znn8e+jKu2LZVmyLbybxcYmAYwScDAJoVkNCd1pphsmbAkNc0jnQGZ6moRJCJ2e5HS6eyYNTDJJaKDTnUmANGQIQxIIOzgJBhkcNtsYbMALtmTLsrWWannmj7oWkqpkSbbkulf8PufUcdV737p6Xl35p6u33rpl7o6IiIRbUaELEBGRkSmsRUQiQGEtIhIBCmsRkQhQWIuIREB8InZaX1/v8+fPn4hdi4hMSmvWrNnl7g3DbZ+QsJ4/fz7Nzc0TsWsRkUnJzN450HZNg4iIRIDCWkQkAhTWIiIRoLAWEYkAhbWISASMOqzNLGZmL5nZQxNZkIiI5BrLmfX1wLqJKkRERIY3qrA2s9nA+cAdE1nMM2+0sqWteyK/hIhIJI32zPoW4AYgM1wHM7vGzJrNrLm1tfWgirn8ruf5+D8+eVDPFRGZzEYMazO7AGhx9zUH6ufut7t7k7s3NTQM+47JEemzEEREco3mzPpU4NNm9jZwD3CGmf2fCa1KREQGGTGs3f1Gd5/t7vOBi4En3P3SCa9MRET6aZ21iEgEjOmqe+7+FPDUhFQiIiLD0pm1iEgEKKxFRCJAYS0iEgEKaxGRCFBYi4hEgMJaRCQCFNYiIhGgsBYRiQCFtYhIBCisRUQiQGEtIhIBCmsRkQhQWIuIRIDCWkQkAhTWIiIRoLAWEYkAhbWISAQorEVEIkBhLSISAQprEZEIUFiLiESAwlpEJAIU1iIiEaCwFhGJAIW1iEgEKKxFRCJAYS0iEgEKaxGRCFBYi4hEgMJaRCQCFNYiIhGgsBYRiQCFtYhIBCisRUQiQGEtIhIBCmsRkQgYMazNrMzMnjezP5jZa2b2jcNRmIiIvC8+ij4J4Ax37zSzYmCVmf3a3Z+b4NpERCQwYli7uwOdwcPi4OYTWZSIiAw2qjlrM4uZ2VqgBXjU3Vfn6XONmTWbWXNra+t41yki8oE2qrB297S7nwDMBj5qZsfl6XO7uze5e1NDQ8N41yki8oE2ptUg7t4OPAmcOzHliIhIPqNZDdJgZrXB/XLgLGD9RBcmIiLvG81qkEbgX80sRjbcf+buD01sWSIiMtBoVoO8DJx4GGoREZFh6B2MIiIRELqwnlpRXOgSRERCZzRz1odNfVUpZy+dUegyRERCJ3Rn1q73RoqI5AhVWJsVugIRkXAKVVi3diTY3ZkodBkiIqETqrAG+M3rOwtdgohI6IQurEVEJJfCWkQkAhTWIiIRoLAWEYkAhbWISAQorEVEIkBhLSISAQprEZEIUFiLiESAwlpEJAIU1iIiEaCwFhGJAIW1iEgEKKxFRCJAYS0iEgGh+gzGhfWVLJ01pdBliIiEjs6sRUQiIHRh7frEXBGRHOEKa31grohIXuEKaxERySt0Ya1JEBGRXKEKa82CiIjkF6qwFhGR/BTWIiIREL6w1qS1iEiOUIW1mWatRUTyCVVYi4hIfqELa9c8iIhIjlCFtSZBRETyGzGszWyOmT1pZq+b2Wtmdv3hKExERN43mkukpoC/cvcXzawaWGNmj7r76xNRkK7jJCKSa8Qza3d/z91fDO53AOuAWRNRjBaDiIjkN6Y5azObD5wIrM6z7Rozazaz5tbW1vGpTkREgDGEtZlVAfcDX3L3fUO3u/vt7t7k7k0NDQ0HXZCmQUREco0qrM2smGxQ/8Tdfz5RxZjWg4iI5DWa1SAG3Amsc/fvTHxJIiIy1GjOrE8FLgPOMLO1wW3lRBWkN8WIiOQacemeu6/iML1fRatBRETyC9U7GEVEJL/QhbVWg4iI5ApdWIuISC6FtYhIBIQurDULIiKSK1RhrU+KERHJL1RhLSIi+YUurLUaREQkV6jCWpMgIiL5hSqsRUQkvxCGteZBRESGClVYazGIiEh+oQprERHJL3RhrdUgIiK5QhXWmgYREckvVGEtIiL5hS6sNQsiIpIrVGGtD8wVEckvVGEtIiL5hS6sXctBRERyhCqstRpERCS/UIW1iIjkp7AWEYmA0IW1ZqxFRHKFKqw1ZS0ikl+owlpERPILXVhr5Z6ISK5whbXW7omI5BWusBYRkbxCF9aaBRERyRWqsNYkiIhIfqEKaxERyS90Ya0LOYmI5ApVWGsxiIhIfqEKaxERyW/EsDazu8ysxcxePRwFiYhIrtGcWf8IOHeC6wC0GkREZDgjhrW7PwO0HYZaRERkGOM2Z21m15hZs5k1t7a2HvR+tBhERCTXuIW1u9/u7k3u3tTQ0HBQ+zAtBxERyUurQUREIiB0Ye26OoiISI7RLN27G/g9sMjMtprZVRNVjCZBRETyi4/Uwd0vORyFiIjI8MI3DaJZEBGRHKEKay0GERHJL1RhLSIi+YUurDUNIiKSK1RhbVoPIiKSV6jCWkRE8gtdWOtNMSIiucIV1poFERHJK1xhLSIieYUurLUaREQkV6jCWrMgIiL5hSqsRUQkv9CFtWZBRERyhSqsdW0QEZH8QhXWIiKSn8JaRCQCwhfWmrQWEckRqrDWhZxERPILVViLiEh+oQtrXchJRCRXqMJaS/dERPILVViLiEh+oQtrXchJRCRXqMJa0yAiIvmFKqxFRCS/0IW1ZkFERHKFKqz1phgRkfxCFdZh1JVI4UNe9UylM/yseQvJdIad+3oLVJmIfJDEC13AUEODsVC27unmvFufpaM3BcAb3zyPkngRqzbu4tI7VwNww30vAzCjppTvX3oSy+ZOLVi9IjK5hSqsw7Qa5PzbVvUHNcCSrz/M6YsaeGxdS07fnfsSfOZ//477r13OSfPqDmeZIqGQSKUpiRVhI/wndnfauvoAqCkvpjeZJpOBKRXFAKQzTqwoREEQJu4+7reTGhvds0ums7fm5uxtYNvNN7u7uw/ou2nuIv+T761yv/rqwX23bXN/8MHBbT/8Yfb5A9suuCDbdsEFg9vds/0Htj34YHa/A9uuvtrd3bs/dHx/266aaT7vyw/5P516yaC+Pb9f7e888vSgtn869RI/6r/9ctCYfNmy7NcfMqbuze8e1jH5smXvtzU2ZttuvvmgjtNwYzrcx2kyjWnfZVd6e1efJ44/ob+tc9p0/+4TG/2lz103qG/f6ud999O/HdSW/NpNvnVPt6dn5o6p58rPD+r75BMv+Yvf/bdBbXdf9VX/639fO6ht7fEr/Mz/+ZT/7tjlg9ov+v5v/Zb/8FeD2j7/pzf5iut+PKjtgY+s9OvuftG3LDi2v62lui7v/6fzr7jFP3PVbYPafnzOlX7+bc/47in17z//mKX+9IYW33PplYP6bvrDG7733vsL8rOXaWz0nr6UJ7920yH97AHN+fJ0/82yfcZXU1OTNzc3j/l5l925ms5Eiv/7hVPHvaaR/OMj66ktL+Gy5fP4yLceo6M3xTc+vZQrPjaf5zbt5uLbnwPgriubOGPxjEHP3d7ew8e+/cSgtlMW1nH1aQtZcXQ9pfEYAH/Y0s6F3/ttf5+asjhrbjqLlbc+y8aWTl77xjlUlMTo6kvzm9d28CcnzhrxTEUOXSKV5t3d3cRjRby8tZ2pFSXc9vhGmt/Zw7xpFZy9ZAZrt7RTFByLE+bU0tqRoLaihK5E9q+v3V19LGyoZOWHGjlhTm3/vnfs7aWiNEZ1aZznNrWxc18v5SUxfrF2G3u6krzb1s229p4x1VtkkDnAf9vimHHinKnMqaugrSvBMxt3kT7AE+JFxoyaMpLpDOUlMcriMSpLY/QkM8yoKaWqNE4ileHtXV1sbOmkqjROfVUJR9SWc8yMakrjRZSXxGjpSLBhRweNU8rY1ZmgtSNBXzrD3u4kPck09VWlfHRBHcfPrsWBjt4kJfEiuhIptrf3MrWihHjM2N7eQ8adZNpJpjO0dfXR2pHgvb2jf32orLiIqtJiSuNF1FYUU19VSm1FMQ1VpSw/chr7epP0pTJkHBLJND3JDG1dCaZXl1FWEqO1I8Hbu7po7UhQXRZnQUMl8SKjvTvJ7s4+upNptrR1U1Yco6M3ydY92WNYX1VK89fOHHWdA5nZGndvGna7whrOveUZ1u/oyGl/+9vnj2k/uzoTNH3zsZz2/3r2Mfzg6U10JlJ5njWytV8/i9qKEpLpDO3dSRqqS4ftu3NfL1PKiykrjh3U14qy7e09zKwpY/veHm564FW6Emle276XvnSGWbXlHD+nlr/51FI27+7imTdaebetmxfebuO99l5SB0q/wLTKEnYHf8IfyBFTynCgN5lmT3cyb5/q0jjTqkqYO62SIxsq6U6kKYkXcczManbu7aUnmaa6LM6njz+C9p4kz29uoyxexMtb97K1vYfTjqonHitiW3s3teUlJDPZMD22sYan32ilvTtJIpmmqizOiqMaOHlhHam088q2vcSK4MxjZ1BbUUJ9VQnlxTHisZHXGngQoCXxsa9LcPdDPvFIZ5w93X1s2NHBtj09dCZS1JQXk0pnSGWcdMbZvKuLqRUltHb20t6dxMzo6E3S1tXH9vZednUmRvW1zGBmTRkN1aX0JtO82dJJxqG2opja8mJiRcb8aZUkM051aZx50yqoLI1TWRLjylMXHNT4PtBh7e7s2NfLVT9q5svnLeYTxzQM2r6nq497XtjC3z+8Pue5/3DRh/mzpjlj/pp9qQy/fvU9rr9nbd7tx82q4f99cQW/e2s3n71j9Zj3v9/SI2q4/9qPAbD4poeH7bfiqHq+8+fHM726rL+tvbuPzkSKI6aUA/BmayfFsSK++NMXyTj86roVo/qP1ZVI8ZWfv8LcunLOWTqTx9a18OjrOzliShn1VaU8saGFqRXFfPncxZy+aHreuUh35+FXd1AcK+KTi/P3GU5vMhvGW9p6uP/FrTy7cdeg7VPKi9nbkz8sB2qoLuU/fXwhiVSGV7bu5c8/OodFM6qpLIlz34tbOXlBHUdNr6KsOEYqneGd4Iyqpy/NkQ2VJFIZ+tIZ3tjRwa2Pb6S1I8H6HR1Ul8a58MQjmFGdPdOMx4pYcVQ9RUXGyQvqPpC/UAuttSNB89ttTKkoZmZNGeUlMUrjMUqDX0CptNPVl6KhupTiAb/AevrSmDGhxyxyYd3Rm+KBvxxbWK95p42HXn6P3mSGTMa56VNLOO7mR3L6/eDSkyiJG5//UTPX/9HR3Pr4xv5t86dVcMO5iykvjrF9bw+fPXnemOsfqKM3yZa2Hlbe9mx/22WnzOO///Fxefv3pTJ096WoLivuD6xMxvnxc+9w84OvHVItB+MTxzSw/MhpPL2hlVsvOYEtbT184SdrsmdsqQwL6yvZtKtrTPtcPLOabXt6+MSiBt5t6+atlk66+tKD+iyaUc1ly+fx9q4u7li1merSOHVVJXxy0XSmVZZw9tKZ3LlqEz9r3pp3/+UlMRbPrOHPmmZzYrA6x93p6ktz+9Nv0dqZ4KR5dZx2dD1lxTGmlBcf/DdJZBxFKqwvv+t59vUkxxTWx938yEFPLwzU/LUzqa8afnqhkHbu6+WuVZu5bPk8ZtWWc8tjG/n9pt08v7kNgL9YsYBrTz+SmvLi/rOBdMZJpNJc829rWPXmrgPtvt/UiuLgl9XIc4MN1aW0diRYNreWuspSHlu3k8tOmcdXzz+WPd19vPRuO0saa3hqQwvf/OU6YkVGIpXJ2c+cunJOPbKetq4+fvP6zrxfa960Ct7Z3Z1321lLZrBs7lT+dNkspteU5e0jEgXjEtZmdi5wKxAD7nD3bx+o/+EK6+6+FEu+nnsGPVB9VSmnL2rgvjW5Z2IAt158AheeMGvMtYbBO7u7SKado6ZXHbBfKp1h7ZZ2FjZU0ZtM0zil7IDTHO7Onas2c/szm6irLGH9jg4+s2wWpx5Zz/z6Co6ZUU112djPSFs6eikrjvHoazsxgxPnTmVBfeWgPr3JNL97axct+xKc/+FGHKgpK2b1pt089UYrfakM8ZhxwzmLtcRLJpVDDmsziwFvAGcBW4EXgEvc/fXhnnMoYb23J8kvRhHWT25oobw41r9K428vXMrXf/H+dMHQOecNOzo455ZnALjxvMVcdNJs6ipLtNpCREJhpLAezZtiPgq86e6bgh3eA1wIDBvWB6vIyK48HMH29h4+9y8v9D/+m08t4fLl87l8+fz+lR1DXxxcNLOazX+3kr50pn8pnYhIVIwmrGcBWwY83gqcPLSTmV0DXAMwd+7cgyqmyCxn/Wgm4+zu6hu0XG3outQVR9f333/4Sx8fdv9mpqAWkUgatws5ufvt7t7k7k0NDQ0jPyFfMQaZIWfWf/frdXzkW4/xwEvb6E2m2duTZPOQVQhHTa8+6LpFRKJgNGfW24CBcwqzg7Zxt2NfL69t3zeo7Z+f3QzAl+5dC/dm2z48e0r/9rEu8xMRiaLRhPULwNFmtoBsSF8M/MeJKObVbfuCf/cyc0r27a/5vLx1LzD2dxiKiETViGHt7ikz+yLwCNmle3e5+4S+S2P9jg4u+F+rJvJLiIhEyqgukeruvwJ+NcG19Bs6b53PkQ2VI/YREZksQvlJMUMvVFRbUcyzN3xyUNtfn7PocJYkIlJQofrwgf3Khiyve+mmszAz3v72+SRSaR5f18I5S2cWqDoRkcMvlGG9/5MkZtSU8vD1Hx/0LsPSeIyVH2osVGkiIgURymmQv/zpiwB8648/xNTKkgJXIyJSeKEM6/2m14TzKngiIodbqMN6pi55KSIChDysp4X0+tIiIodbqMNa1ysWEckKdViLiEhWqML66tMO7lOBRUQmu1CF9bWnH9V//5fXrShgJSIi4RKqsB44Qz2RH/kuIhI1oQrrgW+AKS4KVWkiIgUVukT858ubOO3oeubUlRe6FBGR0AjdtUHOWjKDs5bMKHQZIiKhErozaxERyaWwFhGJAIW1iEgEKKxFRCJAYS0iEgEKaxGRCFBYi4hEgMJaRCQCzN3Hf6dmrcA7B/n0emDXOJYTBRrz5PdBGy9ozGM1z90bhts4IWF9KMys2d2bCl3H4aQxT34ftPGCxjzeNA0iIhIBCmsRkQgIY1jfXugCCkBjnvw+aOMFjXlchW7OWkREcoXxzFpERIZQWIuIREBowtrMzjWzDWb2ppl9pdD1HAozm2NmT5rZ62b2mpldH7TXmdmjZrYx+Hdq0G5mdlsw9pfNbNmAfV0R9N9oZlcUakyjYWYxM3vJzB4KHi8ws9XBuO41s5KgvTR4/Gawff6AfdwYtG8ws3MKM5LRM7NaM7vPzNab2TozWz6Zj7OZ/efgZ/pVM7vbzMom43E2s7vMrMXMXh3QNm7H1cxOMrNXgufcZmbGSNy94DcgBrwFLARKgD8ASwpd1yGMpxFYFtyvBt4AlgD/AHwlaP8K8PfB/ZXAr8l+ZvApwOqgvQ7YFPw7Nbg/tdDjO8C4/wvwU+Ch4PHPgIuD+z8Arg3ufwH4QXD/YuDe4P6S4NiXAguCn4lYocc1wpj/FfiL4H4JUDtZjzMwC9gMlA84vldOxuMMfBxYBrw6oG3cjivwfNDXgueeN2JNhf6mBIUvBx4Z8PhG4MZC1zWO4/sFcBawAWgM2hqBDcH9HwKXDOi/Idh+CfDDAe2D+oXpBswGHgfOAB4Kfgh3AfGhxxh4BFge3I8H/WzocR/YL4w3YEoQXjakfVIe5yCstwThEw+O8zmT9TgD84eE9bgc12Db+gHtg/oNdwvLNMj+H4L9tgZtkRf86XcisBqY4e7vBZt2APs/bHK48Ufp+3ILcAOQCR5PA9rdPRU8Hlh7/7iC7XuD/lEaL2TPCluBfwmmf+4ws0om6XF2923A/wDeBd4je9zWMPmP837jdVxnBfeHth9QWMJ6UjKzKuB+4Evuvm/gNs/+Sp0U6ybN7AKgxd3XFLqWwyxO9k/l77v7iUAX2T+P+02y4zwVuJDsL6kjgErg3IIWVSCFOK5hCettwJwBj2cHbZFlZsVkg/on7v7zoHmnmTUG2xuBlqB9uPFH5ftyKvBpM3sbuIfsVMitQK2ZxYM+A2vvH1ewfQqwm+iMd7+twFZ3Xx08vo9seE/W43wmsNndW909Cfyc7LGf7Md5v/E6rtuC+0PbDygsYf0CcHTwqnIJ2RcjHixwTQcteGX3TmCdu39nwKYHgf2vCF9Bdi57f/vlwavKpwB7gz+3HgHONrOpwVnN2UFbqLj7je4+293nkz12T7j7Z4EngYuCbkPHu//7cFHQ34P2i4NVBAuAo8m+EBNK7r4D2GJmi4KmPwJeZ5IeZ7LTH6eYWUXwM75/vJP6OA8wLsc12LbPzE4Jvo+XD9jX8Ao9iT9gkn0l2VUTbwFfLXQ9hziWFWT/RHoZWBvcVpKdr3sc2Ag8BtQF/Q34XjD2V4CmAfv6PPBmcPtcocc2irGfzvurQRaS/U/4JvDvQGnQXhY8fjPYvnDA878afB82MIpXyAt9A04AmoNj/QDZV/0n7XEGvgGsB14Ffkx2RcekO87A3WTn5ZNk/4K6ajyPK9AUfA/fAr7LkBep8930dnMRkQgIyzSIiIgcgMJaRCQCFNYiIhGgsBYRiQCFtYhIBCisRUQiQGEtIhIB/x8nVFg1e2MYhQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "props = {\n",
    "    \"width\": 1,\n",
    "    \"headwidth\": 10\n",
    "}\n",
    "plt.plot(reward_list)#reward per step\n",
    "plt.axhline(y = 0.2, color = \"r\", linestyle = \"--\", linewidth = 1)\n",
    "#plt.annotate('y = %s' % round(reward_list[-1], 4), xy=(8000, reward_list[-1]), xytext=(9000, -0.2), arrowprops = props)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6864198c",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "id": "6864198c",
    "outputId": "f46a717f-1923-409f-c0d2-ce4f970e256e"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgcVb3/8fe3Z8ky2ffJAkmQLWyBjGERBVljAPPDBzXoBVyDIve63Z83yFW5evWqF3DDhxCVn8sjm0okLBoSBEGFyCRAEkJCFgJkIZkQsieT6env74+uCT09PcksXV3dXZ/X8/Qz1adqqr6VmvS365xT55i7IyIi8ZaIOgAREYmekoGIiCgZiIiIkoGIiKBkICIiQGXUAXTFkCFDfOzYsVGHISJSUhYtWrTV3YfmWleSyWDs2LHU19dHHYaISEkxs1fbW6dqIhERUTIQERElAxERQclARERQMhAREZQMREQEJQMRESFmyWBNw2527W+KOgwRkaITq2TwnYdf4r769VGHISJSdGKVDMYNqSGV0mQ+IiLZ8pIMzOxOM9tiZssyygaZ2XwzWxX8HNjO714TbLPKzK7JRzwiItI5+boz+CUwJatsJvCYux8NPBa8b8XMBgHfAE4HJgPfaC9piIhIePKSDNz9SWBbVvE04FfB8q+A/5PjVy8G5rv7Nnd/C5hP26QiIiIhC7PNYLi7bwqW3wCG59hmFPB6xvv1QVkbZjbDzOrNrL6hoSG/kYqIxFxBGpDd3YFutdy6+2x3r3P3uqFDcw7HLSIiXRRmMthsZrUAwc8tObbZAIzJeD86KBMRkQIKMxnMBVp6B10DPJBjm3nARWY2MGg4vigoExGRAspX19K7gaeBY81svZl9EvgucKGZrQIuCN5jZnVm9nMAd98GfAt4Nnh9MygLjXevtkpEpCzlZdpLd7+ynVXn59i2HvhUxvs7gTvzEcfhmBXiKCIipSdWTyCLiEhuSgYiIqJkICIiSgYiIoKSgYiIoGQgIiIoGYiICDFMBq5nzkRE2ohVMjA9dSYiklOskoGIiOSmZCAiIkoGIiKiZCAiIigZiIgISgYiIkLIycDMjjWz5zNeO83sC1nbnGtmOzK2+XqYMYmISFt5mdymPe6+EpgIYGYVpOc3npNj06fc/dIwYzkYUyEOIiJSYgpZTXQ+sMbdXy3gMVvRI2ciIrkVMhlMB+5uZ92ZZvaCmf3JzE7ItYGZzTCzejOrb2hoCC9KEZEYKkgyMLNq4P3A73KsXgwc6e6nAD8B/phrH+4+293r3L1u6NCh4QUrIhJDhbozeB+w2N03Z69w953uvjtYfgSoMrMhBYpLREQoXDK4knaqiMxshAUjyJnZ5CCmNwsUl4iIEHJvIgAzqwEuBK7NKPsMgLvPAq4APmtmSWAfMN1dA02LiBRS6MnA3fcAg7PKZmUs3wbcFnYcIiLSPj2BLCIi8UsGqoASEWkrXslAT52JiOQUr2QgIiI5KRmIiIiSgYiIxC0ZOLjGLRURaSP05wyKyR1PrgXgunPfEXEkIiLFJV53BiIikpOSgYiIKBmIiIiSgYiIoGQgIiIoGYiICEoGIiJCAZKBma0zs6Vm9ryZ1edYb2b2YzNbbWZLzOy0sGMSEZHWCvXQ2XvdfWs7694HHB28TgduD36KiEiBFEM10TTg1572DDDAzGqjDkpEJE4KkQwceNTMFpnZjBzrRwGvZ7xfH5SJiEiBFKKa6Gx332Bmw4D5ZrbC3Z/s7E6CRDID4Igjjsh3jCIisRb6nYG7bwh+bgHmAJOzNtkAjMl4Pzooy97PbHevc/e6oUOHhhWuiEgshZoMzKzGzPq2LAMXAcuyNpsLXB30KjoD2OHum8KMS0REWgu7mmg4MMfMWo51l7v/2cw+A+Dus4BHgKnAamAv8PGQYxIRkSyhJgN3XwuckqN8VsayA58LMw4RETm0YuhaKiIiEVMyEBERJQMREVEyEBERlAxERAQlAxERQclARERQMhAREZQMREQEJQMREUHJQEREUDIQERGUDEREBCUDERFByUBERIhZMjjrqMFRhyAiUpRCSwZmNsbMHjez5Wb2opl9Psc255rZDjN7Pnh9Pax4ACoSFubuRURKVpgznSWBL7v74mAe5EVmNt/dl2dt95S7XxpiHAcF02+KiEiW0O4M3H2Tuy8OlncBLwGjwjpeRygViIjkVpA2AzMbC5wKLMyx+kwze8HM/mRmJxxiHzPMrN7M6hsaGkKKVEQknkJPBmbWB/gD8AV335m1ejFwpLufAvwE+GN7+3H32e5e5+51Q4cO7WIsXfo1EZGyF2oyMLMq0ongt+5+f/Z6d9/p7ruD5UeAKjMbElo8Ye1YRKTEhdmbyIBfAC+5+63tbDMi2A4zmxzE82ZYMYmISG5h9iZ6F3AVsNTMng/KvgocAeDus4ArgM+aWRLYB0x3dw8xJgB2Nybp0yPMUxcRKS2hfSK6+984TM2Mu98G3BZWDNlaupae+I15rPvuJYU6rIhI0YvVE8hqMxARyS1WyUBERHKLVTLQE8giIrnFLBlEHYGISHGKVTK47JSRUYcgIlKUYtW/8v2njGTc4Bpm3r8k6lBERIpKrO4MIF1VFP6TDCIipSV2yQBAuUBEpLXYJQM1IouItBW7ZABQgBEvRERKSuySgek5ZBGRNuKXDNSALCLSRuySgYiItBW7ZGAGrv5EXXbezU9Qv25b1GGISJ7FLxlgqibqAndn7MyHWbt1D1fMejrqcEQkzwoxB/IUM1tpZqvNbGaO9T3M7N5g/UIzG3vYnW7cmP6K3/JatCj9yiy76ab0tiNHvl02aRJm8IXf3dx6240b4cEHW5fNnt0S4Nuvyy5Ll112WetySG+fWfbgg23jnDEjve2kSW+XjQyGyLjppi6fE5Ded4jnZIkE569eyLBdb7Lue5eWxTmV43XSOemcDnlOh2BhdrM0swrgZeBCYD3wLHCluy/P2OY64GR3/4yZTQcud/cPH2q/dXV1Xl9f36WYXt68i+t+u5gFXzqnS79f7h5esolkKsXn73me333mTD6YdRfw+L+fy3tvfoI7rprE31Zt5e5/vsbgPtU8PfN8EgmLKGoR6QgzW+TudbnWhT020WRgtbuvDQK5B5gGLM/YZhpwU7D8e+A2M7Owpr809JzBoXzursUHl7MTwYpvTaFnVQUA1/5m0cHyzTsbeWbtm5z1jiGFCVJE8i7saqJRwOsZ79cHZTm3cfcksAMYnL0jM5thZvVmVt/Q0NDlgMziOxzFgy9s5Jm1b/LnZW/kXL9zf9PB5YljBrRat+67lxxMBIu/duHB8v96/wkA/Pgvq3jntxfw9Jo3Wb5xZ75DF5GQlcyope4+G5gN6WqiiMMpSf9693Ot3n+objT/MeU4nn99O+cfP5zFr77FmEG9mPeF99CrqoJNO/bTv1cVNT1a/5kMqqluNYf0ky838NiKLQBc+bNnADTHtEiJCTsZbADGZLwfHZTl2ma9mVUC/YE3wwspnrcGzam2J31f/Xruq18PwCUn1zJucA0njx5A7+r0n8XIAb06tO/PnnvUwWTQ4kAyRXVl7DqriZSssP+3PgscbWbjzKwamA7MzdpmLnBNsHwF8Jew2gsgvtVER331EQAG9K7Kuf7hJZu47fHVPLxkU6f3XTd20MHl1d9+H8eN6Mtzr73VtUBFJBKh3hm4e9LMrgfmARXAne7+opl9E6h397nAL4DfmNlqYBvphBGauPd3ef7rF7G/qZmUOxO+Pg+A2VdNYkbQIPzwv53dpf3O+pfTWPnGbiorEhw3oi+vbN3D6ePbNP2ISJEKvc3A3R8BHskq+3rG8n7gg2HHkXX8Qh4ucnsPJAG4Z8YZAAcbgld8awqNyRT9er79Z3DCyP5dOsaUE2uZcmJ6ecyg3mze2diNiEWk0EqmATlfzCx21UTv+f7jAJyR9U29Z1XFwcSQzwbfAb2reX3b3rztT0TCF7sWvvRzBlFHEb5P/7qesTMfpjHZzNbdBwp67Nr+PVm6YUdBjyki3RO7ZBAHr725l/nLNwNw7H/+Geh6W0BXvOsdQ3j+9e00JpsLdkwR6Z7YJYN0b6LyvjXYtGNfm7IJtf0Kdvz+vaoY0qe64HckItJ18UsGMRi19MOz0w9+/e4zZwIw9/p3YVbYflQjB/Ri3dY9BT2miHRdDBuQy7vN4KePrwbg0pNreefYQfx95nmM6uDDY/k0emBvtu5WjyKRUhG7O4Ny8I0HlnHGdx5rU755537+d95KAG7+4CkAkSQCgME11Tz32vZIji0inadkUIJ+9fSrvLFzf5vy7/5pRXr9JyYf7DIalbOOGswv/7Eu0hhEpONilwzS1USlXU90xvj08A/b9rRuoJ3zXHrYp3OOGVrwmLJdOGE4PasS7G5MRh2KiHRA7JJBqWtqTvHM2vQcxKd9az6bs+4QfnLlqVGE1YaZcfSwvsxrZ7hsESkusUsGpf4E8h1/XQPAmEHptoDTg7aD7XvTdwnnHTcsmsByeO9xw3hNTyKLlIT4JQNKuzfRy5t3M7xfD/79omMBOGlUf97ac4CGXemeO9lzD0Rp1ICevKiJbkRKQvySQYk/dDb3hY18+t3juWjCCK6cfARLN+zg1G/N58IfPMmwvj2iDq+VE0f15/GVW9jfpCeRRYpd7JJBKWv59n/U0D70qq7gfz5wUqv1W3YVV7/+CbX96NuzkrUNevhMpNjFLhmU6hPIW3c38s5vLwDSdfG53HHVpEKGdFhmxvEj+rXp9SQixSd+yaBEZzqr++8FOcs/cvoRQHroiYtPGFHIkDpkWL8erHhD7QYixS6U1kYz+1/gMuAAsAb4uLu3eRzVzNYBu4BmIOnudWHEU8o279zPw0s2cdZRg/nHmje579ozW63/zuUn8Z3LT2rnt6N30qj+LNNw1iJFL6w7g/nAie5+MvAycMMhtn2vu08sVCIotd5Ep3/nMb750HL+seZNvjntBCaPG3T4Xyoi5x8/nAeXbCr5B/1Eyl0oycDdH3X3lkdPnwFGh3GcLjHY05jk/sXro46kjcdXbGHp+re/RT+7blur9QN7Vxc6pG4bN6SG5pTnHD5DRIpHIdoMPgH8qZ11DjxqZovMbMahdmJmM8ys3szqGxoauhzM4JoeXHPWWH7yl9Vd3kdYPv7LZ7nyZ8/w7YeXs/i1t7jut4tbrZ84ZkBEkXXPhNp+6lEkUuS63GZgZguAXC2WN7r7A8E2NwJJ4Lft7OZsd99gZsOA+Wa2wt2fzLWhu88GZgPU1dV1uc6hImF8+J1jmPdicQ6TsLsxyc+eeoWfPfXKwbIfTZ/ItImjIoyqe0b078niV9/iXe8YEnUoItKOLicDd7/gUOvN7GPApcD53k6FsbtvCH5uMbM5wGQgZzLIt1Kpw776zCNLOhEAnDl+sKqJRIpcKNVEZjYF+ArwfnfPOTiNmdWYWd+WZeAiYFkY8bQ5diEO0kmp1NvJac51ZwEwfkgNXw6GnShlA2uqW7WFiEjxCWsgm9uAHqSrfgCecffPmNlI4OfuPhUYDswJ1lcCd7n7n0OKp41iuy9YtjH9Ybnuu5cA6ecHPnvOUfTvVRVlWHlxfG1fXtyoZCBSzEJJBu7+jnbKNwJTg+W1wClhHP9wCjwdcIdMD+YtblHMzw501lFD+9DU7Lh7wediFpGOid0TyC2KpcnA3WlMNrP3QPkO5tazqoIDzSk27VC7gUixKp7xjgvIiqjVYNwNjxxcPr+I5iLIt3cM68P85Zu55qyxUYciIjnE986g6FoN4KcfPS3qEEIz9aRaDVgnUsRimQyKtdo66knswzS4ppol69sMTyUiRSKW1URQPG0GQ/pU88jn382wvj2jDiVU44fWMPvJ3VGHISLtiOWdQbFwd7bvbWJAr9Ibc6izTh49gA3b97Fh+76oQxGRHGKbDIrhzmDD9n0kU051Zflfhv69qhjYu4oHnt8QdSgikkP5fwoVqX0Hmln5xi5OHt0/6lAKZtrEUfxt1daowxCRHGKZDIqhAfkbc5fxyV/VM6AEh6XuqvccM0Q9ikSKVGwbkKNWkUjn4Sdf7vpw3KXm6GF9eW3bXj796/q87XNInx78zwfK52ltkajEMhmYGRu27+OWR1dGNhDcgN7pMYdOGhWfaqLRA3tx+79MYn9Tfp62bk45n7/nOSUDkTyIZTJo8ZO/rI4kGWzeuZ/bn1jDteeM54b3HV/w40fFzDjnmKF521+yOUWqCDoCiJSDeLYZRHz807/zGAD1696KOJLSVpEwmpUNRPIilsmgWHz63eOjDqGkmRlmpTNRkUgxi2UyKIbeRP17VTHlxFyzhkpnJEx3ByL5EFoyMLObzGyDmT0fvKa2s90UM1tpZqvNbGZY8RQDd+cPi9YD8NzXLow4mvJQYUaz7gxEui3sBuQfuPvN7a00swrgp8CFwHrgWTOb6+7Lwwwqcwjrhl2NDO3bI8zDHXTBrX9lTcMeABKJIrg9KQPpaqKooxApfVFXE00GVrv7Wnc/ANwDTCtkANNnP12Q46zesutgIpD8USOySH6EnQyuN7MlZnanmQ3MsX4U8HrG+/VBWRtmNsPM6s2svqGhew9qZbYZFOoD+oJbnzy4fPMHI5ntsyypmkgkP7qVDMxsgZkty/GaBtwOHAVMBDYBt3TnWO4+293r3L1u6ND89VUvlGvfk+45VF2R4IpJoyOOpnyYgaeijkKk9HWrzcDdL+jIdmb2M+ChHKs2AGMy3o8OykKV/UVyznPrufzU8D6gm1POXQtfA2DeF98T2nHiqCKhOwORfAitAdnMat19U/D2cmBZjs2eBY42s3Gkk8B04CNhxdQilfXh8cV7XwgtGfx99VZe27aXXY1JZl81iXFDakI5TlxVJBL8293PdWoY8LOOGsyn9IyHSCth9ib6vplNBBxYB1wLYGYjgZ+7+1R3T5rZ9cA8oAK4091fDDEmoG0yCNNHf77w4PIZRw0u2HHj4hfX1LF1d2OHt1++cSePr9xScsng9W17uWLWP0iYMff6swvWA07iI7Rk4O5XtVO+EZia8f4R4JGw4sglqt4n/XpWRXLccnbKmAGd2r5XVQV/X1N6cyps2bWfYX17smt/Ezv2HVAykLyLumtpJJIFTAbH1/Zr9VOilUgYqRJscN7flKJPj0qqKxMF/fuV+IjlqKW57gweXpJu3qiqMM4/fjgVeXgobO+BJC9t2sl/XnJ8yVVLlKuoG5ybmlP8+ulXaWruXEZas2U3PasSVCQSeq5CQhHLZHDEoN4AjOjXk1EDe7Ho1bd4eOlGAP66soEH//Vsxg/t0+3j/GFxumOUupIWjyjGMtrdmOTGOUtpak6xa3+SVZt3M23iyE7tY1BNNRefMIIfPvaykoGEIpbJoGdVBeu+e0nOdefd8kTexsj/2h/THajiNLVlsatIWEE7EABs2r6PhWu38bVLJwAwbkgNE0Z2rdrwJ4+vVjKQUMQyGRyKoSGRy1lFBHcGB5pTDKyp5pKTa7u9rwqLrgOElLdYNiAfipmR67/ab55ex62Pruz0/m7/6GndjknyJ4qxjJqaneqK/AxMqLGYJCy6M8iSaGcUzB89toqtuw/wpU5MkzmgdxWTxw3KY3TSXVFUEzU1p6iqyM/3roqEsedAkj2NSR5Zuol/vrKt0/NzDKrpwcz3HZeXeKR8KBlkMQzPcW/Q2c+P5pSzfW8T/Xvp2YJiUpEwNm3fz5fve6Fgx2zY3Zi3ZDBqQG+uv+s5APYeaObac8YzvhNPtbvDV+csVTKQNpQMsrQ3Pn5nv0uu2rKL6soElXn6EJD8GDu4N/99+YkcSBb2YYNjhvfNy35u+dAp3PKh7o16e8OcpaRSrjk1pBUlgxy6W43QsKuRKT98iklH5hq1W6JUWZFg2sSco6THRmXCSKac6iJKBrc+upL1b+0L/TiXnTKS9x43LPTjlCIlgyxm1u2Zsxa8tBmAPY3JPEQkkl+ViQTJVIrqIuo/8st/rOMrU46jV1VFaMf4+5qtLHhps5JBO5QMsuTju9IN9y8FYMUbu/KwN5H8arkzKCb7kymumDSaniEmg2Z3Fq7dFtr+S52SQZZ22wz07IGUiYoKI9lc+L/nZDtDcDjpHlc9OjEMeVf0rKpg4/Z9/KMTAxUO6dMjb+09xU7JIEvC2ulN1MHfz0waf/2/5+YnKJE8qkwkeGjJxryOoptIGBefMJwelbm/2S96dRtXzHqaRDv9YGv79cQ620e2k44NPtR//NiqDm1/IJli0479PH3D+WGGVTSUDLK0d2fQUXsPNNOrqoKXvjUlf0GJ5NFHJo9h0atv5XWfT63aSm3/nrxzbO7nahp2HeDC44cz++q6vB63M44d0Ze7Z5zR4e237m7koh88efgNy0QoycDM7gVans4aAGx394k5tlsH7AKagaS7R/eXEjA63ptod2OSWU+soSljTOR9B5rp10s5VopXZx6c7KgP3/H0IUdibUw20yPE9oAwVCUSnR5dtpSF8qnl7h9uWTazW4Adh9j8ve5ePLONtDMcRS5rG3bz+0XrueassQfLBvSCr196QiihiRSrqooEDbsauf2JNTmrWZdt2EFNdWl9SaqMqG0lKqFeHUtXAn4IOC/M4+RTeqC6tuW5ypqandoBPfnsuUeFHpdIMausMP6+eiv1697iohNGtFl/xKAazjlmaASRdV1lhZEsxZmQuijsVP1uYLO7t9di48CjZubAHe4+u70dmdkMYAbAEUcckfdAW+zc38QfFq/nla17WpXv2NfUZtum5hRVieLpqy0SlcqEsbsxyUmj+5fNUBeViQRNzY67h964XQy6nAzMbAHQ9isA3OjuDwTLVwJ3H2I3Z7v7BjMbBsw3sxXunrPFJkgUswHq6upCu3db27CHtQ172N/U3GZd9tSVyWanMk+jUYqUsspEgj2NzYzoVz5jcVUkjB6VCc7+3uOdHgwwLBNq+4XWCN/lZODuFxxqvZlVAh8AJh1iHxuCn1vMbA4wGYi8+f7Iwb259UOt27sn1PZj4/b9rcqaUimNPSQCVFcmWPjKm4w9pXMzuBW7v888j30H2n4xjEp1iM9ihFlNdAGwwt3X51ppZjVAwt13BcsXAd8MMZ4Oa68vdKY1Dbv58WOrGFzTowARiRS3L114DJecXMtJo/pHHUpeDekTn//fYSaD6WRVEZnZSODn7j4VGA7MCeriKoG73P3PIcbTYR25JXw5GGriq1PLo35UpDvGDqlhbCeG0pbiE1oycPeP5SjbCEwNltcC3RuLNyTt3RlkdplLppyRA3oxfmifQoUlIhIaVXjnkGtk3+zeBMlUisoiGgJYRKQ7lAxysA6MXZpsdiqUDESkTCgZ5NCRNoPmlOsZAxEpG/o0y6HdNoOMpxuSKadCzxiISJlQMsghVy7ILNqxt4ntew+ozUBEykZpjRxVANMmjmRC1pPGmZLNKd757QUM6F3FtedoTCIRKQ9KBll+NP3UQ65PphwM/nnjIR/AFhEpKaom6qTmlOfseioiUsqUDDop5U5FsYxaJSKSJ0oGHdTy+Z9Kped7FREpJ0oGndTsethMRMqPkkEnpdw7NKqpiEgpUTLoBHcnlVIyEJHyo2TQQS0f/+lqokhDERHJO32sdVJzSr2JRKT8KBl0knvb4axFREpdt5KBmX3QzF40s5SZ1WWtu8HMVpvZSjO7uJ3fH2dmC4Pt7jWz6u7EE7Zn1m7jlkdXqjeRiJSd7g5HsYz0pPd3ZBaa2QTS016eAIwEFpjZMe6ePbP094AfuPs9ZjYL+CRwezdjCsXFJ444+HzBtImjIo5GRCS/upUM3P0lyFltMg24x90bgVfMbDUwGXi6ZQNL/9J5wEeCol8BN1GkyaC2fy+uPnNs1GGIiIQirDaDUcDrGe/XB2WZBgPb3T15iG0OMrMZZlZvZvUNDQ15DVZEJO4Oe2dgZguAETlW3ejuD+Q/pNzcfTYwG6Curs4Ps7mIiHTCYZOBu3dlrOYNwJiM96ODskxvAgPMrDK4O8i1jYiIFEBY1URzgelm1sPMxgFHA//M3MDdHXgcuCIougYo2J2GiIi8rbtdSy83s/XAmcDDZjYPwN1fBO4DlgN/Bj7X0pPIzB4xs5HBLv4D+FLQwDwY+EV34hERka4x99Krfq+rq/P6+vqowxARKSlmtsjd63Kt0xPIIiKiZCAiIiVaTWRmDcCrXfz1IcDWPIZTCnTO8RC3c47b+UL3z/lIdx+aa0VJJoPuMLP69urMypXOOR7ids5xO18I95xVTSQiIkoGIiISz2QwO+oAIqBzjoe4nXPczhdCPOfYtRmIiEhbcbwzEBGRLEoGIiISn2RgZlOCKThXm9nMqOPpDjMbY2aPm9nyYNrRzwflg8xsvpmtCn4ODMrNzH4cnPsSMzstY1/XBNuvMrNrojqnjjKzCjN7zsweCt7nnDo1GCTx3qB8oZmNzdjHYadkLRZmNsDMfm9mK8zsJTM7s9yvs5l9Mfi7XmZmd5tZz3K7zmZ2p5ltMbNlGWV5u65mNsnMlga/82OzDkzc7u5l/wIqgDXAeKAaeAGYEHVc3TifWuC0YLkv8DIwAfg+MDMonwl8L1ieCvwJMOAMYGFQPghYG/wcGCwPjPr8DnPuXwLuAh4K3t8HTA+WZwGfDZavA2YFy9OBe4PlCcH17wGMC/4uKqI+r0Oc76+ATwXL1cCAcr7OpCe4egXolXF9P1Zu1xl4D3AasCyjLG/XlfQo0WcEv/Mn4H2HjSnqf5QC/cOfCczLeH8DcEPUceXx/B4ALgRWArVBWS2wMli+A7gyY/uVwforgTsyylttV2wv0nNePEZ6utSHgj/0rUBl9nUG5gFnBsuVwXaWfe0ztyu2F9A/+GC0rPKyvc68PUvioOC6PQRcXI7XGRiblQzycl2DdSsyyltt194rLtVEHZmGsyQFt8WnAguB4e6+KVj1BjA8WG7v/Evt3+WHwFeAVPD+UFOnHjy3YP2OYPtSOudxQAPw/4KqsZ+bWQ1lfJ3dfQNwM/AasIn0dVtEeV/nFvm6rqOC5ezyQ4pLMihLZtYH+APwBXffmbnO018JyqbfsJldCmxx90VRx1JAlaSrEm5391OBPcCxkLQAAAHgSURBVKSrDw4qw+s8EJhGOhGOBGqAKZEGFYEormtckkFHpuEsKWZWRToR/Nbd7w+KN5tZbbC+FtgSlLd3/qX07/Iu4P1mtg64h3RV0Y8Ipk4NtsmM/+C5Bev7k55qtZTOeT2w3t0XBu9/Tzo5lPN1vgB4xd0b3L0JuJ/0tS/n69wiX9d1Q7CcXX5IcUkGzwJHBz0Sqkk3NM2NOKYuC3oG/AJ4yd1vzVg1l/T0odB6GtG5wNVBr4QzgB3B7eg84CIzGxh8I7soKCs67n6Du49297Gkr99f3P2jtD91aua/xRXB9k4HpmQtFu7+BvC6mR0bFJ1PevbAsr3OpKuHzjCz3sHfecs5l+11zpCX6xqs22lmZwT/hlfTkSmFo25EKWBjzVTSvW7WADdGHU83z+Vs0reQS4Dng9dU0nWljwGrgAXAoGB7A34anPtSoC5jX58AVgevj0d9bh08/3N5uzfReNL/yVcDvwN6BOU9g/erg/XjM37/xuDfYiUd6GUR8blOBOqDa/1H0r1Gyvo6A/8FrACWAb8h3SOorK4zcDfpNpEm0neAn8zndQXqgn+/NcBtZHVCyPXScBQiIhKbaiIRETkEJQMREVEyEBERJQMREUHJQEREUDIQERGUDEREBPj/FlEpXUdycsEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "props = {\n",
    "    \"width\": 1,\n",
    "    \"headwidth\": 10\n",
    "}\n",
    "\n",
    "plt.plot(maximal_value_a, linewidth=1)\n",
    "plt.axhline(y = 0.36, color = \"r\", linestyle = \"--\", linewidth = 1)\n",
    "#plt.annotate('y = %s' % round(maximal_value_a[-1], 4), xy=(8000, maximal_value_a[-1]), xytext=(9000, -2), arrowprops = props)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bb3e8727",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "id": "bb3e8727",
    "outputId": "adabc075-7930-453a-9851-553806cd765f"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgU1bnH8e87C6vsjIiA7C7EBWWCqLiCCKjXxJtEzI3LjYpGEzXexKBRYzRRkxs1cUmURHI1V71qElxBRdw1osOmrLLKDsMiw+IMs7z3j65pume6Z+uGman+fZ6nH6pOnT51qmt4u/rUqXPM3RERkcyQ1dgVEBGR/UdBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBchq7AjXp2rWr9+nTp7GrISLSbMycOXOzu+cl296kg36fPn0oKCho7GqIiDQbZvZFTdvVvCMikkEU9EVEMoiCvohIBlHQFxHJIAr6IiIZREFfRCSDKOiLiGSQUAb9dz8vZPXW3Y1dDRGRJqdJP5zVUBdP+hiAlfec3cg1ERFpWkJ5pS8iIokp6IuIZJB6BX0zm2Rmm8xsXkxaZzObZmZLgn87JXnvJUGeJWZ2SaoVFxGR+qvvlf7/AKOrpE0Aprv7QGB6sB7HzDoDvwCOB4YCv0j25SAiIvtOvYK+u78LbK2SfB7weLD8OPCNBG89C5jm7lvdfRswjepfHiIiso+lo02/m7uvD5Y3AN0S5OkBrI5ZXxOkVWNm482swMwKCgsL01A9ERGplNYbue7ugKdYxkR3z3f3/Ly8pPMAiIhIA6Qj6G80s+4Awb+bEuRZC/SKWe8ZpImIyH6UjqD/IlDZG+cS4IUEeV4DRplZp+AG7qggTURE9qP6dtl8GvgXcJiZrTGzy4B7gDPNbAkwMljHzPLN7C8A7r4VuBP4JHjdEaSJiMh+VK9hGNz9wiSbRiTIWwBcHrM+CZhUr9qJiEha6YlcEZEMoqAvIpJBFPRFRDKIgr6ISAZR0BcRySAK+iIiGURBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBFPRFRDKIgr6ISAZR0BcRySAK+iIiGURBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBFPRFRDJIykHfzA4zszkxryIzu75KntPMbHtMnttS3a+IiNRfvSZGT8TdFwODAcwsG1gLTE6Q9T13PyfV/YmISMOlu3lnBLDM3b9Ic7kiIpIG6Q7644Cnk2w7wczmmtlUM/tasgLMbLyZFZhZQWFhYZqrJyKS2dIW9M2sBfBvwHMJNs8Cerv7McCDwPPJynH3ie6e7+75eXl56aqeiIiQ3iv9McAsd99YdYO7F7n7zmB5CpBrZl3TuG8REamDdAb9C0nStGNmB5mZBctDg/1uSeO+RUSkDlLuvQNgZm2BM4ErY9KuAnD3R4BvAT8wszLgK2Ccu3s69i0iInWXlqDv7ruALlXSHolZfgh4KB37EhGRhtMTuSIiGURBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBFPRFRDKIgr6ISAZR0BcRySAK+iIiGURBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBFPRFRDKIgr6ISAZR0BcRySAK+iIiGURBX0Qkgyjoi4hkkLQFfTNbaWafmdkcMytIsN3M7AEzW2pmn5rZcenat4iI1E1aJkaPcbq7b06ybQwwMHgdD/wp+FdERPaT/dm8cx7whEd8BHQ0s+77cf8iIhkvnUHfgdfNbKaZjU+wvQewOmZ9TZAWx8zGm1mBmRUUFhamsXoiIpLOoD/c3Y8j0oxzjZmd0pBC3H2iu+e7e35eXl4aqyciImkL+u6+Nvh3EzAZGFoly1qgV8x6zyBNRET2k7QEfTNra2btKpeBUcC8KtleBC4OevEMA7a7+/p07F9EROomXb13ugGTzayyzKfc/VUzuwrA3R8BpgBjgaXAbuA/07RvERGpo7QEfXdfDhyTIP2RmGUHrknH/kREpGFC/URu5HtGREQqhTroP/b+isaugohIkxLqoD979ZeNXQURkSYl1EE/O3JjWUREAuEO+lkK+iIisUId9LN0pS8iEifkQb+xayAi0rSEPOgr6ouIxAp10F++eWdjV0FEpEkJddAvq9DDWSIisUId9NW4IyISL9RBX9f5IiLxQh3027fKbewqiIg0KaEO+icN6NLYVRARaVJCHfTVZVNEJF6og76IiMRT0BcRySAK+iIiGURBX0Qkg6Qc9M2sl5m9ZWYLzGy+mV2XIM9pZrbdzOYEr9tS3a+IiNRfOiZGLwP+y91nmVk7YKaZTXP3BVXyvefu56RhfyIi0kApX+m7+3p3nxUs7wAWAj1SLVdERNIvrW36ZtYHOBaYkWDzCWY218ymmtnXaihjvJkVmFlBYWFhOqsnIpLx0hb0zewA4B/A9e5eVGXzLKC3ux8DPAg8n6wcd5/o7vnunp+Xl5dSnVyD74iIxElL0DezXCIB/0l3/2fV7e5e5O47g+UpQK6ZdU3HvkVEpO7S0XvHgMeAhe5+X5I8BwX5MLOhwX63pLrv2uu2r/cgItK8pKP3zknARcBnZjYnSLsZOATA3R8BvgX8wMzKgK+Ace5qfBER2d9SDvru/j61zFfi7g8BD6W6r/rS14qISDw9kSsikkEU9EVEMoiCvohIBlHQFxHJIAr6IiIZREFfRCSDhDroO+qzKSISK9RBX0RE4oU66FvNz4yJiGScUAd9ERGJF+qgrzZ9EZF44Q76ivkiInFCHfRFRCReqIO+LvRFROKFO+gr6ouIxAl30Ne1vohInFAH/cIdJY1dBRGRJiXUQf+vH6xs7CqIiDQpTTvor1sXmd288jVzZuQVm3b77ZG8Bx8cTXvpf64D4K5XH4zPu24dvPRSfNrEiZH3x6ade24k7dxz49Mhkj827aWXqtdz/PhI3iFD9qYdfHAk7fbbG3RMDBkSSRs/XsekY9Ix6ZiSH1MtLB3zk5vZaOAPQDbwF3e/p8r2lsATwBBgC3CBu6+srdz8/HwvKCiod336THglurzynrPr/X4RkebKzGa6e36y7Slf6ZtZNvAwMAYYBFxoZoOqZLsM2ObuA4D7gd+kul8REam/dDTvDAWWuvtyd98D/B9wXpU85wGPB8t/B0aYVf4WSr/c7L1F/+zvn+6r3YiINDvpCPo9gNUx62uCtIR53L0M2A50SVSYmY03swIzKygsLGxQhd644VQAWmRnMWf1lw0qQxpuy84SNu9sWj2n7p66kD4TXmHTjuLGropIo2pyN3LdfaK757t7fl5eXoPK6N2lLSvvOZsRRxyovvr7mbsz5FdvkP+rN+gz4RWKikvTWv7k2Wt45pNVbCwq5r0ldb8oePSd5QA8/uHKlPa/p6yCPhNeibtvFOvDZZuZvnBjtfSZX2zjB/87k+27Sxn48yn0mfAKpeUVKdVFwqmkrHyflp+ThjLWAr1i1nsGaYnyrDGzHKADkRu60oRVVDjl7uRm135tsKesgkNvmVot/ejbX2fJr8fUqYxKF0/6mI3bi3npR8NpkbP3fdt27eHHz8yNy/vp7aNo3yo3aVmXP/4JbyzcFF1v27LmP/k9ZRVc8UQBlw3vyymH5vHfry3i4beWAfDD0wfw0FtLo3krA/8HE87gqz3ljLzvnVqPbeq8DdHlF+es49+H9Kz1PRJ+ywp38ubCTfx6ykIAbjn7CC4/ud8+2VfKvXeCIP45MIJIcP8E+K67z4/Jcw1wlLtfZWbjgPPd/Tu1ld3Q3juVrn5yJks27mRa0NwjNbt58mc8NWMVs289k9Ytsjn81leB5D2giopLKS93OrVtwbkPvs9na7dHt/145KHc/8bnAPTp0oa3f3p6nepQXFoe3W+slfecnfTq+vNfjaFFThY7S8o48hevAdC+VQ6vXn8KJ97zZtJ9Tb3uZCa9v4LnZq7h1etPpnObFgy9a3qd6pku6l2WOYqKS2mTm01OzAVQSVk5h91S/e8dYPldY8nKqv+tz9p676R8pe/uZWb2Q+A1Il02J7n7fDO7Ayhw9xeBx4C/mdlSYCswLtX9SnotK9zJUzNWAXDsndMY2rdzdNumHcUc2K4Vz89ey1el5Zx/XA+mL9zE1U/OSlredSMHsnFHMU/NWMXKLbupqPCkf8DTF27kxbnr+MO4Y7n39cUJ8yQL+AA/eW4uD1x4LO8s3tvcU1RcVmPABxjzh/eiy6N//14NOeNNufZk7nx5Af9aXv3H6qUn9uF/EjQhXT68LzeOPpzfvb6YcV/vxRn3Rn4V/GPmGl3th0hRcSlH3/56dD3LoKLKdfVFw3pz27mDuPSvH/PB0vi/ocO6tWPU17pRuKOkQQG/LtLST39fSfVK/5onZ7F4447ojV1J7sa/z+XZgjUJt931zaO4efJndSqn6pVrbLDu0rYFT10xjNfnb+AHp/UnJzuL0vIKBv68erNQMjePPZz+eQfQu0tbJvzjUwq+2Bbd1qZFNrv3VG8PffDCY7n1hXl8OOEMBt32Wq376Ngmly93R+5FjD3qIO6/YDAtc7KZ+cVWvnZwB1rlZkfzbtu1h7IKp8KdvANa1vk/6pzVX/KNhz8A4NtDenLB13uR36dzLe+SpqLyImbWqm2c/8cPUy5vxd1jSVeHxtqu9BX0M9zOkjL+tWwLVzzR8M+50jWn9+enZx0el/bS3HX86OnZ9S5r/i/PIifbKN5TwTF37L1yWnTn6Ligm+gXwMp7zmbago1c8UQBl57Yh9v/7WvV8vzypfnRYTpW3D2WJ2es4jdTF3HVaf0Z9/VeDPnVG3z/pL7cdm7VR07SJ1HdrzylH9ePPJTWLbITvEPSwd25/pk5XHVqf47o3j4u3Z1av7j/+PZSfvtq4l+k0Tz/cRz/+9EXLCvcycaiErq1b8m0G05l1ZbdnPPg+9F8+b07MfHifDq3bZHaQcXI7KD/1CwWrS9i+n+dlr5KhUzVwHPZ8L489v4KIPFP09vPHcTtLy0A4Psn9eXmsYdTuLOEr/aU07tLW7IT/IepqWmm0sEdWrFue6Q75bVnDOCGUYdFt63euptObVtwQIKbsDtLyrj4sRnMWhXpmtsqN4tFd46pdX8AC9cXcVi3dvvsZ3Rt3J2+N01JuG3ubaPo0Cb5DWqpn/IKZ8wf3uXzjTtrzXvlqf248azDWbNtN727tI2muzuTZ6/lhmfnVnvPOUd35+VP1zPu672459+PrrH8svIKPl6xlRP6d0nb1X0sBX0F/aS27drDsXdOi64v/tVodpeUR9v0nxk/LC4offPYHtx/wWAKd5TQrlVO3BV3TWYs38IFEz/ioe8eyw+fmk1OllHhHveF8tr1p9CpTS4tc7IbFOx27ylj7bav6N2lbVyPn+agrLyCM+9/lxWbd0XTxp/Sj5vHHoG775PAEFbbd5fSvnUOX+4uZfW23cxd/SW3vjC/9jfW0zeP7cHk2ZFOik3tZnxGB/0fPjWLBeuLeFNBP6G7pyzk0Xcj/dfv/MaRXDSsd7U8lVfpDe1JUJvi0nJWb93NwG7t0l52c+Lu/HPWWk4+tCtDfx3fg+iDCWfQo2PrRqrZvrenrCLhF7W7s3TTTnp3acuXu/dwYPtWCd9fWl6BAQPqcG+obYtsjuvdiXu/fQybdpTQtmUOSzbu4OSBeSxYX8Tzs9fyt4++qLGMs4/qzsP/cVydjq0x7PPeO9I8lJZXUOFOy5xsfvrcXJ6bufem7Y/OGJAw4APMvvVMssz2WRNIq9zsjA/4AGYW7cXTrlUOO4rLotvO/+MHfPCzM1i5ZRcDDoz/rCoqHDMoKatg/roihvTuVKf91dSbqj5KysrJzcpqUFnz123n7Af2tm8/cOGxvL+kEMO4buTAhL2vrjy1X/RBu/p4+UfDObJHh7i0yi+Rvl0jTThDendiSO9OXDdyILlZWRzQKof+N0+JDHR5y5l8vGILowYd1GjNgekS6iv9Hz09m/lrt/PmT05LX6Waqcor9somllhLfz0mru+wNK5kzypU9dxVJ/DtR/6VcNvvvn0Mfbq0obTceX3BhoRzS1x7xgAeeHMp2VnGT0YdxuUn963XQ3R/fnd59GGiWCcN6ML/XnZ8wmapDduLeXvxJib8s269weprwR1nce6D73P2Ud3j7gtlkoxu3sn0oP/u54VcPOnjWvM1tTZJifwyKy4t5/LHC5ixYut+2+8j3zuO0Ud2rzHP3NVf8sD0JUxftClpnvsvOIZH31nOog07ommXD+/LX4JOArGuGzGQP0xfkrCca88YwIXHH0KbFjmMm/gRC9cX8a0hPfl78Ev1vRtPp2ObXMornI5t0tcDpjnL6KB/7dOzmZfBQb+mXjN3fuNIbn1+HvN/eVatQxNI43F33luymYsnfczMW0byyDvL6N2lLbc8Py8u36mH5nFi/y7cPXURZw7qxrQF1cf/+dnow/nNq4uA+N5SiVw7YiDXjRgY1xsr9tmCWDeNOZy7py5i+ICuvL90c52P7Z9Xn8hxh1RvjvLgJn+inmBSu4wP+p+t3c5bCvpxHr1oCGd97aD9XBtJp+kLN3LZ4wUsu2tswuC49suvaNcqh5wsIzc7q8Zmm8r2/e/++SM+XFb3IbEe//5QTj20+qCIif7u7v32MfzXc5GujvpluW/pRm4GStQm/I8fnMC//ynS/nvmEd0ao1qSRiOO6FZj8KxPb5/KG5NPXTGMRRuKah2SYsEdZ9GmRfLQ8cGEMxj/RAGPXjSEnp3aRNM13ETTEOqgbxb5qZhppny2Pm59aN/ODOndmWV3jaW0vKLZ9z6Qfefwg9qz8p6zWbJxB2fe/y4A3dq3ZGNRSZ1v+Pfo2JpXrj15X1dVGijUQT9TxT4xGHtjLjvLyM7S4/1Su4Hd2qkZJqRCHfQNQjmFyorNu3hjwUa6d2zF5FlreezSrwORMUG6tdv7AMvTVwzjhP4JJygTkQwV6qAfViPve4fymDEMFqwr4rCD2lUbBEoBX0SqCnXQNzPC2KRfXmUUtLEPvMc1p/dvpNqISHOixzCbmZ0lZQnTK6f0q/TxzSP2R3VEpJkJddCPtOmH61J/UswTjbGzW1V6/PtDGTWoW9LBqUQks4U66DcHq7fups+EVxhx79vVulomUvkgzu8vGMyzV57A94YdErf91EPzmHhx0ucyRCTDhTvoG026Tb+iwjn5t28BsKxwF1c/OStp802lB9+MjFEy5qjIE7W/+sZR+7aSIhIqKQV9M/tvM1tkZp+a2WQz65gk30oz+8zM5phZ6vPyhcCc1V/S7+bqsyYd+Yvkc7j++d3lFJdWANAyZ29/+7d/cho3jj6MRXeOTn9FRSRUUr3SnwYc6e5HA58DN9WQ93R3H1zTmBDpZjTd3juJBq6qlGjsEndPOIwtQJ+ubbn6tAF1nslKRDJXSkHf3V9398r2iI+AjB9cY09ZBSPufZvpC6uPcpjMLWcfwcPfjZ+J55TfvsUD05ewsaiYPhNe4e3PC6PbJl99YtrqKyKZJZ1t+t8Hks1X5sDrZjbTzMbXVIiZjTezAjMrKCwsrClrrRpjatHCnSUsK9zFTcEkEW8t3sQvXtg7DO6G7cWUllfEvefyk/tx9tHdOTHmYapVW3dz37TPOf6uyNR5//nXTwB45HtDODbBcLQiInVR68NZZvYGkGgc3p+7+wtBnp8DZcCTSYoZ7u5rzexAYJqZLXL3dxNldPeJwESIDK1ch2NoUnYHN2I37ShhR3FpNFhfc8aA6Nynt50zKJr/k5+PjC6f0K9LrUPbtm8d6ufpRGQfqzWCuPvImrab2aXAOcAITzKkpbuvDf7dZGaTgaFAwqCfTsb+H2UzdmKKo25/Pbp87oN75wK94+UFANx6ziDy2rWMpl9xSj+O79eF7zyaeAo8gBP7d01ndUUkw6R02Whmo4EbgVPdfXeSPG2BLHffESyPAu5IZb9N2fgnEndO2lhUUi2tfav4j79VbjZD+3Zm4R2j2bKrhE9WbuXHz8yt9j4RkYZKtU3/IaAdkSabOWb2CICZHWxmlf0RuwHvm9lc4GPgFXevfdbnNDDb/6NslpRV1J4pcEjnNgnTW7fIpmenNpx79MEAHBQ8XfvU5cenXkERyWgpXem7+4Ak6euAscHycuCYVPbTHLXIyWJPgi+AljlZ0S+G4/vVPApmTnYWK+4eC0QGjxMRSVWon8htzH76iQJ+j46tGXNk5J54h9a5dSrHzBTwRSRt1BUkTZ6asYrBvTrSsU0uZxx+IMMHdOWGZ+fyl4vzOfnQrixcv4NB3dtTXFbO83PW8d3jD6m9UBGRNAt10I+06e/7S/2Vm3dx8+TPousdWudy/nE9Of+4vc+qDe4VGaGiRU4Wc28bxQGtQv3Ri0gTFermndo8P3st89ZuT7mc7V+Vxq0/V7Cmxvwd2uRGR8sUEdmfQn25aTWMsrllZwnXPzMHIOUJoJ+c8UXcem0jZYqINJZQB/1kTv/d26zYvCu6/tma7RzVs0ODy3u2ypX9qEHdGlyWiMi+FPLmHavWor9he3FcwAdYvS3hc2V1UlxaXi3ttnMHJcgpItL4Qh70qxt29/RqaVW/BOqjcEf1J21ba4hjEWmiQh30a2rTj/Xfry2ud9nbdu3hJ8/Njc58dUF+r+i2ti0zstVMRJoBRacGuvJvM/l45dbo+rx123nvxtNpmZulyUxEpMnKgKC/b/rpr6lyH2Dixfn06Nh6n+xLRCRdwt28U2V9Xw6zrIAvIs1BqIM+xLfpl1XEB/3uHVo1uNzYcfPvOf+oBpcjIrI/hTrox45T9ug7yzj/jx/Gbb/vO4MbXPZRPfb26x83VOPoiEjzEOqgD3tb9O+euojPqgy5MKxfZ646tT8tcur/MXRr35K8di15/cenpKGWIiL7R6iDvlVr1d/r7KO7Y2Ys3bSDPWUVzFq1rV5lf1VaziGd23Bot3apVlNEZL8JddCH5DdvX/l0PQDvL90MwKvzNtSpvGc+WcW0BRspLq2gVW7oPz4RCZlQd9msz9wjG4uKa80za9U2fvaPvUMoDx+gScpFpHkJ/aWqA4feMrVaesugHf/Adq3i1muyqcoXQ+WvBBGR5iKloG9mt5vZ2mBS9DlmNjZJvtFmttjMlprZhFT2Wa/6Bf8mmrrwjRtOBeDG0YcB8b1x6uqy4X0bXDcRkcaQjuad+939d8k2mlk28DBwJrAG+MTMXnT3BWnYd60SNel3apNLr85tgL0zWrXMqX3ohC279sStn3JoXuoVFBHZj/ZH885QYKm7L3f3PcD/Aefth/0mnVD82hEDo8u52ZGPoLSi+q+BqnaXxA+j3CI79K1jIhIy6YhaPzSzT81skpl1SrC9B7A6Zn1NkJaQmY03swIzKygsLEypYrv3lFWbyhDg6J4do8s5wbSF67/c216/aEMRL81dV+19SzbtiFvvekCLlOonIrK/1Rr0zewNM5uX4HUe8CegPzAYWA/cm2qF3H2iu+e7e35eXmrNJ1VntKqUm733F0BOcLX+0FtLo2mjf/8eP3p6Nrv3xE97WHX0TI2mKSLNTa1t+u4+si4FmdmfgZcTbFoL9IpZ7xmkNZqsmGaf2Caa4tJy7pv2eXS9pLSCNjEX8xuLiunQOjf666F1CwV9EWleUu290z1m9ZvAvATZPgEGmllfM2sBjANeTGW/6RTbVfOpGauY+O7y6HpJTK+fPWUVvDZ/Y1xzURsFfRFpZlLtvfNbMxtMpDv8SuBKADM7GPiLu4919zIz+yHwGpANTHL3+SnuN22ysvZe9X9Qpd99SdneG7cbtld/eKtVHXr8iIg0JSkFfXe/KEn6OmBszPoUYEoq+0qnZMPqT1+0KW499ko/K+Y3UavcLIpLK+K+MEREmoNQD8OQzKEHHVCnfJt3lkQHVKt8wOuubx7FOcd0Z1uVPvsiIs1BxnU0v27EwDo9iAXwk2fnsqsk0oNnT3kk6Hdsk0v7Vrn07tJ2n9VRRGRfCXXQ75cXH5i/3qcTV53av87vX7e9mIeDrpyVV/p6IEtEmrNQR7BHvjckbv3G0Ycn7GY5+9Yzo8vPXXUCc27buz5vXREQE/QbMOGKiEhTEeoIdkgwvk6liorEd3A7td3bGX9A3gG0bbn3VkfVQdvqMhqniEhTFeoIVjVAJ4n5cXKyLToeD8A7n0eGgvjbR19Et4uINFeh7r1TdcA1J3nUn3Ltybw2fwPtWuVW27Z5ZwlTg5m1srNC/T0pIiEX6qBfVbL++QCDDm7PoIPbR9fbtcphR3Gk586sL/bOn3tMz/qPuy8i0lRk1GVrRU1Rv4obzzosujz+bzOjy8mGaxYRaQ4yLOjXPe9FJ/TZZ/UQEWksGRX0h/bpnNL7P5xwRppqIiLSODIm6E8Yk7iPfk2qTofYoXX1m7wiIs1JxgT9hkxiPumS/Lh19dEXkeYuY6JYbgOGT8ip8p6q6yIizY2imIhIBlHQFxHJIAr6dXTywK6NXQURkZQp6Nfip8FDWr84d1Aj10REJHWhH4bh+WtOYt7a7Q1+/zWnD+Ca0weksUYiIo0npaBvZs8AleMVdAS+dPfBCfKtBHYA5UCZu+dXzbOvDO7VkcG9Ou6v3YmINGmpTox+QeWymd0L1HRJfbq7b05lfyIikpq0NO9YZBSy7wAap0BEpAlL143ck4GN7r4kyXYHXjezmWY2vqaCzGy8mRWYWUFhYWGaqiciIlCHK30zewM4KMGmn7v7C8HyhcDTNRQz3N3XmtmBwDQzW+Tu7ybK6O4TgYkA+fn59RgXU0REalNr0Hf3kTVtN7Mc4HxgSLI87r42+HeTmU0GhgIJg76IiOw76WjeGQkscvc1iTaaWVsza1e5DIwC5qVhvyIiUk/pCPrjqNK0Y2YHm9mUYLUb8L6ZzQU+Bl5x91fTsF8REamnlHvvuPulCdLWAWOD5eXAManuR0REUmdej3lj9zczKwS+aODbuwKZ9lyAjjn8Mu14QcdcX73dPS/ZxiYd9FNhZgX788nfpkDHHH6ZdrygY043DbgmIpJBFPRFRDJImIP+xMauQCPQMYdfph0v6JjTKrRt+iIiUl2Yr/RFRKQKBX0RkQwSuqBvZqPNbLGZLTWzCY1dn1SYWS8ze8vMFpjZfDO7LkjvbGbTzGxJ8G+nIN3M7IHg2Bs8+zMAAAQhSURBVD81s+NiyrokyL/EzC5prGOqCzPLNrPZZvZysN7XzGYEx/WMmbUI0lsG60uD7X1iyrgpSF9sZmc1zpHUnZl1NLO/m9kiM1toZieE+Tyb2Y+Dv+l5Zva0mbUK43k2s0lmtsnM5sWkpe28mtkQM/sseM8DZma1VsrdQ/MCsoFlQD+gBTAXGNTY9UrheLoDxwXL7YDPgUHAb4EJQfoE4DfB8lhgKmDAMGBGkN4ZWB782ylY7tTYx1fDcd8APAW8HKw/C4wLlh8BfhAsXw08EiyPA54JlgcF574l0Df4m8hu7OOq5ZgfBy4PllsQmYkulOcZ6AGsAFrHnN9Lw3iegVOA44B5MWlpO69EhrYZFrxnKjCm1jo19oeS5g/4BOC1mPWbgJsau15pPL4XgDOBxUD3IK07sDhYfhS4MCb/4mD7hcCjMelx+ZrSC+gJTCcyIc/LwR/zZiCn6jkGXgNOCJZzgnxW9bzH5muKL6BDEAStSnooz3MQ9FcHQSwnOM9nhfU8A32qBP20nNdg26KY9Lh8yV5ha96p/GOqtCZIa/aCn7THAjOAbu6+Pti0gcigdpD8+JvT5/J74EagIljvQmTu5bJgPbbu0eMKtm8P8jen44XIVWoh8NegWesvwYi0oTzPHhlq/XfAKmA9kfM2k/Cf50rpOq89guWq6TUKW9APJTM7APgHcL27F8Vu88hXfCj63ZrZOcAmd5/Z2HXZz3KINAH8yd2PBXYR+dkfFbLz3Ak4j8iX3cFAW2B0o1aqkTTGeQ1b0F8L9IpZ7xmkNVtmlksk4D/p7v8MkjeaWfdge3dgU5Ce7Piby+dyEvBvZrYS+D8iTTx/ADpaZLIeiK979LiC7R2ALTSf4620Bljj7jOC9b8T+RII63keCaxw90J3LwX+SeTch/08V0rXeV0bLFdNr1HYgv4nwMCgF0ALIjd9XmzkOjVYcCf+MWChu98Xs+lFoPIO/iVE2vor0y8OegEMA7YHPyNfA0aZWafgKmtUkNakuPtN7t7T3fsQOXdvuvt/AG8B3wqyVT3eys/hW0F+D9LHBb0++gIDidzwapLcfQOw2swOC5JGAAsI6Xkm0qwzzMzaBH/jlccb6vMcIy3nNdhWZGbDgs/x4piykmvsmxz74KbJWCK9XJYRmce30euUwrEMJ/LT71NgTvAaS6Q9czqwBHgD6BzkN+Dh4Ng/A/Jjyvo+sDR4/WdjH1sdjv009vbe6UfkP/NS4DmgZZDeKlhfGmzvF/P+nwefw2Lq0KOhsV/AYKAgONfPE+mlEdrzDPwSWERkFr2/EemBE7rzTGSCqfVAKZFfdJel87wC+cFnuAx4iCqdARK9NAyDiEgGCVvzjoiI1EBBX0Qkgyjoi4hkEAV9EZEMoqAvIpJBFPRFRDKIgr6ISAb5f7PbJYgbvTJ6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "props = {\n",
    "    \"width\": 1,\n",
    "    \"headwidth\": 10\n",
    "}\n",
    "plt.plot(maximal_value_b)\n",
    "plt.axhline(y = 0.36, color = \"r\", linestyle = \"--\", linewidth = 1)\n",
    "#plt.annotate('y = %s' % round(maximal_value_b[-1], 4), xy=(300000, maximal_value_b[-1]), xytext=(350000, -2), arrowprops = props)\n",
    "plt.annotate\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Weighted Qlearning.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}