maze_solver_with_rat_as_agent .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import os, sys, time, datetime, json, random\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation\n",
    "from keras.optimizers import SGD , Adam, RMSprop\n",
    "from keras.layers.advanced_activations import PReLU\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "maze = np.array([\n",
    "    [ 1.,  0.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  1.,  0.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  1.,  0.,  1.,  1.,  1.,  1.],\n",
    "    [ 0.,  0.,  1.,  0.,  0.,  1.,  0.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  0.,  1.,  0.,  1.,  0.,  0.,  0.,  1.],\n",
    "    [ 1.,  1.,  0.,  1.,  0.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  1.,  1.,  0.,  0.,  0.,  0.],\n",
    "    [ 1.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  0.,  1.,  1.]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "visited_mark = 0.8  # Cell vis by rat- gray 0.8\n",
    "rat_mark = 0.5      \n",
    "LEFT = 0\n",
    "UP = 1\n",
    "RIGHT = 2\n",
    "DOWN = 3\n",
    "actions_dict = {\n",
    "    LEFT: 'left',\n",
    "    UP: 'up',\n",
    "    RIGHT: 'right',\n",
    "    DOWN: 'down',\n",
    "}\n",
    "\n",
    "num_actions = len(actions_dict)\n",
    "\n",
    "# Exploration fac\n",
    "epsilon = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Qmaze(object):\n",
    "    def __init__(self, maze, rat=(0,0)):\n",
    "        self._maze = np.array(maze)\n",
    "        nrows, ncols = self._maze.shape\n",
    "        self.target = (nrows-1, ncols-1)   # target cell where the \"cheese\" is\n",
    "        self.free_cells = [(r,c) for r in range(nrows) for c in range(ncols) if self._maze[r,c] == 1.0]\n",
    "        self.free_cells.remove(self.target)\n",
    "        if self._maze[self.target] == 0.0:\n",
    "            raise Exception(\"Invalid maze: target cell cannot be blocked!\")\n",
    "        if not rat in self.free_cells:\n",
    "            raise Exception(\"Invalid Rat Location: must sit on a free cell\")\n",
    "        self.reset(rat)\n",
    "\n",
    "    def reset(self, rat):\n",
    "        self.rat = rat\n",
    "        self.maze = np.copy(self._maze)\n",
    "        nrows, ncols = self.maze.shape\n",
    "        row, col = rat\n",
    "        self.maze[row, col] = rat_mark\n",
    "        self.state = (row, col, 'start')\n",
    "        self.min_reward = -0.5 * self.maze.size\n",
    "        self.total_reward = 0\n",
    "        self.visited = set()\n",
    "\n",
    "    def update_state(self, action):\n",
    "        nrows, ncols = self.maze.shape\n",
    "        nrow, ncol, nmode = rat_row, rat_col, mode = self.state\n",
    "\n",
    "        if self.maze[rat_row, rat_col] > 0.0:\n",
    "            self.visited.add((rat_row, rat_col))  # mark visited cell\n",
    "\n",
    "        valid_actions = self.valid_actions()\n",
    "                \n",
    "        if not valid_actions:\n",
    "            nmode = 'blocked'\n",
    "        elif action in valid_actions:\n",
    "            nmode = 'valid'\n",
    "            if action == LEFT:\n",
    "                ncol -= 1\n",
    "            elif action == UP:\n",
    "                nrow -= 1\n",
    "            if action == RIGHT:\n",
    "                ncol += 1\n",
    "            elif action == DOWN:\n",
    "                nrow += 1\n",
    "        else:                  # invalid action, no change in rat position\n",
    "            mode = 'invalid'\n",
    "\n",
    "        # new state\n",
    "        self.state = (nrow, ncol, nmode)\n",
    "\n",
    "    def get_reward(self):\n",
    "        rat_row, rat_col, mode = self.state\n",
    "        nrows, ncols = self.maze.shape\n",
    "        if rat_row == nrows-1 and rat_col == ncols-1:\n",
    "            return 1.0\n",
    "        if mode == 'blocked':\n",
    "            return self.min_reward - 1\n",
    "        if (rat_row, rat_col) in self.visited:\n",
    "            return -0.25\n",
    "        if mode == 'invalid':\n",
    "            return -0.75\n",
    "        if mode == 'valid':\n",
    "            return -0.04\n",
    "\n",
    "    def act(self, action):\n",
    "        self.update_state(action)\n",
    "        reward = self.get_reward()\n",
    "        self.total_reward += reward\n",
    "        status = self.game_status()\n",
    "        envstate = self.observe()\n",
    "        return envstate, reward, status\n",
    "\n",
    "    def observe(self):\n",
    "        canvas = self.draw_env()\n",
    "        envstate = canvas.reshape((1, -1))\n",
    "        return envstate\n",
    "\n",
    "    def draw_env(self):\n",
    "        canvas = np.copy(self.maze)\n",
    "        nrows, ncols = self.maze.shape\n",
    "        # clear all visual marks\n",
    "        for r in range(nrows):\n",
    "            for c in range(ncols):\n",
    "                if canvas[r,c] > 0.0:\n",
    "                    canvas[r,c] = 1.0\n",
    "        # draw the rat\n",
    "        row, col, valid = self.state\n",
    "        canvas[row, col] = rat_mark\n",
    "        return canvas\n",
    "\n",
    "    def game_status(self):\n",
    "        if self.total_reward < self.min_reward:\n",
    "            return 'lose'\n",
    "        rat_row, rat_col, mode = self.state\n",
    "        nrows, ncols = self.maze.shape\n",
    "        if rat_row == nrows-1 and rat_col == ncols-1:\n",
    "            return 'win'\n",
    "\n",
    "        return 'not_over'\n",
    "\n",
    "    def valid_actions(self, cell=None):\n",
    "        if cell is None:\n",
    "            row, col, mode = self.state\n",
    "        else:\n",
    "            row, col = cell\n",
    "        actions = [0, 1, 2, 3]\n",
    "        nrows, ncols = self.maze.shape\n",
    "        if row == 0:\n",
    "            actions.remove(1)\n",
    "        elif row == nrows-1:\n",
    "            actions.remove(3)\n",
    "\n",
    "        if col == 0:\n",
    "            actions.remove(0)\n",
    "        elif col == ncols-1:\n",
    "            actions.remove(2)\n",
    "\n",
    "        if row>0 and self.maze[row-1,col] == 0.0:\n",
    "            actions.remove(1)\n",
    "        if row<nrows-1 and self.maze[row+1,col] == 0.0:\n",
    "            actions.remove(3)\n",
    "\n",
    "        if col>0 and self.maze[row,col-1] == 0.0:\n",
    "            actions.remove(0)\n",
    "        if col<ncols-1 and self.maze[row,col+1] == 0.0:\n",
    "            actions.remove(2)\n",
    "\n",
    "        return actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show(qmaze):\n",
    "    plt.grid('on')\n",
    "    nrows, ncols = qmaze.maze.shape\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks(np.arange(0.5, nrows, 1))\n",
    "    ax.set_yticks(np.arange(0.5, ncols, 1))\n",
    "    ax.set_xticklabels([])\n",
    "    ax.set_yticklabels([])\n",
    "    canvas = np.copy(qmaze.maze)\n",
    "    for row,col in qmaze.visited:\n",
    "        canvas[row,col] = 0.6\n",
    "    rat_row, rat_col, _ = qmaze.state\n",
    "    canvas[rat_row, rat_col] = 0.3   # rat cell\n",
    "    canvas[nrows-1, ncols-1] = 0.9 # cheese cell\n",
    "    img = plt.imshow(canvas, interpolation='none', cmap='gray')\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "maze = [\n",
    "    [ 1.,  0.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  0.,  1.,  1.,  1.,  0.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  0.,  1.,  0.,  1.],\n",
    "    [ 1.,  1.,  1.,  0.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  0.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  0.,  1.,  0.,  0.,  0.],\n",
    "    [ 1.,  1.,  1.,  0.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  1.,  0.,  1.,  1.,  1.]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reward= -0.04\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91885\\Anaconda3\\envs\\deep\\lib\\site-packages\\matplotlib\\cbook\\__init__.py:424: MatplotlibDeprecationWarning: \n",
      "Passing one of 'on', 'true', 'off', 'false' as a boolean is deprecated; use an actual boolean (True/False) instead.\n",
      "  warn_deprecated(\"2.2\", \"Passing one of 'on', 'true', 'off', 'false' as a \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x29236a8fa90>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABb1JREFUeJzt3bFq02scx+FfDoKDLSg90MVRiHtyAenmlXgF2bwDSWfBK3AVLyC5gGRw7OZQkEIXUSeH/xmOwnE4J8lpfZvv6/NAt8D334QPbZf+RsMwFJDlj7t+AGB/woVAwoVAwoVAwoVAwoVAwoVAwoVAwoVA9/Z58dHR0XBycvKrnuUn3759q48fPzbZevr0aT148KDJ1tevX7vcar3X69aHDx/q+vp6tO11e4V7cnJSL168+P9PtYfPnz/XfD5vsvXq1auazWZNtlarVZdbrfd63ZpOpzu9zq/KEEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EGivf4j+6dOnevfu3a96lp+0/Efe3I7NZlNnZ2dNtpbLZZOdQzUahuG/XzAaPa+q51VVjx49mrx8+bLFc9Xx8XFdXl422RqPx3V0dNRk68uXL11uVVVdXV35zG5oPp/Xer2++QmSYRheV9XrqqqHDx8Ob9++vYXH2242mzU7QbJcLrs8Z9H6BMn5+bnPrBF/40Ig4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UKgvU6QPHnypNkJktVqVduuLNzmVq9Go63/FP/WLJfLZp/Z+fl5s3Mni8Xi4P4h+l4nSE5PTydv3rxp8VzdnupovXVxcdFkq6rtWZCW504eP35cp6enTbZ2PUFSwzDs/DWZTIZWlsulrVvYqqpmXy2/t8Vi0ez7WiwWzb6v741tbdHfuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBDICZI72Gp1FqTl6Yyqvj+zVltOkBzwVnV4OuPH92brZpwggY4JFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJt6o2m02NRqMmX5vNZq/rETf5mkwmd/3W8ou4HVRVV1dXdXl52WSr5T2flu9h671et9wO2sNisejynk/L97D1Xq9bbgdBx4QLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYRbVZPJpOlZkJbnTlpqfcql161dOEFyB1sXFxdNtlqeO6lqf8qlx635fF7DMDhBcohb1eG5k2Fof8qlx62/k3SCBLokXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAh0764fgH78OOXSwmq16nJrOp3u9DonSO5gq9cTJD1/Zq225vN5rddrJ0gOcas6PUHS82fWyvfGnCCBHgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAgkXAjlB0vlWq3MnVVXj8bjb9/H+/ftNtubzeb1//37rCZKt4f7TdDod1uv1jR5sV6vVqmazma0bbp2dnTXZqqpaLpfdvo/j8bjJ1rNnz3YK16/KEEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EGivEyRVNa6qVjct/qyqa1sxW633et0aD8NwvO1Fe50gaWk0Gq2HYZjaythqvfe7b/lVGQIJFwIdcrivbUVttd77rbcO9m9c4N8d8k9c4F8IFwIJFwIJFwIJFwL9BWUcjy5LKgWMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "qmaze = Qmaze(maze)\n",
    "canvas, reward, game_over = qmaze.act(DOWN)\n",
    "print(\"reward=\", reward)\n",
    "show(qmaze)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x29236aff208>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABghJREFUeJzt3TFqm3ccx+GfmkCGuGBsg5eAR2WXDmBvWYKXQG6QE2jLFeTZxCfImhNIB5CGjF5CBkNiMFniTMG8HdpCO6SW6vRvfd8+D2gTfBWJD5EX/QZd1xWQ5Zf7fgHA+oQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgR6u8+Stra1ud3f3v3otf/P9+/f69OlTk62nT5/W48ePm2x9+/atl1ut9/q69fHjx7q6uhrc9ry1wt3d3a3Xr1//+1e1hq9fv9ZkMmmydXp6WoeHh0225vN5L7da7/V1azwer/Q8X5UhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAh0Fo/iM7P8fz58yY7x8fHTX8Qfblc1tHRUZOt2WzWZGdTDbqu++cnDAavqupVVdXe3t7o9PS0xeuqm5uburi4aLI1HA5ra2urydb19XV9/vy5ydb29nbt7e012aqqury87O1n1mprMpnUYrG4+wmSruvOquqsqurg4KD78uXLT3h5t2t5gmQ2mzU9Z/Hu3bsmW8fHx/XixYsmW1VVJycnvf3MWn5zWYW/cSGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCHQxp4gOTg4qDdv3jTZurq6qrOzsyZbOzs7zU6QbG9v12Bw64/i/zSz2axuu4zxs5ycnDQ7dzKdTjfuB9E39gTJgwcP6ubmxtYdtz58+NBkq6rtWZCW506ePHlS+/v7TbbiT5Ds7OyUrbtvtToJUtX2LEjLcyfT6bRevnzZZGtV/saFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQGudINnf3x+9ffu2xeuq6+vrZucsWm+dn5832Wp5OqOq359Zq61VT5BU13UrP0ajUdfKbDbr7VZVNXlMp9Nm/64//2227uaPxm5t0VdlCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCTcqloulzUYDJo8lsvlWtcj7vIYjUb3/dbyH3E7qKouLy/r4uKiyVbLez4t38PWe33dcjtoDdPptJf3fFq+h633+rrldhD0mHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHAhkHCrajQaNT0L0vLcSUutT7n0dWsVTpDcw9b5+XmTrZbnTqran3Lp49ZkMqmu65wg2cSt6uG5k65rf8qlj1u/J+kECfSScCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCHQw/t+AfTHn6dcWpjP573cGo/HKz3PCZJ72OrrCZI+f2attiaTSS0WCydINnGrenqCpM+fWSt/NOYECfSRcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQEyQ932p17qSqajgc9vZ9fPToUZOtyWRS79+/v/UEya3h/tV4PO4Wi8WdXtiq5vN5HR4e2rrj1tHRUZOtqqrZbNbb93E4HDbZevbs2Urh+qoMgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgdY6QVJVw6pqddNir6qubMVstd7r69aw67pfb3vSWidIWhoMBouu68a2MrZa7/3ft3xVhkDChUCbHO6Zrait1nv/662N/RsX+LFN/h8X+AHhQiDhQiDhQiDhQqDfAKnxsfwvvOomAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "qmaze.act(DOWN)  # move down\n",
    "qmaze.act(RIGHT)  # move right\n",
    "qmaze.act(RIGHT)  # move right\n",
    "qmaze.act(RIGHT)  # move right\n",
    "qmaze.act(UP)  # move up\n",
    "show(qmaze)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_game(model, qmaze, rat_cell):\n",
    "    qmaze.reset(rat_cell)\n",
    "    envstate = qmaze.observe()\n",
    "    while True:\n",
    "        prev_envstate = envstate\n",
    "        # get next action\n",
    "        q = model.predict(prev_envstate)\n",
    "        action = np.argmax(q[0])\n",
    "\n",
    "        # apply action, get rewards and new state\n",
    "        envstate, reward, game_status = qmaze.act(action)\n",
    "        if game_status == 'win':\n",
    "            return True\n",
    "        elif game_status == 'lose':\n",
    "            return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def completion_check(model, qmaze):\n",
    "    for cell in qmaze.free_cells:\n",
    "        if not qmaze.valid_actions(cell):\n",
    "            return False\n",
    "        if not play_game(model, qmaze, cell):\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Experience(object):\n",
    "    def __init__(self, model, max_memory=100, discount=0.95):\n",
    "        self.model = model\n",
    "        self.max_memory = max_memory\n",
    "        self.discount = discount\n",
    "        self.memory = list()\n",
    "        self.num_actions = model.output_shape[-1]\n",
    "\n",
    "    def remember(self, episode):\n",
    "        # episode = [envstate, action, reward, envstate_next, game_over]\n",
    "        # memory[i] = episode\n",
    "        # envstate == flattened 1d maze cells info, including rat cell (see method: observe)\n",
    "        self.memory.append(episode)\n",
    "        if len(self.memory) > self.max_memory:\n",
    "            del self.memory[0]\n",
    "\n",
    "    def predict(self, envstate):\n",
    "        return self.model.predict(envstate)[0]\n",
    "\n",
    "    def get_data(self, data_size=10):\n",
    "        env_size = self.memory[0][0].shape[1]   # envstate 1d size (1st element of episode)\n",
    "        mem_size = len(self.memory)\n",
    "        data_size = min(mem_size, data_size)\n",
    "        inputs = np.zeros((data_size, env_size))\n",
    "        targets = np.zeros((data_size, self.num_actions))\n",
    "        for i, j in enumerate(np.random.choice(range(mem_size), data_size, replace=False)):\n",
    "            envstate, action, reward, envstate_next, game_over = self.memory[j]\n",
    "            inputs[i] = envstate\n",
    "            # There should be no target values for actions not taken.\n",
    "            targets[i] = self.predict(envstate)\n",
    "            # Q_sa = derived policy = max quality env/action = max_a' Q(s', a')\n",
    "            Q_sa = np.max(self.predict(envstate_next))\n",
    "            if game_over:\n",
    "                targets[i, action] = reward\n",
    "            else:\n",
    "                # reward + gamma * max_a' Q(s', a')\n",
    "                targets[i, action] = reward + self.discount * Q_sa\n",
    "        return inputs, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qtrain(model, maze, **opt):\n",
    "    global epsilon\n",
    "    n_epoch = opt.get('n_epoch', 15000)\n",
    "    max_memory = opt.get('max_memory', 1000)\n",
    "    data_size = opt.get('data_size', 50)\n",
    "    weights_file = opt.get('weights_file', \"\")\n",
    "    name = opt.get('name', 'model')\n",
    "    start_time = datetime.datetime.now()\n",
    "\n",
    "    # If you want to continue training from a previous model,\n",
    "    # just supply the h5 file name to weights_file option\n",
    "    if weights_file:\n",
    "        print(\"loading weights from file: %s\" % (weights_file,))\n",
    "        model.load_weights(weights_file)\n",
    "\n",
    "    # Construct environment/game from numpy array: maze (see above)\n",
    "    qmaze = Qmaze(maze)\n",
    "\n",
    "    # Initialize experience replay object\n",
    "    experience = Experience(model, max_memory=max_memory)\n",
    "\n",
    "    win_history = []   # history of win/lose game\n",
    "    n_free_cells = len(qmaze.free_cells)\n",
    "    hsize = qmaze.maze.size//2   # history window size\n",
    "    win_rate = 0.0\n",
    "    imctr = 1\n",
    "\n",
    "    for epoch in range(n_epoch):\n",
    "        loss = 0.0\n",
    "        rat_cell = random.choice(qmaze.free_cells)\n",
    "        qmaze.reset(rat_cell)\n",
    "        game_over = False\n",
    "\n",
    "        # get initial envstate (1d flattened canvas)\n",
    "        envstate = qmaze.observe()\n",
    "\n",
    "        n_episodes = 0\n",
    "        while not game_over:\n",
    "            valid_actions = qmaze.valid_actions()\n",
    "            if not valid_actions: break\n",
    "            prev_envstate = envstate\n",
    "            # Get next action\n",
    "            if np.random.rand() < epsilon:\n",
    "                action = random.choice(valid_actions)\n",
    "            else:\n",
    "                action = np.argmax(experience.predict(prev_envstate))\n",
    "\n",
    "            # Apply action, get reward and new envstate\n",
    "            envstate, reward, game_status = qmaze.act(action)\n",
    "            if game_status == 'win':\n",
    "                win_history.append(1)\n",
    "                game_over = True\n",
    "            elif game_status == 'lose':\n",
    "                win_history.append(0)\n",
    "                game_over = True\n",
    "            else:\n",
    "                game_over = False\n",
    "\n",
    "            # Store episode (experience)\n",
    "            episode = [prev_envstate, action, reward, envstate, game_over]\n",
    "            experience.remember(episode)\n",
    "            n_episodes += 1\n",
    "\n",
    "            # Train neural network model\n",
    "            inputs, targets = experience.get_data(data_size=data_size)\n",
    "            h = model.fit(\n",
    "                inputs,\n",
    "                targets,\n",
    "                epochs=8,\n",
    "                batch_size=16,\n",
    "                verbose=0,\n",
    "            )\n",
    "            loss = model.evaluate(inputs, targets, verbose=0)\n",
    "\n",
    "        if len(win_history) > hsize:\n",
    "            win_rate = sum(win_history[-hsize:]) / hsize\n",
    "    \n",
    "        dt = datetime.datetime.now() - start_time\n",
    "        t = format_time(dt.total_seconds())\n",
    "        template = \"Epoch: {:03d}/{:d} | Loss: {:.4f} | Episodes: {:d} | Win count: {:d} | Win rate: {:.3f} | time: {}\"\n",
    "        print(template.format(epoch, n_epoch-1, loss, n_episodes, sum(win_history), win_rate, t))\n",
    "        # we simply check if training has exhausted all free cells and if in all\n",
    "        # cases the agent won\n",
    "        if win_rate > 0.9 : epsilon = 0.05\n",
    "        if sum(win_history[-hsize:]) == hsize and completion_check(model, qmaze):\n",
    "            print(\"Reached 100%% win rate at epoch: %d\" % (epoch,))\n",
    "            break\n",
    "\n",
    "    # Save trained model weights and architecture, this will be used by the visualization code\n",
    "    h5file = name + \".h5\"\n",
    "    json_file = name + \".json\"\n",
    "    model.save_weights(h5file, overwrite=True)\n",
    "    with open(json_file, \"w\") as outfile:\n",
    "        json.dump(model.to_json(), outfile)\n",
    "    end_time = datetime.datetime.now()\n",
    "    dt = datetime.datetime.now() - start_time\n",
    "    seconds = dt.total_seconds()\n",
    "    t = format_time(seconds)\n",
    "    print('files: %s, %s' % (h5file, json_file))\n",
    "    print(\"n_epoch: %d, max_mem: %d, data: %d, time: %s\" % (epoch, max_memory, data_size, t))\n",
    "    return seconds\n",
    "\n",
    "# This is a small utility for printing readable time strings:\n",
    "def format_time(seconds):\n",
    "    if seconds < 400:\n",
    "        s = float(seconds)\n",
    "        return \"%.1f seconds\" % (s,)\n",
    "    elif seconds < 4000:\n",
    "        m = seconds / 60.0\n",
    "        return \"%.2f minutes\" % (m,)\n",
    "    else:\n",
    "        h = seconds / 3600.0\n",
    "        return \"%.2f hours\" % (h,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(maze, lr=0.001):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(maze.size, input_shape=(maze.size,)))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dense(maze.size))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dense(num_actions))\n",
    "    model.compile(optimizer='adam', loss='mse')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x29236b63208>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABU5JREFUeJzt3b+KU2kcx+H3LKIQXWwWTjOlEPukFeJVeAdewWm9g1gLXsH0XsDkAiaF5XQWggxMOdbvFgq7IG4m/pnffuc8D6QLfI8mH52pfkPvvQFZ/qh+AOB4woVAwoVAwoVAwoVAwoVAwoVAwoVAwoVA94558/379/tisfhdz/KfFotF+/TpU8n206dP28OHD0u2P3/+bHtG2x8+fGhXV1fDofcdFe5isWjPnj378af6CZvNpk3TVLL95s2bttlsSrZ3u53tGW2v1+sbvc+PyhBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBBIuBDoqKNfT548ae/evftdz/Kfdrtd672XbVfZ7/ft+fPnJdvb7bZ0u+rwVmutDcPBg3mlhkMxDMPwsrX2srXWxnFcnZ6e3sZzfeP6+ro9evRodtuXl5ft48ePJdsnJyel2+M4lmxfX1+3i4uLku1pmlrv/fC/Gr33G79Wq1WvcnZ2Nsvt7XbbW2slr+rtKmdnZ2V/7i9JHm7R77gQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQ6Khw9/t9G4ah5DXX7dVqddRhtl/5qt7m+446s/n48ePVq1evbuO5vlF98rFqe7lczvK8aPX2nTqz2QpPD1affKzanut50ertyu96d2YT7ibhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQqCjwq0+uzjH7WpzPG263+9Lv2s3+lwOfTn+fWZzHMfV6enpT38ZfkT12cW5bledm6w+qzqOY8n2NE3t/Pz8157ZXK1WvUr12cW5brcZnjbdbrdlf+dfGzvYot9xIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIVDMmc3Ly8vSs4tz3a46N1l9XrRq+86d2aw+uzjX7SrV50WrOLMJd5hwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZBwIZAzmzewXC5nefLR9u1zZvMXvuZ68tH27XNmE+4w4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UKgmDObcz27WHle9OTkpI3jWLJd/Xk/ePCgZHuapvb+/fuDZzbvHXpD7/1ta+1ta62t1+u+2Wx+/ul+wG63a3Pcfv36dZumqWR7u922Fy9elGxXf97L5bJk+6b8qAyBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuBjjqz2VpbttYufvdDfcdfrbUr27bv+Pay9/7noTcdDPf/YhiG89772rZt235UhkjChUBJ4b61bdv2FzG/4wL/SPofF/hKuBBIuBBIuBBIuBDob/Kuv6l+4Z2QAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "maze =  np.array([\n",
    "    [ 1.,  0.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  0.,  0.,  1.,  0.],\n",
    "    [ 0.,  0.,  0.,  1.,  1.,  1.,  0.],\n",
    "    [ 1.,  1.,  1.,  1.,  0.,  0.,  1.],\n",
    "    [ 1.,  0.,  0.,  0.,  1.,  1.,  1.],\n",
    "    [ 1.,  0.,  1.,  1.,  1.,  1.,  1.],\n",
    "    [ 1.,  1.,  1.,  0.,  1.,  1.,  1.]\n",
    "])\n",
    "\n",
    "qmaze = Qmaze(maze)\n",
    "show(qmaze)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/14999 | Loss: 0.0017 | Episodes: 108 | Win count: 0 | Win rate: 0.000 | time: 66.7 seconds\n",
      "Epoch: 001/14999 | Loss: 0.0242 | Episodes: 104 | Win count: 0 | Win rate: 0.000 | time: 81.5 seconds\n",
      "Epoch: 002/14999 | Loss: 0.0013 | Episodes: 103 | Win count: 0 | Win rate: 0.000 | time: 96.0 seconds\n",
      "Epoch: 003/14999 | Loss: 0.0523 | Episodes: 107 | Win count: 0 | Win rate: 0.000 | time: 111.0 seconds\n",
      "Epoch: 004/14999 | Loss: 0.1172 | Episodes: 104 | Win count: 0 | Win rate: 0.000 | time: 125.8 seconds\n",
      "Epoch: 005/14999 | Loss: 0.0015 | Episodes: 105 | Win count: 0 | Win rate: 0.000 | time: 140.6 seconds\n",
      "Epoch: 006/14999 | Loss: 0.0025 | Episodes: 103 | Win count: 0 | Win rate: 0.000 | time: 155.1 seconds\n",
      "Epoch: 007/14999 | Loss: 0.0008 | Episodes: 1 | Win count: 1 | Win rate: 0.000 | time: 155.2 seconds\n",
      "Epoch: 008/14999 | Loss: 0.0573 | Episodes: 49 | Win count: 2 | Win rate: 0.000 | time: 162.3 seconds\n",
      "Epoch: 009/14999 | Loss: 0.0008 | Episodes: 104 | Win count: 2 | Win rate: 0.000 | time: 177.0 seconds\n",
      "Epoch: 010/14999 | Loss: 0.0016 | Episodes: 33 | Win count: 3 | Win rate: 0.000 | time: 181.7 seconds\n",
      "Epoch: 011/14999 | Loss: 0.0034 | Episodes: 4 | Win count: 4 | Win rate: 0.000 | time: 182.2 seconds\n",
      "Epoch: 012/14999 | Loss: 0.0056 | Episodes: 102 | Win count: 4 | Win rate: 0.000 | time: 196.6 seconds\n",
      "Epoch: 013/14999 | Loss: 0.0069 | Episodes: 95 | Win count: 5 | Win rate: 0.000 | time: 210.0 seconds\n",
      "Epoch: 014/14999 | Loss: 0.0025 | Episodes: 17 | Win count: 6 | Win rate: 0.000 | time: 212.3 seconds\n",
      "Epoch: 015/14999 | Loss: 0.0017 | Episodes: 102 | Win count: 6 | Win rate: 0.000 | time: 227.2 seconds\n",
      "Epoch: 016/14999 | Loss: 0.0221 | Episodes: 6 | Win count: 7 | Win rate: 0.000 | time: 228.1 seconds\n",
      "Epoch: 017/14999 | Loss: 0.0032 | Episodes: 5 | Win count: 8 | Win rate: 0.000 | time: 228.9 seconds\n",
      "Epoch: 018/14999 | Loss: 0.0046 | Episodes: 19 | Win count: 9 | Win rate: 0.000 | time: 231.8 seconds\n",
      "Epoch: 019/14999 | Loss: 0.0033 | Episodes: 14 | Win count: 10 | Win rate: 0.000 | time: 233.9 seconds\n",
      "Epoch: 020/14999 | Loss: 0.0018 | Episodes: 3 | Win count: 11 | Win rate: 0.000 | time: 234.4 seconds\n",
      "Epoch: 021/14999 | Loss: 0.0049 | Episodes: 5 | Win count: 12 | Win rate: 0.000 | time: 235.1 seconds\n",
      "Epoch: 022/14999 | Loss: 0.0124 | Episodes: 104 | Win count: 12 | Win rate: 0.000 | time: 249.6 seconds\n",
      "Epoch: 023/14999 | Loss: 0.0023 | Episodes: 78 | Win count: 13 | Win rate: 0.000 | time: 260.5 seconds\n",
      "Epoch: 024/14999 | Loss: 0.0030 | Episodes: 15 | Win count: 14 | Win rate: 0.583 | time: 262.5 seconds\n",
      "Epoch: 025/14999 | Loss: 0.0022 | Episodes: 1 | Win count: 15 | Win rate: 0.625 | time: 262.7 seconds\n",
      "Epoch: 026/14999 | Loss: 0.0023 | Episodes: 21 | Win count: 16 | Win rate: 0.667 | time: 265.6 seconds\n",
      "Epoch: 027/14999 | Loss: 0.0014 | Episodes: 15 | Win count: 17 | Win rate: 0.708 | time: 267.6 seconds\n",
      "Epoch: 028/14999 | Loss: 0.0022 | Episodes: 11 | Win count: 18 | Win rate: 0.750 | time: 269.2 seconds\n",
      "Epoch: 029/14999 | Loss: 0.0017 | Episodes: 11 | Win count: 19 | Win rate: 0.792 | time: 270.8 seconds\n",
      "Epoch: 030/14999 | Loss: 0.0012 | Episodes: 17 | Win count: 20 | Win rate: 0.833 | time: 273.2 seconds\n",
      "Epoch: 031/14999 | Loss: 0.0127 | Episodes: 22 | Win count: 21 | Win rate: 0.833 | time: 276.2 seconds\n",
      "Epoch: 032/14999 | Loss: 0.0024 | Episodes: 5 | Win count: 22 | Win rate: 0.833 | time: 276.9 seconds\n",
      "Epoch: 033/14999 | Loss: 0.0008 | Episodes: 4 | Win count: 23 | Win rate: 0.875 | time: 277.5 seconds\n",
      "Epoch: 034/14999 | Loss: 0.0005 | Episodes: 26 | Win count: 24 | Win rate: 0.875 | time: 281.2 seconds\n",
      "Epoch: 035/14999 | Loss: 0.0064 | Episodes: 4 | Win count: 25 | Win rate: 0.875 | time: 281.7 seconds\n",
      "Epoch: 036/14999 | Loss: 0.0013 | Episodes: 28 | Win count: 26 | Win rate: 0.917 | time: 285.6 seconds\n",
      "Epoch: 037/14999 | Loss: 0.0011 | Episodes: 8 | Win count: 27 | Win rate: 0.917 | time: 286.8 seconds\n",
      "Epoch: 038/14999 | Loss: 0.0007 | Episodes: 2 | Win count: 28 | Win rate: 0.917 | time: 287.1 seconds\n",
      "Epoch: 039/14999 | Loss: 0.0012 | Episodes: 4 | Win count: 29 | Win rate: 0.958 | time: 287.7 seconds\n",
      "Epoch: 040/14999 | Loss: 0.0010 | Episodes: 25 | Win count: 30 | Win rate: 0.958 | time: 291.1 seconds\n",
      "Epoch: 041/14999 | Loss: 0.0008 | Episodes: 34 | Win count: 31 | Win rate: 0.958 | time: 295.9 seconds\n",
      "Epoch: 042/14999 | Loss: 0.0008 | Episodes: 6 | Win count: 32 | Win rate: 0.958 | time: 296.7 seconds\n",
      "Epoch: 043/14999 | Loss: 0.0012 | Episodes: 24 | Win count: 33 | Win rate: 0.958 | time: 300.0 seconds\n",
      "Epoch: 044/14999 | Loss: 0.0017 | Episodes: 2 | Win count: 34 | Win rate: 0.958 | time: 300.3 seconds\n",
      "Epoch: 045/14999 | Loss: 0.0020 | Episodes: 25 | Win count: 35 | Win rate: 0.958 | time: 303.8 seconds\n",
      "Epoch: 046/14999 | Loss: 0.0004 | Episodes: 19 | Win count: 36 | Win rate: 1.000 | time: 306.5 seconds\n",
      "Epoch: 047/14999 | Loss: 0.0004 | Episodes: 2 | Win count: 37 | Win rate: 1.000 | time: 307.0 seconds\n",
      "Epoch: 048/14999 | Loss: 0.0007 | Episodes: 26 | Win count: 38 | Win rate: 1.000 | time: 310.8 seconds\n",
      "Epoch: 049/14999 | Loss: 0.0003 | Episodes: 10 | Win count: 39 | Win rate: 1.000 | time: 312.4 seconds\n",
      "Epoch: 050/14999 | Loss: 0.0008 | Episodes: 17 | Win count: 40 | Win rate: 1.000 | time: 315.0 seconds\n",
      "Epoch: 051/14999 | Loss: 0.0005 | Episodes: 13 | Win count: 41 | Win rate: 1.000 | time: 317.2 seconds\n",
      "Epoch: 052/14999 | Loss: 0.0005 | Episodes: 9 | Win count: 42 | Win rate: 1.000 | time: 318.6 seconds\n",
      "Epoch: 053/14999 | Loss: 0.0011 | Episodes: 15 | Win count: 43 | Win rate: 1.000 | time: 321.1 seconds\n",
      "Epoch: 054/14999 | Loss: 0.0009 | Episodes: 8 | Win count: 44 | Win rate: 1.000 | time: 322.4 seconds\n",
      "Epoch: 055/14999 | Loss: 0.0002 | Episodes: 3 | Win count: 45 | Win rate: 1.000 | time: 323.1 seconds\n",
      "Epoch: 056/14999 | Loss: 0.0008 | Episodes: 3 | Win count: 46 | Win rate: 1.000 | time: 323.7 seconds\n",
      "Epoch: 057/14999 | Loss: 0.0009 | Episodes: 14 | Win count: 47 | Win rate: 1.000 | time: 325.9 seconds\n",
      "Epoch: 058/14999 | Loss: 0.0001 | Episodes: 2 | Win count: 48 | Win rate: 1.000 | time: 326.4 seconds\n",
      "Epoch: 059/14999 | Loss: 0.0003 | Episodes: 3 | Win count: 49 | Win rate: 1.000 | time: 327.0 seconds\n",
      "Epoch: 060/14999 | Loss: 0.0005 | Episodes: 13 | Win count: 50 | Win rate: 1.000 | time: 329.1 seconds\n",
      "Epoch: 061/14999 | Loss: 0.0002 | Episodes: 25 | Win count: 51 | Win rate: 1.000 | time: 332.7 seconds\n",
      "Epoch: 062/14999 | Loss: 0.0004 | Episodes: 3 | Win count: 52 | Win rate: 1.000 | time: 333.4 seconds\n",
      "Epoch: 063/14999 | Loss: 0.0012 | Episodes: 4 | Win count: 53 | Win rate: 1.000 | time: 334.2 seconds\n",
      "Epoch: 064/14999 | Loss: 0.0001 | Episodes: 16 | Win count: 54 | Win rate: 1.000 | time: 336.6 seconds\n",
      "Epoch: 065/14999 | Loss: 0.0001 | Episodes: 2 | Win count: 55 | Win rate: 1.000 | time: 337.0 seconds\n",
      "Epoch: 066/14999 | Loss: 0.0004 | Episodes: 26 | Win count: 56 | Win rate: 1.000 | time: 340.7 seconds\n",
      "Epoch: 067/14999 | Loss: 0.0002 | Episodes: 5 | Win count: 57 | Win rate: 1.000 | time: 341.6 seconds\n",
      "Epoch: 068/14999 | Loss: 0.0002 | Episodes: 24 | Win count: 58 | Win rate: 1.000 | time: 345.4 seconds\n",
      "Epoch: 069/14999 | Loss: 0.0004 | Episodes: 15 | Win count: 59 | Win rate: 1.000 | time: 347.9 seconds\n",
      "Epoch: 070/14999 | Loss: 0.0008 | Episodes: 21 | Win count: 60 | Win rate: 1.000 | time: 351.0 seconds\n",
      "Reached 100% win rate at epoch: 70\n",
      "files: model.h5, model.json\n",
      "n_epoch: 70, max_mem: 392, data: 32, time: 352.0 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "351.984063"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_model(maze)\n",
    "qtrain(model, maze, epochs=1000, max_memory=8*maze.size, data_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}