Miner_Training_Colab_CodeSample.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Miner_Training_Colab_CodeSample.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHcugJLenzKZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "dcbaa3c4-e70c-4cb7-fb12-6221a3bf016a"
      },
      "source": [
        "import sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import datetime\n",
        "import json\n",
        "from array import *\n",
        "import os\n",
        "import math\n",
        "from random import randrange\n",
        "import random\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.models import model_from_json\n",
        "from keras.layers import Dense, Activation\n",
        "from keras import optimizers\n",
        "\n",
        "import tensorflow.compat.v1 as tf\n",
        "from tensorflow.compat.v1.keras import backend as K\n",
        "tf.disable_v2_behavior()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "non-resource variables are not supported in the long term\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3tvcApSyW1g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Classes in GAME_SOCKET_DUMMY.py\n",
        "class ObstacleInfo:\n",
        "    # initial energy for obstacles: Land (key = 0): -1, Forest(key = -1): 0 (random), Trap(key = -2): -10, Swamp (key = -3): -5\n",
        "    types = {0: -1, -1: 0, -2: -10, -3: -5}\n",
        "\n",
        "    def __init__(self):\n",
        "        self.type = 0\n",
        "        self.posx = 0\n",
        "        self.posy = 0\n",
        "        self.value = 0\n",
        "        \n",
        "class GoldInfo:\n",
        "    def __init__(self):\n",
        "        self.posx = 0\n",
        "        self.posy = 0\n",
        "        self.amount = 0\n",
        "\n",
        "    def loads(self, data):\n",
        "        golds = []\n",
        "        for gd in data:\n",
        "            g = GoldInfo()\n",
        "            g.posx = gd[\"posx\"]\n",
        "            g.posy = gd[\"posy\"]\n",
        "            g.amount = gd[\"amount\"]\n",
        "            golds.append(g)\n",
        "        return golds\n",
        "\n",
        "class PlayerInfo:\n",
        "    STATUS_PLAYING = 0\n",
        "    STATUS_ELIMINATED_WENT_OUT_MAP = 1\n",
        "    STATUS_ELIMINATED_OUT_OF_ENERGY = 2\n",
        "    STATUS_ELIMINATED_INVALID_ACTION = 3\n",
        "    STATUS_STOP_EMPTY_GOLD = 4\n",
        "    STATUS_STOP_END_STEP = 5\n",
        "\n",
        "    def __init__(self, id):\n",
        "        self.playerId = id\n",
        "        self.score = 0\n",
        "        self.energy = 0\n",
        "        self.posx = 0\n",
        "        self.posy = 0\n",
        "        self.lastAction = -1\n",
        "        self.status = PlayerInfo.STATUS_PLAYING\n",
        "        self.freeCount = 0\n",
        "\n",
        "class GameInfo:\n",
        "    def __init__(self):\n",
        "        self.numberOfPlayers = 1\n",
        "        self.width = 0\n",
        "        self.height = 0\n",
        "        self.steps = 100\n",
        "        self.golds = []\n",
        "        self.obstacles = []\n",
        "\n",
        "    def loads(self, data):\n",
        "        m = GameInfo()\n",
        "        m.width = data[\"width\"]\n",
        "        m.height = data[\"height\"]\n",
        "        m.golds = GoldInfo().loads(data[\"golds\"])\n",
        "        m.obstacles = data[\"obstacles\"]\n",
        "        m.numberOfPlayers = data[\"numberOfPlayers\"]\n",
        "        m.steps = data[\"steps\"]\n",
        "        return m\n",
        "\n",
        "class UserMatch:\n",
        "    def __init__(self):\n",
        "        self.playerId = 1\n",
        "        self.posx = 0\n",
        "        self.posy = 0\n",
        "        self.energy = 50\n",
        "        self.gameinfo = GameInfo()\n",
        "\n",
        "    def to_json(self):\n",
        "        return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)\n",
        "\n",
        "class StepState:\n",
        "    def __init__(self):\n",
        "        self.players = []\n",
        "        self.golds = []\n",
        "        self.changedObstacles = []\n",
        "\n",
        "    def to_json(self):\n",
        "        return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Madmz8hE1op6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Main class in GAME_SOCKET_DUMMY.py\n",
        "class GameSocket:\n",
        "    bog_energy_chain = {-5: -20, -20: -40, -40: -100, -100: -100}\n",
        "\n",
        "    def __init__(self):\n",
        "        self.stepCount = 0\n",
        "        self.maxStep = 0\n",
        "        self.mapdir = \"Maps\"  # where to load all pre-defined maps\n",
        "        self.mapid = \"\"\n",
        "        self.userMatch = UserMatch()\n",
        "        self.user = PlayerInfo(1)\n",
        "        self.stepState = StepState()\n",
        "        self.maps = {}  # key: map file name, value: file content\n",
        "        self.map = []  # running map info: 0->Land, -1->Forest, -2->Trap, -3:Swamp, >0:Gold\n",
        "        self.energyOnMap = []  # self.energyOnMap[x][y]: <0, amount of energy which player will consume if it move into (x,y)\n",
        "        self.E = 50\n",
        "        self.resetFlag = True\n",
        "        self.craftUsers = []  # players that craft at current step - for calculating amount of gold\n",
        "        self.bots = []\n",
        "        self.craftMap = {}  # cells that players craft at current step, key: x_y, value: number of players that craft at (x,y)\n",
        "\n",
        "    def init_bots(self):\n",
        "        self.bots = [Bot1(2), Bot2(3), Bot3(4)]  # use bot1(id=2), bot2(id=3), bot3(id=4)\n",
        "        for (bot) in self.bots:  # at the beginning, all bots will have same position, energy as player\n",
        "            bot.info.posx = self.user.posx\n",
        "            bot.info.posy = self.user.posy\n",
        "            bot.info.energy = self.user.energy\n",
        "            bot.info.lastAction = -1\n",
        "            bot.info.status = PlayerInfo.STATUS_PLAYING\n",
        "            bot.info.score = 0\n",
        "            self.stepState.players.append(bot.info)\n",
        "        self.userMatch.gameinfo.numberOfPlayers = len(self.stepState.players)\n",
        "        print(\"numberOfPlayers: \", self.userMatch.gameinfo.numberOfPlayers)\n",
        "\n",
        "    def reset(self, requests):  # load new game by given request: [map id (filename), posx, posy, initial energy]\n",
        "        # load new map\n",
        "        self.reset_map(requests[0])\n",
        "        self.userMatch.posx = int(requests[1])\n",
        "        self.userMatch.posy = int(requests[2])\n",
        "        self.userMatch.energy = int(requests[3])\n",
        "        self.userMatch.gameinfo.steps = int(requests[4])\n",
        "        self.maxStep = self.userMatch.gameinfo.steps\n",
        "\n",
        "        # init data for players\n",
        "        self.user.posx = self.userMatch.posx  # in\n",
        "        self.user.posy = self.userMatch.posy\n",
        "        self.user.energy = self.userMatch.energy\n",
        "        self.user.status = PlayerInfo.STATUS_PLAYING\n",
        "        self.user.score = 0\n",
        "        self.stepState.players = [self.user]\n",
        "        self.E = self.userMatch.energy\n",
        "        self.resetFlag = True\n",
        "        self.init_bots()\n",
        "        self.stepCount = 0\n",
        "\n",
        "    def reset_map(self, id):  # load map info\n",
        "        self.mapId = id\n",
        "        self.map = json.loads(self.maps[self.mapId])\n",
        "        self.userMatch = self.map_info(self.map)\n",
        "        self.stepState.golds = self.userMatch.gameinfo.golds\n",
        "        self.map = json.loads(self.maps[self.mapId])\n",
        "        self.energyOnMap = json.loads(self.maps[self.mapId])\n",
        "        for x in range(len(self.map)):\n",
        "            for y in range(len(self.map[x])):\n",
        "                if self.map[x][y] > 0:  # gold\n",
        "                    self.energyOnMap[x][y] = -4\n",
        "                else:  # obstacles\n",
        "                    self.energyOnMap[x][y] = ObstacleInfo.types[self.map[x][y]]\n",
        "\n",
        "    def connect(self): # simulate player's connect request\n",
        "        print(\"Connected to server.\")\n",
        "        for mapid in range(len(Maps)):\n",
        "            filename = \"map\" + str(mapid)\n",
        "            print(\"Found: \" + filename)\n",
        "            self.maps[filename] = str(Maps[mapid])\n",
        "\n",
        "    def map_info(self, map):  # get map info\n",
        "        # print(map)\n",
        "        userMatch = UserMatch()\n",
        "        userMatch.gameinfo.height = len(map)\n",
        "        userMatch.gameinfo.width = len(map[0])\n",
        "        i = 0\n",
        "        while i < len(map):\n",
        "            j = 0\n",
        "            while j < len(map[i]):\n",
        "                if map[i][j] > 0:  # gold\n",
        "                    g = GoldInfo()\n",
        "                    g.posx = j\n",
        "                    g.posy = i\n",
        "                    g.amount = map[i][j]\n",
        "                    userMatch.gameinfo.golds.append(g)\n",
        "                else:  # obstacles\n",
        "                    o = ObstacleInfo()\n",
        "                    o.posx = j\n",
        "                    o.posy = i\n",
        "                    o.type = -map[i][j]\n",
        "                    o.value = ObstacleInfo.types[map[i][j]]\n",
        "                    userMatch.gameinfo.obstacles.append(o)\n",
        "                j += 1\n",
        "            i += 1\n",
        "        return userMatch\n",
        "\n",
        "    def receive(self):  # send data to player (simulate player's receive request)\n",
        "        if self.resetFlag:  # for the first time -> send game info\n",
        "            self.resetFlag = False\n",
        "            data = self.userMatch.to_json()\n",
        "            for (bot) in self.bots:\n",
        "                bot.new_game(data)\n",
        "            # print(data)\n",
        "            return data\n",
        "        else:  # send step state\n",
        "            self.stepCount = self.stepCount + 1\n",
        "            if self.stepCount >= self.maxStep:\n",
        "                for player in self.stepState.players:\n",
        "                    player.status = PlayerInfo.STATUS_STOP_END_STEP\n",
        "            data = self.stepState.to_json()\n",
        "            for (bot) in self.bots:  # update bots' state\n",
        "                bot.new_state(data)\n",
        "            # print(data)\n",
        "            return data\n",
        "\n",
        "    def send(self, message):  # receive message from player (simulate send request from player)\n",
        "        if message.isnumeric():  # player send action\n",
        "            self.resetFlag = False\n",
        "            self.stepState.changedObstacles = []\n",
        "            action = int(message)\n",
        "            # print(\"Action = \", action)\n",
        "            self.user.lastAction = action\n",
        "            self.craftUsers = []\n",
        "            self.step_action(self.user, action)\n",
        "            for bot in self.bots:\n",
        "                if bot.info.status == PlayerInfo.STATUS_PLAYING:\n",
        "                    action = bot.next_action()\n",
        "                    bot.info.lastAction = action\n",
        "                    # print(\"Bot Action: \", action)\n",
        "                    self.step_action(bot.info, action)\n",
        "            self.action_5_craft()\n",
        "            for c in self.stepState.changedObstacles:\n",
        "                self.map[c[\"posy\"]][c[\"posx\"]] = -c[\"type\"]\n",
        "                self.energyOnMap[c[\"posy\"]][c[\"posx\"]] = c[\"value\"]\n",
        "\n",
        "        else:  # reset game\n",
        "            requests = message.split(\",\")\n",
        "            print(\"Reset game: \", requests)\n",
        "            self.reset(requests)\n",
        "\n",
        "    def step_action(self, user, action):\n",
        "        switcher = {\n",
        "            0: self.action_0_left,\n",
        "            1: self.action_1_right,\n",
        "            2: self.action_2_up,\n",
        "            3: self.action_3_down,\n",
        "            4: self.action_4_free,\n",
        "            5: self.action_5_craft_pre\n",
        "        }\n",
        "        func = switcher.get(action, self.invalidAction)\n",
        "        func(user)\n",
        "\n",
        "    def action_5_craft_pre(self, user):  # collect players who craft at current step\n",
        "        user.freeCount = 0\n",
        "        if self.map[user.posy][user.posx] <= 0:  # craft at the non-gold cell\n",
        "            user.energy -= 10\n",
        "            if user.energy <= 0:\n",
        "                user.status = PlayerInfo.STATUS_ELIMINATED_OUT_OF_ENERGY\n",
        "                user.lastAction = 6 #eliminated\n",
        "        else:\n",
        "            user.energy -= 5\n",
        "            if user.energy > 0:\n",
        "                self.craftUsers.append(user)\n",
        "                key = str(user.posx) + \"_\" + str(user.posy)\n",
        "                if key in self.craftMap:\n",
        "                    count = self.craftMap[key]\n",
        "                    self.craftMap[key] = count + 1\n",
        "                else:\n",
        "                    self.craftMap[key] = 1\n",
        "            else:\n",
        "                user.status = PlayerInfo.STATUS_ELIMINATED_OUT_OF_ENERGY\n",
        "                user.lastAction = 6 #eliminated\n",
        "\n",
        "    def action_0_left(self, user):  # user go left\n",
        "        user.freeCount = 0\n",
        "        user.posx = user.posx - 1\n",
        "        if user.posx < 0:\n",
        "            user.status = PlayerInfo.STATUS_ELIMINATED_WENT_OUT_MAP\n",
        "            user.lastAction = 6 #eliminated\n",
        "        else:\n",
        "            self.go_to_pos(user)\n",
        "\n",
        "    def action_1_right(self, user):  # user go right\n",
        "        user.freeCount = 0\n",
        "        user.posx = user.posx + 1\n",
        "        if user.posx >= self.userMatch.gameinfo.width:\n",
        "            user.status = PlayerInfo.STATUS_ELIMINATED_WENT_OUT_MAP\n",
        "            user.lastAction = 6 #eliminated\n",
        "        else:\n",
        "            self.go_to_pos(user)\n",
        "\n",
        "    def action_2_up(self, user):  # user go up\n",
        "        user.freeCount = 0\n",
        "        user.posy = user.posy - 1\n",
        "        if user.posy < 0:\n",
        "            user.status = PlayerInfo.STATUS_ELIMINATED_WENT_OUT_MAP\n",
        "            user.lastAction = 6 #eliminated\n",
        "        else:\n",
        "            self.go_to_pos(user)\n",
        "\n",
        "    def action_3_down(self, user):  # user go right\n",
        "        user.freeCount = 0\n",
        "        user.posy = user.posy + 1\n",
        "        if user.posy >= self.userMatch.gameinfo.height:\n",
        "            user.status = PlayerInfo.STATUS_ELIMINATED_WENT_OUT_MAP\n",
        "            user.lastAction = 6 #eliminated\n",
        "        else:\n",
        "            self.go_to_pos(user)\n",
        "\n",
        "    def action_4_free(self, user):  # user free\n",
        "        user.freeCount += 1\n",
        "        if user.freeCount == 1:\n",
        "            user.energy += int(self.E / 4)\n",
        "        elif user.freeCount == 2:\n",
        "            user.energy += int(self.E / 3)\n",
        "        elif user.freeCount == 3:\n",
        "            user.energy += int(self.E / 2)\n",
        "        else:\n",
        "            user.energy = self.E\n",
        "        if user.energy > self.E:\n",
        "            user.energy = self.E\n",
        "\n",
        "    def action_5_craft(self):\n",
        "        craftCount = len(self.craftUsers)\n",
        "        # print (\"craftCount\",craftCount)\n",
        "        if (craftCount > 0):\n",
        "            for user in self.craftUsers:\n",
        "                x = user.posx\n",
        "                y = user.posy\n",
        "                key = str(user.posx) + \"_\" + str(user.posy)\n",
        "                c = self.craftMap[key]\n",
        "                m = min(math.ceil(self.map[y][x] / c), 50)\n",
        "                user.score += m\n",
        "                # print (\"user\", user.playerId, m)\n",
        "            for user in self.craftUsers:\n",
        "                x = user.posx\n",
        "                y = user.posy\n",
        "                key = str(user.posx) + \"_\" + str(user.posy)\n",
        "                if key in self.craftMap:\n",
        "                    c = self.craftMap[key]\n",
        "                    del self.craftMap[key]\n",
        "                    m = min(math.ceil(self.map[y][x] / c), 50)\n",
        "                    self.map[y][x] -= m * c\n",
        "                    if self.map[y][x] < 0:\n",
        "                        self.map[y][x] = 0\n",
        "                        self.energyOnMap[y][x] = ObstacleInfo.types[0]\n",
        "                    for g in self.stepState.golds:\n",
        "                        if g.posx == x and g.posy == y:\n",
        "                            g.amount = self.map[y][x]\n",
        "                            if g.amount == 0:\n",
        "                                self.stepState.golds.remove(g)\n",
        "                                self.add_changed_obstacle(x, y, 0, ObstacleInfo.types[0])\n",
        "                                if len(self.stepState.golds) == 0:\n",
        "                                    for player in self.stepState.players:\n",
        "                                        player.status = PlayerInfo.STATUS_STOP_EMPTY_GOLD\n",
        "                            break;\n",
        "            self.craftMap = {}\n",
        "\n",
        "    def invalidAction(self, user):\n",
        "        user.status = PlayerInfo.STATUS_ELIMINATED_INVALID_ACTION\n",
        "        user.lastAction = 6 #eliminated\n",
        "\n",
        "    def go_to_pos(self, user):  # player move to cell(x,y)\n",
        "        if self.map[user.posy][user.posx] == -1:\n",
        "            user.energy -= randrange(16) + 5\n",
        "        elif self.map[user.posy][user.posx] == 0:\n",
        "            user.energy += self.energyOnMap[user.posy][user.posx]\n",
        "        elif self.map[user.posy][user.posx] == -2:\n",
        "            user.energy += self.energyOnMap[user.posy][user.posx]\n",
        "            self.add_changed_obstacle(user.posx, user.posy, 0, ObstacleInfo.types[0])\n",
        "        elif self.map[user.posy][user.posx] == -3:\n",
        "            user.energy += self.energyOnMap[user.posy][user.posx]\n",
        "            self.add_changed_obstacle(user.posx, user.posy, 3,\n",
        "                                      self.bog_energy_chain[self.energyOnMap[user.posy][user.posx]])\n",
        "        else:\n",
        "            user.energy -= 4\n",
        "        if user.energy <= 0:\n",
        "            user.status = PlayerInfo.STATUS_ELIMINATED_OUT_OF_ENERGY\n",
        "            user.lastAction = 6 #eliminated\n",
        "\n",
        "    def add_changed_obstacle(self, x, y, t, v):\n",
        "        added = False\n",
        "        for o in self.stepState.changedObstacles:\n",
        "            if o[\"posx\"] == x and o[\"posy\"] == y:\n",
        "                added = True\n",
        "                break\n",
        "        if added == False:\n",
        "            o = {}\n",
        "            o[\"posx\"] = x\n",
        "            o[\"posy\"] = y\n",
        "            o[\"type\"] = t\n",
        "            o[\"value\"] = v\n",
        "            self.stepState.changedObstacles.append(o)\n",
        "\n",
        "    def close(self):\n",
        "        print(\"Close socket.\")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZEExD0BCyePu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Bots :bot1\n",
        "class Bot1:\n",
        "    ACTION_GO_LEFT = 0\n",
        "    ACTION_GO_RIGHT = 1\n",
        "    ACTION_GO_UP = 2\n",
        "    ACTION_GO_DOWN = 3\n",
        "    ACTION_FREE = 4\n",
        "    ACTION_CRAFT = 5\n",
        "\n",
        "    def __init__(self, id):\n",
        "        self.state = State()\n",
        "        self.info = PlayerInfo(id)\n",
        "\n",
        "    def next_action(self):\n",
        "        if self.state.mapInfo.gold_amount(self.info.posx, self.info.posy) > 0:\n",
        "            if self.info.energy >= 6:\n",
        "                return self.ACTION_CRAFT\n",
        "            else:\n",
        "                return self.ACTION_FREE\n",
        "        if self.info.energy < 5:\n",
        "            return self.ACTION_FREE\n",
        "        else:\n",
        "            action = self.ACTION_GO_UP\n",
        "            if self.info.posy % 2 == 0:\n",
        "                if self.info.posx < self.state.mapInfo.max_x:\n",
        "                    action = self.ACTION_GO_RIGHT\n",
        "            else:\n",
        "                if self.info.posx > 0:\n",
        "                    action = self.ACTION_GO_LEFT\n",
        "                else:\n",
        "                    action = self.ACTION_GO_DOWN\n",
        "            return action\n",
        "\n",
        "    def new_game(self, data):\n",
        "        try:\n",
        "            self.state.init_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n",
        "\n",
        "    def new_state(self, data):\n",
        "        # action = self.next_action();\n",
        "        # self.socket.send(action)\n",
        "        try:\n",
        "            self.state.update_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UYHsBcVEyiCm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Bots :bot2\n",
        "class Bot2:\n",
        "    ACTION_GO_LEFT = 0\n",
        "    ACTION_GO_RIGHT = 1\n",
        "    ACTION_GO_UP = 2\n",
        "    ACTION_GO_DOWN = 3\n",
        "    ACTION_FREE = 4\n",
        "    ACTION_CRAFT = 5\n",
        "\n",
        "    def __init__(self, id):\n",
        "        self.state = State()\n",
        "        self.info = PlayerInfo(id)\n",
        "\n",
        "    def next_action(self):\n",
        "        if self.state.mapInfo.gold_amount(self.info.posx, self.info.posy) > 0:\n",
        "            if self.info.energy >= 6:\n",
        "                return self.ACTION_CRAFT\n",
        "            else:\n",
        "                return self.ACTION_FREE\n",
        "        if self.info.energy < 5:\n",
        "            return self.ACTION_FREE\n",
        "        else:\n",
        "            action = np.random.randint(0, 4)            \n",
        "            return action\n",
        "\n",
        "    def new_game(self, data):\n",
        "        try:\n",
        "            self.state.init_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n",
        "\n",
        "    def new_state(self, data):\n",
        "        # action = self.next_action();\n",
        "        # self.socket.send(action)\n",
        "        try:\n",
        "            self.state.update_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CCQo94-0ykm6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Bots :bot3\n",
        "class Bot3:\n",
        "    ACTION_GO_LEFT = 0\n",
        "    ACTION_GO_RIGHT = 1\n",
        "    ACTION_GO_UP = 2\n",
        "    ACTION_GO_DOWN = 3\n",
        "    ACTION_FREE = 4\n",
        "    ACTION_CRAFT = 5\n",
        "\n",
        "    def __init__(self, id):\n",
        "        self.state = State()\n",
        "        self.info = PlayerInfo(id)\n",
        "\n",
        "    def next_action(self):\n",
        "        if self.state.mapInfo.gold_amount(self.info.posx, self.info.posy) > 0:\n",
        "            if self.info.energy >= 6:\n",
        "                return self.ACTION_CRAFT\n",
        "            else:\n",
        "                return self.ACTION_FREE\n",
        "        if self.info.energy < 5:\n",
        "            return self.ACTION_FREE\n",
        "        else:\n",
        "            action = self.ACTION_GO_LEFT\n",
        "            if self.info.posx % 2 == 0:\n",
        "                if self.info.posy < self.state.mapInfo.max_y:\n",
        "                    action = self.ACTION_GO_DOWN\n",
        "            else:\n",
        "                if self.info.posy > 0:\n",
        "                    action = self.ACTION_GO_UP\n",
        "                else:\n",
        "                    action = self.ACTION_GO_RIGHT            \n",
        "            return action\n",
        "\n",
        "    def new_game(self, data):\n",
        "        try:\n",
        "            self.state.init_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n",
        "\n",
        "    def new_state(self, data):\n",
        "        # action = self.next_action();\n",
        "        # self.socket.send(action)\n",
        "        try:\n",
        "            self.state.update_state(data)\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "agO3td72yvaS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#MinerState.py\n",
        "def str_2_json(str):\n",
        "    return json.loads(str, encoding=\"utf-8\")\n",
        "\n",
        "\n",
        "class MapInfo:\n",
        "    def __init__(self):\n",
        "        self.max_x = 0 #Width of the map\n",
        "        self.max_y = 0 #Height of the map\n",
        "        self.golds = [] #List of the golds in the map\n",
        "        self.obstacles = []\n",
        "        self.numberOfPlayers = 0\n",
        "        self.maxStep = 0 #The maximum number of step is set for this map\n",
        "\n",
        "    def init_map(self, gameInfo):\n",
        "        #Initialize the map at the begining of each episode\n",
        "        self.max_x = gameInfo[\"width\"] - 1\n",
        "        self.max_y = gameInfo[\"height\"] - 1\n",
        "        self.golds = gameInfo[\"golds\"]\n",
        "        self.obstacles = gameInfo[\"obstacles\"]\n",
        "        self.maxStep = gameInfo[\"steps\"]\n",
        "        self.numberOfPlayers = gameInfo[\"numberOfPlayers\"]\n",
        "\n",
        "    def update(self, golds, changedObstacles):\n",
        "        #Update the map after every step\n",
        "        self.golds = golds\n",
        "        for cob in changedObstacles:\n",
        "            newOb = True\n",
        "            for ob in self.obstacles:\n",
        "                if cob[\"posx\"] == ob[\"posx\"] and cob[\"posy\"] == ob[\"posy\"]:\n",
        "                    newOb = False\n",
        "                    #print(\"cell(\", cob[\"posx\"], \",\", cob[\"posy\"], \") change type from: \", ob[\"type\"], \" -> \",\n",
        "                    #      cob[\"type\"], \" / value: \", ob[\"value\"], \" -> \", cob[\"value\"])\n",
        "                    ob[\"type\"] = cob[\"type\"]\n",
        "                    ob[\"value\"] = cob[\"value\"]\n",
        "                    break\n",
        "            if newOb:\n",
        "                self.obstacles.append(cob)\n",
        "                #print(\"new obstacle: \", cob[\"posx\"], \",\", cob[\"posy\"], \", type = \", cob[\"type\"], \", value = \",\n",
        "                #      cob[\"value\"])\n",
        "\n",
        "    def get_min_x(self):\n",
        "        return min([cell[\"posx\"] for cell in self.golds])\n",
        "\n",
        "    def get_max_x(self):\n",
        "        return max([cell[\"posx\"] for cell in self.golds])\n",
        "\n",
        "    def get_min_y(self):\n",
        "        return min([cell[\"posy\"] for cell in self.golds])\n",
        "\n",
        "    def get_max_y(self):\n",
        "        return max([cell[\"posy\"] for cell in self.golds])\n",
        "\n",
        "    def is_row_has_gold(self, y):\n",
        "        return y in [cell[\"posy\"] for cell in self.golds]\n",
        "\n",
        "    def is_column_has_gold(self, x):\n",
        "        return x in [cell[\"posx\"] for cell in self.golds]\n",
        "\n",
        "    def gold_amount(self, x, y): #Get the amount of golds at cell (x,y)\n",
        "        for cell in self.golds:\n",
        "            if x == cell[\"posx\"] and y == cell[\"posy\"]:\n",
        "                return cell[\"amount\"]\n",
        "        return 0 \n",
        "\n",
        "    def get_obstacle(self, x, y):  # Get the kind of the obstacle at cell(x,y)\n",
        "        for cell in self.obstacles:\n",
        "            if x == cell[\"posx\"] and y == cell[\"posy\"]:\n",
        "                return cell[\"type\"]\n",
        "        return -1  # No obstacle at the cell (x,y)\n",
        "\n",
        "\n",
        "class State:\n",
        "    STATUS_PLAYING = 0\n",
        "    STATUS_ELIMINATED_WENT_OUT_MAP = 1\n",
        "    STATUS_ELIMINATED_OUT_OF_ENERGY = 2\n",
        "    STATUS_ELIMINATED_INVALID_ACTION = 3\n",
        "    STATUS_STOP_EMPTY_GOLD = 4\n",
        "    STATUS_STOP_END_STEP = 5\n",
        "\n",
        "    def __init__(self):\n",
        "        self.end = False\n",
        "        self.score = 0\n",
        "        self.lastAction = None\n",
        "        self.id = 0\n",
        "        self.x = 0\n",
        "        self.y = 0\n",
        "        self.energy = 0\n",
        "        self.mapInfo = MapInfo()\n",
        "        self.players = []\n",
        "        self.stepCount = 0\n",
        "        self.status = State.STATUS_PLAYING\n",
        "\n",
        "    def init_state(self, data): #parse data from server into object\n",
        "        game_info = str_2_json(data)\n",
        "        self.end = False\n",
        "        self.score = 0\n",
        "        self.lastAction = None\n",
        "        self.id = game_info[\"playerId\"]\n",
        "        self.x = game_info[\"posx\"]\n",
        "        self.y = game_info[\"posy\"]\n",
        "        self.energy = game_info[\"energy\"]\n",
        "        self.mapInfo.init_map(game_info[\"gameinfo\"])\n",
        "        self.stepCount = 0\n",
        "        self.status = State.STATUS_PLAYING\n",
        "        self.players = [{\"playerId\": 2, \"posx\": self.x, \"posy\": self.y},\n",
        "                        {\"playerId\": 3, \"posx\": self.x, \"posy\": self.y},\n",
        "                        {\"playerId\": 4, \"posx\": self.x, \"posy\": self.y}]\n",
        "\n",
        "    def update_state(self, data):\n",
        "        new_state = str_2_json(data)\n",
        "        for player in new_state[\"players\"]:\n",
        "            if player[\"playerId\"] == self.id:\n",
        "                self.x = player[\"posx\"]\n",
        "                self.y = player[\"posy\"]\n",
        "                self.energy = player[\"energy\"]\n",
        "                self.score = player[\"score\"]\n",
        "                self.lastAction = player[\"lastAction\"]\n",
        "                self.status = player[\"status\"]\n",
        "\n",
        "        self.mapInfo.update(new_state[\"golds\"], new_state[\"changedObstacles\"])\n",
        "        self.players = new_state[\"players\"]\n",
        "        for i in range(len(self.players), 4, 1):\n",
        "            self.players.append({\"playerId\": i, \"posx\": self.x, \"posy\": self.y})\n",
        "        self.stepCount = self.stepCount + 1"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QHa-DcAcyyMc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#MinerEnv.py\n",
        "TreeID = 1\n",
        "TrapID = 2\n",
        "SwampID = 3\n",
        "class MinerEnv:\n",
        "    def __init__(self):\n",
        "        self.socket = GameSocket()\n",
        "        self.state = State()\n",
        "        \n",
        "        self.score_pre = self.state.score#Storing the last score for designing the reward function\n",
        "\n",
        "    def start(self): #connect to server\n",
        "        self.socket.connect()\n",
        "\n",
        "    def end(self): #disconnect server\n",
        "        self.socket.close()\n",
        "\n",
        "    def send_map_info(self, request):#tell server which map to run\n",
        "        self.socket.send(request)\n",
        "\n",
        "    def reset(self): #start new game\n",
        "        try:\n",
        "            message = self.socket.receive() #receive game info from server\n",
        "            self.state.init_state(message) #init state\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n",
        "\n",
        "    def step(self, action): #step process\n",
        "        self.socket.send(action) #send action to server\n",
        "        try:\n",
        "            message = self.socket.receive() #receive new state from server\n",
        "            self.state.update_state(message) #update to local state\n",
        "        except Exception as e:\n",
        "            import traceback\n",
        "            traceback.print_exc()\n",
        "\n",
        "    # Functions are customized by client\n",
        "    def get_state(self):\n",
        "        # Building the map\n",
        "        view = np.zeros([self.state.mapInfo.max_x + 1, self.state.mapInfo.max_y + 1], dtype=int)\n",
        "        for i in range(self.state.mapInfo.max_x + 1):\n",
        "            for j in range(self.state.mapInfo.max_y + 1):\n",
        "                if self.state.mapInfo.get_obstacle(i, j) == TreeID:  # Tree\n",
        "                    view[i, j] = -TreeID\n",
        "                if self.state.mapInfo.get_obstacle(i, j) == TrapID:  # Trap\n",
        "                    view[i, j] = -TrapID\n",
        "                if self.state.mapInfo.get_obstacle(i, j) == SwampID: # Swamp\n",
        "                    view[i, j] = -SwampID\n",
        "                if self.state.mapInfo.gold_amount(i, j) > 0:\n",
        "                    view[i, j] = self.state.mapInfo.gold_amount(i, j)\n",
        "\n",
        "        DQNState = view.flatten().tolist() #Flattening the map matrix to a vector\n",
        "        \n",
        "        # Add position and energy of agent to the DQNState\n",
        "        DQNState.append(self.state.x)\n",
        "        DQNState.append(self.state.y)\n",
        "        DQNState.append(self.state.energy)\n",
        "        #Add position of bots \n",
        "        for player in self.state.players:\n",
        "            if player[\"playerId\"] != self.state.id:\n",
        "                DQNState.append(player[\"posx\"])\n",
        "                DQNState.append(player[\"posy\"])\n",
        "                \n",
        "        #Convert the DQNState from list to array for training\n",
        "        DQNState = np.array(DQNState)\n",
        "\n",
        "        return DQNState\n",
        "\n",
        "    def get_reward(self):\n",
        "        # Calculate reward\n",
        "        reward = 0\n",
        "        score_action = self.state.score - self.score_pre\n",
        "        self.score_pre = self.state.score\n",
        "        if score_action > 0:\n",
        "            #If the DQN agent crafts golds, then it should obtain a positive reward (equal score_action)\n",
        "            reward += score_action\n",
        "            \n",
        "        #If the DQN agent crashs into obstacels (Tree, Trap, Swamp), then it should be punished by a negative reward\n",
        "        if self.state.mapInfo.get_obstacle(self.state.x, self.state.y) == TreeID:  # Tree\n",
        "            reward -= TreeID\n",
        "        if self.state.mapInfo.get_obstacle(self.state.x, self.state.y) == TrapID:  # Trap\n",
        "            reward -= TrapID\n",
        "        if self.state.mapInfo.get_obstacle(self.state.x, self.state.y) == SwampID:  # Swamp\n",
        "            reward -= SwampID\n",
        "\n",
        "        # If out of the map, then the DQN agent should be punished by a larger nagative reward.\n",
        "        if self.state.status == State.STATUS_ELIMINATED_WENT_OUT_MAP:\n",
        "            reward += -10\n",
        "            \n",
        "        #Run out of energy, then the DQN agent should be punished by a larger nagative reward.\n",
        "        if self.state.status == State.STATUS_ELIMINATED_OUT_OF_ENERGY:\n",
        "            reward += -10\n",
        "        # print (\"reward\",reward)\n",
        "        return reward\n",
        "\n",
        "    def check_terminate(self):\n",
        "        #Checking the status of the game\n",
        "        #it indicates the game ends or is playing\n",
        "        return self.state.status != State.STATUS_PLAYING"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qf2sasVey0sm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#DQNModel.py\n",
        "class DQN: \n",
        "   \n",
        "    def __init__(\n",
        "            self,\n",
        "            input_dim, #The number of inputs for the DQN network\n",
        "            action_space, #The number of actions for the DQN network\n",
        "            gamma = 0.99, #The discount factor\n",
        "            epsilon = 1, #Epsilon - the exploration factor\n",
        "            epsilon_min = 0.01, #The minimum epsilon \n",
        "            epsilon_decay = 0.999,#The decay epislon for each update_epsilon time\n",
        "            learning_rate = 0.00025, #The learning rate for the DQN network\n",
        "            tau = 0.125, #The factor for updating the DQN target network from the DQN network\n",
        "            model = None, #The DQN model\n",
        "            target_model = None, #The DQN target model \n",
        "            sess=None\n",
        "            \n",
        "    ):\n",
        "      self.input_dim = input_dim\n",
        "      self.action_space = action_space\n",
        "      self.gamma = gamma\n",
        "      self.epsilon = epsilon\n",
        "      self.epsilon_min = epsilon_min\n",
        "      self.epsilon_decay = epsilon_decay\n",
        "      self.learning_rate = learning_rate\n",
        "      self.tau = tau\n",
        "            \n",
        "      #Creating networks\n",
        "      self.model        = self.create_model() #Creating the DQN model\n",
        "      self.target_model = self.create_model() #Creating the DQN target model\n",
        "      \n",
        "      #Tensorflow GPU optimization\n",
        "      config = tf.ConfigProto()\n",
        "      config.gpu_options.allow_growth = True\n",
        "      self.sess = tf.Session(config=config)\n",
        "      K.set_session(sess)\n",
        "      self.sess.run( tf.global_variables_initializer()) \n",
        "      \n",
        "    def create_model(self):\n",
        "      #Creating the network\n",
        "      #Two hidden layers (300,300), their activation is ReLu\n",
        "      #One output layer with action_space of nodes, activation is linear.\n",
        "      model = Sequential()\n",
        "      model.add(Dense(300, input_dim=self.input_dim))\n",
        "      model.add(Activation('relu'))\n",
        "      model.add(Dense(300))\n",
        "      model.add(Activation('relu'))\n",
        "      model.add(Dense(self.action_space))\n",
        "      model.add(Activation('linear'))    \n",
        "      #adam = optimizers.adam(lr=self.learning_rate)\n",
        "      sgd = optimizers.SGD(lr=self.learning_rate, decay=1e-6, momentum=0.95)\n",
        "      model.compile(optimizer = sgd,\n",
        "              loss='mse')\n",
        "      return model\n",
        "  \n",
        "    \n",
        "    def act(self,state):\n",
        "      #Get the index of the maximum Q values      \n",
        "      a_max = np.argmax(self.model.predict(state.reshape(1,len(state))))      \n",
        "      if (random.random() < self.epsilon):\n",
        "        a_chosen = randrange(self.action_space)\n",
        "      else:\n",
        "        a_chosen = a_max      \n",
        "      return a_chosen\n",
        "    \n",
        "    \n",
        "    def replay(self,samples,batch_size):\n",
        "      inputs = np.zeros((batch_size, self.input_dim))\n",
        "      targets = np.zeros((batch_size, self.action_space))\n",
        "      \n",
        "      for i in range(0,batch_size):\n",
        "        state = samples[0][i,:]\n",
        "        action = samples[1][i]\n",
        "        reward = samples[2][i]\n",
        "        new_state = samples[3][i,:]\n",
        "        done= samples[4][i]\n",
        "        \n",
        "        inputs[i,:] = state\n",
        "        targets[i,:] = self.target_model.predict(state.reshape(1,len(state)))        \n",
        "        if done:\n",
        "          targets[i,action] = reward # if terminated, only equals reward\n",
        "        else:\n",
        "          Q_future = np.max(self.target_model.predict(new_state.reshape(1,len(new_state))))\n",
        "          targets[i,action] = reward + Q_future * self.gamma\n",
        "      #Training\n",
        "      loss = self.model.train_on_batch(inputs, targets)  \n",
        "    \n",
        "    def target_train(self): \n",
        "      weights = self.model.get_weights()\n",
        "      target_weights = self.target_model.get_weights()\n",
        "      for i in range(0, len(target_weights)):\n",
        "        target_weights[i] = weights[i] * self.tau + target_weights[i] * (1 - self.tau)\n",
        "      \n",
        "      self.target_model.set_weights(target_weights) \n",
        "    \n",
        "    \n",
        "    def update_epsilon(self):\n",
        "      self.epsilon =  self.epsilon*self.epsilon_decay\n",
        "      self.epsilon =  max(self.epsilon_min, self.epsilon)\n",
        "    \n",
        "    \n",
        "    def save_model(self, model_name):\n",
        "        # serialize model to JSON\n",
        "        model_json = self.model.to_json()\n",
        "        with open(model_name + \".json\", \"w\") as json_file:\n",
        "            json_file.write(model_json)\n",
        "            # serialize weights to HDF5\n",
        "            self.model.save_weights( model_name + \".h5\")\n",
        "            print(\"Saved model to disk\")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MARetIYHy4qp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Memory.py\n",
        "class Memory:        \n",
        "    capacity = None  \n",
        "    \n",
        "    def __init__(\n",
        "            self,\n",
        "            capacity,\n",
        "            length = None,\n",
        "            states = None,\n",
        "            actions = None,\n",
        "            rewards = None,\n",
        "            dones = None,\n",
        "            states2 = None,       \n",
        "    ):\n",
        "        self.capacity = capacity\n",
        "        self.length = 0\n",
        "        self.states = states\n",
        "        self.actions = actions\n",
        "        self.rewards = rewards\n",
        "        self.dones = dones\n",
        "        self.states2 = states2\n",
        "\n",
        "    def push(self, s, a, r, done, s2):\n",
        "        if self.states is None:\n",
        "            self.states = s\n",
        "            self.actions = a\n",
        "            self.rewards = r\n",
        "            self.dones = done\n",
        "            self.states2 = s2\n",
        "        else:\n",
        "            self.states = np.vstack((self.states,s))\n",
        "            self.actions = np.vstack((self.actions,a))\n",
        "            self.rewards = np.vstack((self.rewards, r))\n",
        "            self.dones = np.vstack((self.dones, done))\n",
        "            self.states2 = np.vstack((self.states2,s2))\n",
        "        \n",
        "        self.length = self.length + 1\n",
        "            \n",
        "        if (self.length > self.capacity): \n",
        "            self.states = np.delete(self.states,(0), axis = 0)\n",
        "            self.actions = np.delete(self.actions,(0), axis = 0)\n",
        "            self.rewards = np.delete(self.rewards,(0), axis = 0)\n",
        "            self.dones = np.delete(self.dones,(0), axis = 0)\n",
        "            self.states2 = np.delete(self.states2,(0), axis = 0)           \n",
        "            self.length = self.length - 1\n",
        "            \n",
        "        \n",
        "    def sample(self,batch_size):\n",
        "        if (self.length >= batch_size):\n",
        "            idx = random.sample(range(0,self.length),batch_size)\n",
        "            s = self.states[idx,:]\n",
        "            a = self.actions[idx,:]\n",
        "            r = self.rewards[idx,:]\n",
        "            d = self.dones[idx,:]\n",
        "            s2 = self.states2[idx,:]\n",
        "                \n",
        "            return list([s,a,r,s2,d])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ZqwV8edy7eX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Creating Maps\n",
        "#This function is used to create 05 maps instead of loading them from Maps folder in the local\n",
        "def CreateMaps():\n",
        "      map1 = [\n",
        "        [0,  0,  -2,  100,  0,  0,  -1,  -1,  -3,  0,  0,  0,  -1,  -1,  0,  0,  -3,  0,  -1,  -1,0],\n",
        "        [-1,-1,  -2,  0, 0,  0,  -3,  -1,  0,  -2,  0,  0,  0,  -1,  0,  -1,  0,  -2,  -1,  0,0],\n",
        "        [0,  0,  -1,  0,  0,  0,  0,  -1,  -1,  -1,  0, 0,  100,  0,  0,  0,  0,  50,  -2,  0,0],\n",
        "        [0,  0,  0,  0,  -2,  0,  0,  0,  0,  0,  0,  0,  -1,  50, -2,  0,  0,  -1,  -1,  0,0],\n",
        "        [-2, 0,  200,  -2,  -2,  300,  0, 0,  -2,  -2,  0,  0,  -3,  0,  -1,  0,  0,  -3,  -1,  0,0],\n",
        "        [0,  -1,  0,  0,  0,  0,  0,  -3,  0,  0,  -1,  -1,  0,  0,  0,  0,  0,  0,  -2,  0,0],\n",
        "        [0,  -1,  -1,  0,  0,  -1,  -1,  0,  0,  700,  -1,  0,  0,  0,  -2,  -1,  -1,  0,  0, 0,100],\n",
        "        [0,  0, 0, 500,  0,  0,  -1,  0,  -2,  -2,  -1,  -1,  0,  0,  -2,  0,  -3,  0,  0,  -1,0],\n",
        "        [-1,  -1, 0,-2 ,  0,  -1,  -2,  0,  400,  -2,  -1,  -1,  500,  0,  -2,  0,  -3,  100,  0, 0,0]\n",
        "      ]\n",
        "      map2 = [\n",
        "        [0,  0,  -2,  0,  0,  0,  -1,  -1,  -3,  0,  0,  0,  -1,  -1,  0,  0,  -3,  0,  -1,  -1,0],\n",
        "        [-1,-1,  -2,  100, 0,  0,  -3,  -1,  0,  -2,  100,  0,  0,  -1,  0,  -1,  0,  -2,  -1,  0,0],\n",
        "        [0,  0,  -1,  0,  0,  0,  0,  -1,  -1,  -1,  0, 0,  0,  0,  0,  0,  50,  0,  -2,  0,0],\n",
        "        [0,  200,  0,  0,  -2,  0,  0,  0,  0,  0,  0,  0,  -1,  50, -2,  0,  0,  -1,  -1,  0,0],\n",
        "        [-2, 0,  0,  -2,  -2,  0,  0, 0,  -2,  -2,  0,  0,  -3,  0,  -1,  0,  0,  -3,  -1,  0,0],\n",
        "        [0,  -1,  0,  0,  300,  0,  0,  -3,  0,  0,  -1,  -1,  0,  0,  0,  0,  0,  0,  -2,  0,0],\n",
        "        [500,  -1,  -1,  0,  0,  -1,  -1,  0,  700,  0,  -1,  0,  0,  0,  -2,  -1,  -1,  0,  0, 0,0],\n",
        "        [0,  0, 0, 0,  0,  0,  -1,  0,  -2,  -2,  -1,  -1,  0,  0,  -2,  0,  -3,  100,  0,  -1,0],\n",
        "        [-1,  -1, 0,-2 ,  0,  -1,  -2,  400,  0,  -2,  -1,  -1,  0,  500,  -2,  0,  -3,  0,  0, 100,0]\n",
        "      ]\n",
        "      map3= [\n",
        "        [0,  0,  -2,  0,  0,  0,  -1,  -1,  -3,  0,  100,  0,  -1,  -1,  0,  0,  -3,  0,  -1,  -1,0],\n",
        "        [-1,-1,  -2,  0, 0,  0,  -3,  -1,  0,  -2,  0,  0,  0,  -1,  0,  -1,  0,  -2,  -1,  0,0 ],\n",
        "        [0,  0,  -1,  0,  0,  0,  100,  -1,  -1,  -1,  0, 0,  50,  0,  0,  0,  50,  0,  -2,  0,0],\n",
        "        [0,  200,  0,  0,  -2,  0,  0,  0,  0,  0,  0,  0,  -1,  0, -2,  0,  0,  -1,  -1,  0,0],\n",
        "        [-2, 0,  0,  -2,  -2,  0,  0, 0,  -2,  -2,  0,  0,  -3,  0,  -1,  0,  0,  -3,  -1,  0,0],\n",
        "        [0,  -1,  0, 300,  0,  0,  0,  -3,  0,  0,  -1,  -1,  0,  0,  0,  0,  0,  0,  -2,  0,0],\n",
        "        [0,  -1,  -1,  0,  0,  -1,  -1,  700,  0,  0,  -1,  0,  0,  0,  -2,  -1,  -1,  0,  0, 0,0],\n",
        "        [0,  0, 0, 0,  0,  500,  -1,  0,  -2,  -2,  -1,  -1,  0,  0,  -2,  0,  -3,  0,  700,  -1,0],\n",
        "        [-1,  -1, 0,-2 ,  0,  -1,  -2,  400,  0,  -2,  -1,  -1,  0,  500,  -2,  0,  -3,  0,  0, 100,0]\n",
        "      ]\n",
        "      map4=[\n",
        "        [0,  0,  -2,  0,  0,  0,  -1,  -1,  -3,  0,  0,  0,  -1,  -1,  0,  0,  -3,  0,  -1,  -1,0],\n",
        "        [-1,-1,  -2,  0, 0,  0,  -3,  -1,  0,  -2,  0,  0,  100,  -1,  0,  -1,  0,  -2,  -1,  0,0],\n",
        "        [0,  0,  -1,  0,  100,  0,  0,  -1,  -1,  -1,  0, 0,  0,  0,  50,  0,  50,  0,  -2,  0,0],\n",
        "        [0,  200,  0,  0,  -2,  0,  0,  0,  0,  0,  0,  0,  -1,  0, -2,  0,  0,  -1,  -1,  0,0],\n",
        "        [-2, 0,  0,  -2,  -2,  0,  0, 0,  -2,  -2,  0,  0,  -3,  0,  -1,  0,  0,  -3,  -1,  0,0],\n",
        "        [0,  -1,  0,  0,  0,  0,  300,  -3,  0,  700,  -1,  -1,  0,  0,  0,  0,  0,  0,  -2,  0,0],\n",
        "        [0,  -1,  -1,  0,  0,  -1,  -1,  0,  0,  0,  -1,  0,  0,  0,  -2,  -1,  -1,  0,  0, 100,0],\n",
        "        [500,  0, 0, 0,  0,  0,  -1,  0,  -2,  -2,  -1,  -1,  0,  0,  -2,  0,  -3,  0,  0,  -1,0],\n",
        "        [-1,  -1, 0,-2 ,  0,  -1,  -2,  400,  0,  -2,  -1,  -1,  0,  500,  -2,  0,  -3,  0,  0, 100,0]\n",
        "\n",
        "      ]\n",
        "      map5=[\n",
        "        [0,  0,  -2,  0,  100,  0,  -1,  -1,  -3,  0,  0,  0,  -1,  -1,  0,  0,  -3,  0,  -1,  -1,0],\n",
        "        [-1,-1,  -2,  0, 0,  0,  -3,  -1,  0,  -2,  100,  0,  0,  -1,  0,  -1,  0,  -2,  -1,  0,0],\n",
        "        [0,  0,  -1,  0,  0,  0,  0,  -1,  -1,  -1,  0, 0,  0,  0,  50,  0,  0,  0,  -2,  0,0],\n",
        "        [0,  200,  0,  0,  -2,  0,  0,  0,  0,  0,  0,  0,  -1,  0, -2,  0,  50,  -1,  -1,  0,0],\n",
        "        [-2, 0,  0,  -2,  -2,  0,  0, 0,  -2,  -2,  0,  0,  -3,  0,  -1,  0,  0,  -3,  -1,  0,0],\n",
        "        [0,  -1,  0,  0,  300,  0,  0,  -3,  0,  0,  -1,  -1,  0,  0,  0,  0,  0,  0,  -2,  0,0],\n",
        "        [500,  -1,  -1,  0,  0,  -1,  -1,  0,  0,  700,  -1,  0,  0,  0,  -2,  -1,  -1,  0,  0, 100,0],\n",
        "        [0,  0, 0, 0,  0,  0,  -1,  0,  -2,  -2,  -1,  -1,  0,  0,  -2,  0,  -3,  0,  0,  -1,0],\n",
        "        [-1,  -1, 0,-2 ,  0,  -1,  -2,  400,  0,  -2,  -1,  -1,  0,  500,  -2,  0,  -3,  0,  0, 100,0]\n",
        "      ]\n",
        "      Maps = (map1,map2,map3,map4,map5)\n",
        "      return Maps   \n",
        "\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKN2xerQy-p8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7ef7de46-4787-456c-e8f2-228058511229"
      },
      "source": [
        "#DQN Algorithm-Main\n",
        "#Create header for saving DQN learning file\n",
        "'''now = datetime.datetime.now()\n",
        "header = [\"Ep\",\"Step\", \"Reward\",\"Total_reward\",\"Action\",\"Epsilon\",\"Done\",\"Termination_Code\"]\n",
        "filename = \"Data/data_\" + now.strftime(\"%Y%m%d-%H%M\") + \".csv\"\n",
        "with open(filename, 'w') as f:\n",
        "    pd.DataFrame(columns = header).to_csv(f,encoding='utf-8', index=False, header = True)'''\n",
        "\n",
        "# Parameters for training a DQN model\n",
        "N_EPISODE = 10000 #The number of episodes for training\n",
        "MAX_STEP = 1000   #The number of steps for each episode\n",
        "BATCH_SIZE = 32   #The number of experiences for each replay \n",
        "MEMORY_SIZE = 100000 #The size of the batch for storing experiences\n",
        "SAVE_NETWORK = 100  # After this number of episodes, the DQN model is saved for testing later. \n",
        "INITIAL_REPLAY_SIZE = 1000 #The number of experiences are stored in the memory batch before starting replaying\n",
        "INPUTNUM = 198 #The number of input values for the DQN model\n",
        "ACTIONNUM = 6  #The number of actions output from the DQN model\n",
        "MAP_MAX_X = 21 #Width of the Map\n",
        "MAP_MAX_Y = 9  #Height of the Map\n",
        "\n",
        "# Initialize network and memory\n",
        "DQNAgent = DQN(INPUTNUM,ACTIONNUM)\n",
        "memory = Memory(MEMORY_SIZE)\n",
        "\n",
        "# Initialize environment\n",
        "Maps = CreateMaps()#Creating 05 maps\n",
        "minerEnv = MinerEnv()#Creating a communication environment between the DQN model and the game environment\n",
        "minerEnv.start() #Connect to the game\n",
        "\n",
        "train = False #The variable is used to indicate that the replay starts, and the epsilon starts decrease.\n",
        "#Training Process\n",
        "#the main part of the deep-q learning agorithm \n",
        "for episode_i in range(0,N_EPISODE):\n",
        "    try:\n",
        "        # Choosing a map in the list\n",
        "        mapID = np.random.randint(0, 5) #Choosing a map ID from 5 maps in Maps folder randomly\n",
        "        posID_x = np.random.randint(MAP_MAX_X) #Choosing a initial position of the DQN agent on X-axes randomly\n",
        "        posID_y = np.random.randint(MAP_MAX_Y) #Choosing a initial position of the DQN agent on Y-axes randomly\n",
        "        #Creating a request for initializing a map, initial position, the initial energy, and the maximum number of steps of the DQN agent\n",
        "        request = (\"map\" + str(mapID) + \",\" + str(posID_x) + \",\" + str(posID_y) + \",50,100\") \n",
        "        #Send the request to the game environment\n",
        "        minerEnv.send_map_info(request)\n",
        "\n",
        "        # Getting the initial state\n",
        "        minerEnv.reset() #Initialize the game environment\n",
        "        s = minerEnv.get_state()#Get the state after reseting. \n",
        "                                #This function (get_state()) is an example of creating a state for the DQN model \n",
        "        total_reward = 0 #The amount of rewards for the entire episode\n",
        "        terminate = False #The variable indicates that the episode ends\n",
        "        maxStep = minerEnv.state.mapInfo.maxStep #Get the maximum number of steps for each episode in training\n",
        "        #Start an episde for training\n",
        "        for step in range(0, maxStep):\n",
        "            action = DQNAgent.act(s)  # Getting an action from the DQN model from the state (s)\n",
        "            minerEnv.step(str(action))  # Performing the action in order to obtain the new state\n",
        "            s_next = minerEnv.get_state()  # Getting a new state\n",
        "            reward = minerEnv.get_reward()  # Getting a reward\n",
        "            terminate = minerEnv.check_terminate()  # Checking the end status of the episode\n",
        "             \n",
        "            # Add this transition to the memory batch\n",
        "            memory.push(s, action, reward, terminate, s_next)\n",
        "\n",
        "            # Sample batch memory to train network\n",
        "            if (memory.length > INITIAL_REPLAY_SIZE):\n",
        "                #If there are INITIAL_REPLAY_SIZE experiences in the memory batch\n",
        "                #then start replaying\n",
        "                batch = memory.sample(BATCH_SIZE) #Get a BATCH_SIZE experiences for replaying\n",
        "                DQNAgent.replay(batch, BATCH_SIZE)#Do relaying\n",
        "                train = True #Indicate the training starts\n",
        "            total_reward = total_reward + reward #Plus the reward to the total rewad of the episode\n",
        "            s = s_next #Assign the next state for the next step.\n",
        "            \n",
        "            #Saving data to file\n",
        "            '''save_data = np.hstack([episode_i+1,step+1,reward,total_reward,action, DQNAgent.epsilon, terminate]).reshape(1,7)\n",
        "            with open(filename, 'a') as f:\n",
        "                pd.DataFrame(save_data).to_csv(f,encoding='utf-8', index=False, header = False)'''\n",
        "\n",
        "            if terminate == True:\n",
        "                #If the episode ends, then go to the next episode\n",
        "                break\n",
        "            \n",
        "        # Iteration to save the network architecture and weights\n",
        "        if (np.mod(episode_i + 1, SAVE_NETWORK) == 0 and train == True):\n",
        "            DQNAgent.target_train()  # Replace the learning weights for target model with soft replacement\n",
        "            #Save the DQN model\n",
        "            now = datetime.datetime.now() #Get the latest datetime          \n",
        "            DQNAgent.save_model( \"DQNmodel_\" + now.strftime(\"%Y%m%d-%H%M\") + \"_ep\" + str(episode_i+1))   \n",
        "        \n",
        "        #Print the training information after the episode\n",
        "        print('Episode %d ends. Number of steps is: %d. Accumlated Reward = %.2f. Epsilon = %.2f .Termination code: %d' % (episode_i+1, step+1, total_reward, DQNAgent.epsilon, terminate))\n",
        "        #Decreasing the epsilon if the replay starts\n",
        "        if  train == True:\n",
        "            DQNAgent.update_epsilon()\n",
        "            \t\n",
        "    except Exception as e:\n",
        "        import traceback\n",
        "        traceback.print_exc()                \n",
        "        #print(\"Finished.\")\n",
        "        break"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Connected to server.\n",
            "Found: map0\n",
            "Found: map1\n",
            "Found: map2\n",
            "Found: map3\n",
            "Found: map4\n",
            "Reset game:  ['map0', '5', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 1 ends. Number of steps is: 7. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '12', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 2 ends. Number of steps is: 1. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '4', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 3 ends. Number of steps is: 28. Accumlated Reward = -15.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '10', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 4 ends. Number of steps is: 6. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '4', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 5 ends. Number of steps is: 17. Accumlated Reward = 33.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '12', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 6 ends. Number of steps is: 13. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '1', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 7 ends. Number of steps is: 10. Accumlated Reward = -15.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '1', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 8 ends. Number of steps is: 3. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '18', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 9 ends. Number of steps is: 15. Accumlated Reward = -17.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '0', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 10 ends. Number of steps is: 5. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '14', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 11 ends. Number of steps is: 50. Accumlated Reward = -19.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '19', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 12 ends. Number of steps is: 5. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '1', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 13 ends. Number of steps is: 17. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '18', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 14 ends. Number of steps is: 5. Accumlated Reward = -18.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '2', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 15 ends. Number of steps is: 4. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '19', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 16 ends. Number of steps is: 7. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '10', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 17 ends. Number of steps is: 32. Accumlated Reward = -24.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '3', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 18 ends. Number of steps is: 20. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '20', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 19 ends. Number of steps is: 2. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '5', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 20 ends. Number of steps is: 13. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '3', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 21 ends. Number of steps is: 8. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '9', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 22 ends. Number of steps is: 14. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '19', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 23 ends. Number of steps is: 8. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '10', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 24 ends. Number of steps is: 10. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '11', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 25 ends. Number of steps is: 3. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '15', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 26 ends. Number of steps is: 23. Accumlated Reward = -27.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '17', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 27 ends. Number of steps is: 4. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '6', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 28 ends. Number of steps is: 14. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '14', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 29 ends. Number of steps is: 10. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '20', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 30 ends. Number of steps is: 5. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '19', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 31 ends. Number of steps is: 5. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '1', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 32 ends. Number of steps is: 4. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '18', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 33 ends. Number of steps is: 13. Accumlated Reward = -18.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '16', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 34 ends. Number of steps is: 6. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '0', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 35 ends. Number of steps is: 2. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '0', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 36 ends. Number of steps is: 26. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '9', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 37 ends. Number of steps is: 19. Accumlated Reward = 90.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '18', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 38 ends. Number of steps is: 2. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '0', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 39 ends. Number of steps is: 11. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '6', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 40 ends. Number of steps is: 14. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '15', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 41 ends. Number of steps is: 14. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '18', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 42 ends. Number of steps is: 14. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '12', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 43 ends. Number of steps is: 13. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '2', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 44 ends. Number of steps is: 8. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '7', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 45 ends. Number of steps is: 10. Accumlated Reward = -20.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '3', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 46 ends. Number of steps is: 3. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '3', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 47 ends. Number of steps is: 9. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '16', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 48 ends. Number of steps is: 12. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '15', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 49 ends. Number of steps is: 3. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '8', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 50 ends. Number of steps is: 33. Accumlated Reward = 38.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '4', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 51 ends. Number of steps is: 16. Accumlated Reward = -23.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '15', '4', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 52 ends. Number of steps is: 13. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '20', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 53 ends. Number of steps is: 4. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '9', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 54 ends. Number of steps is: 11. Accumlated Reward = -14.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '13', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 55 ends. Number of steps is: 7. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '9', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 56 ends. Number of steps is: 5. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '1', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 57 ends. Number of steps is: 5. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '15', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 58 ends. Number of steps is: 13. Accumlated Reward = -18.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '5', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 59 ends. Number of steps is: 8. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '8', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 60 ends. Number of steps is: 17. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '14', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 61 ends. Number of steps is: 6. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '20', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 62 ends. Number of steps is: 10. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '3', '7', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 63 ends. Number of steps is: 23. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '0', '0', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 64 ends. Number of steps is: 1. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '8', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 65 ends. Number of steps is: 8. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '13', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 66 ends. Number of steps is: 19. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '18', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 67 ends. Number of steps is: 6. Accumlated Reward = -22.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '15', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 68 ends. Number of steps is: 6. Accumlated Reward = -19.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '17', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 69 ends. Number of steps is: 40. Accumlated Reward = -25.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '7', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 70 ends. Number of steps is: 51. Accumlated Reward = -13.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '19', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 71 ends. Number of steps is: 10. Accumlated Reward = -16.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '17', '6', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 72 ends. Number of steps is: 12. Accumlated Reward = -24.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '7', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 73 ends. Number of steps is: 27. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '1', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 74 ends. Number of steps is: 11. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map1', '13', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 75 ends. Number of steps is: 22. Accumlated Reward = 10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '17', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 76 ends. Number of steps is: 1. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map3', '9', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 77 ends. Number of steps is: 26. Accumlated Reward = -18.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '8', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 78 ends. Number of steps is: 13. Accumlated Reward = -10.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map2', '6', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 79 ends. Number of steps is: 18. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '11', '5', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 80 ends. Number of steps is: 11. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '13', '8', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 81 ends. Number of steps is: 4. Accumlated Reward = 40.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '5', '2', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 82 ends. Number of steps is: 8. Accumlated Reward = -15.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map4', '16', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 83 ends. Number of steps is: 7. Accumlated Reward = -5.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '19', '3', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 84 ends. Number of steps is: 11. Accumlated Reward = -12.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '10', '1', '50', '100']\n",
            "numberOfPlayers:  4\n",
            "Episode 85 ends. Number of steps is: 12. Accumlated Reward = -11.00. Epsilon = 1.00 .Termination code: 1\n",
            "Reset game:  ['map0', '3', '5', '50', '100']\n",
            "numberOfPlayers:  4\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}