park_now.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "park_now.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ankit1khare/Easy_street_parking_with_MASK-RCNN/blob/master/park_now.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "xRtTTRR-IZDh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "34f4e6db-5d13-4cad-f023-5f7f68768511"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xsw7UKpUFj-P",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# !pip install jupyterlab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Tia0L7qmGmgt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# !npm config set prefix /usr/local"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_S3IQsrSFrh4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# !npm install localtunnel\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lPlbXThWFwAw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# jupyter_running = !jupyter notebook list | grep 8888\n",
        "# if not jupyter_running:\n",
        "#   !mkdir -p /content/.jupyter\n",
        "#   !echo '{ \"NotebookApp\": { \"password\": \"4d6ed2c10da3aeaba70c40a26ed2e659b75e3dfd\" } }' > /content/.jupyter/jupyter_notebook_config.json\n",
        "#   get_ipython().system_raw('jupyter lab &')\n",
        "# !ssh -o ServerAliveInterval=60 -o StrictHostKeyChecking=no -R deviprasadtripathy:80:localhost:8888 serveo.net 1>/dev/null"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vHUNxm5VFnf3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# from IPython.lib import passwd\n",
        "# password = passwd(\"secret\")\n",
        "# password"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YpoGpJ1ALSfD",
        "colab_type": "code",
        "outputId": "7ab6052c-a14b-43cd-eeeb-75cc1c3aead8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        }
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/matterport/Mask_RCNN.git"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Mask_RCNN'...\n",
            "remote: Enumerating objects: 923, done.\u001b[K\n",
            "remote: Total 923 (delta 0), reused 0 (delta 0), pack-reused 923\u001b[K\n",
            "Receiving objects: 100% (923/923), 139.77 MiB | 11.17 MiB/s, done.\n",
            "Resolving deltas: 100% (521/521), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ghN6PD0zR7dw",
        "colab_type": "code",
        "outputId": "a01e61df-81b5-479c-ebe9-d90fbef2a843",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        }
      },
      "cell_type": "code",
      "source": [
        "!pip3 install twilio"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting twilio\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/79/04/93727ff416f0661ad8f97327a2c47ca94d31e985db81c43c4eb45e7c344a/twilio-6.24.0-py2.py3-none-any.whl (948kB)\n",
            "\u001b[K    100% |████████████████████████████████| 952kB 21.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.0.0; python_version >= \"3.0\" in /usr/local/lib/python3.6/dist-packages (from twilio) (2.18.4)\n",
            "Requirement already satisfied: pysocks; python_version >= \"3.0\" in /usr/local/lib/python3.6/dist-packages (from twilio) (1.6.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from twilio) (1.11.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from twilio) (2018.9)\n",
            "Collecting PyJWT>=1.4.2 (from twilio)\n",
            "  Downloading https://files.pythonhosted.org/packages/87/8b/6a9f14b5f781697e51259d81657e6048fd31a113229cf346880bb7545565/PyJWT-1.7.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0; python_version >= \"3.0\"->twilio) (1.22)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0; python_version >= \"3.0\"->twilio) (2.6)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0; python_version >= \"3.0\"->twilio) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0; python_version >= \"3.0\"->twilio) (2018.11.29)\n",
            "Installing collected packages: PyJWT, twilio\n",
            "Successfully installed PyJWT-1.7.1 twilio-6.24.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "jS4wbKWTL42_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"/content/drive/My Drive/AK49/Mask_RCNN/\")\n",
        "# os.chdir(\"..\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "87-fJea6aqVa",
        "colab_type": "code",
        "outputId": "de6b3d52-96e7-4c5e-8290-3200787a8d0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import cv2\n",
        "import mrcnn.config\n",
        "import mrcnn.utils\n",
        "from mrcnn.model import MaskRCNN\n",
        "from pathlib import Path\n",
        "from twilio.rest import Client"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "wv6XJBd_bzv5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import matplotlib as pt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pvYO3F9g-j0X",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# !rm -r ak\n",
        "# !mkdir ak"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jQ9SvpJXHvua",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Mask-RCNN config\n",
        "class MaskRCNNConfig(mrcnn.config.Config):\n",
        "    NAME = \"coco_pretrained_model_config\"\n",
        "    IMAGES_PER_GPU = 1\n",
        "    GPU_COUNT = 1\n",
        "    NUM_CLASSES = 1 + 80  \n",
        "    DETECTION_MIN_CONFIDENCE = 0.6 #setted to 60%\n",
        "\n",
        "\n",
        "# Filter to only cars\n",
        "def get_car_boxes(boxes, class_ids):\n",
        "    car_boxes = []\n",
        "\n",
        "    for i, box in enumerate(boxes):\n",
        "        if class_ids[i] in [3, 8, 6]:\n",
        "            car_boxes.append(box)\n",
        "\n",
        "    return np.array(car_boxes)\n",
        "\n",
        "# Twilio config\n",
        "twilio_account_sid = 'aaaaaaaaaaaaaaaaaaaaaaa'\n",
        "twilio_auth_token = 'aaaaaaaaaaaaaaaaaaa93113efe'\n",
        "twilio_phone_number = '+199999999'\n",
        "destination_phone_number = '+19999999999'\n",
        "client = Client(twilio_account_sid, twilio_auth_token)\n",
        "\n",
        "\n",
        "# Root dir\n",
        "ROOT_DIR = Path(\".\")\n",
        "\n",
        "#Trained model loc\n",
        "MODEL_DIR = os.path.join(ROOT_DIR, \"logs\")\n",
        "\n",
        "COCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n",
        "\n",
        "if not os.path.exists(COCO_MODEL_PATH):\n",
        "    mrcnn.utils.download_trained_weights(COCO_MODEL_PATH)\n",
        "\n",
        "VIDEO_SOURCE = \"test_images/now.mp4\"\n",
        "\n",
        "model = MaskRCNN(mode=\"inference\", model_dir=MODEL_DIR, config=MaskRCNNConfig())\n",
        "\n",
        "model.load_weights(COCO_MODEL_PATH, by_name=True)\n",
        "\n",
        "# spotted parking spaces\n",
        "parked_car_boxes = None\n",
        "\n",
        "video_capture = cv2.VideoCapture(VIDEO_SOURCE)\n",
        "\n",
        "free_space_frames = 0\n",
        "sms_sent = False\n",
        "count = 0\n",
        "temp = np.array(4,)\n",
        "\n",
        "parked_car_boxes1 = [None] * 11\n",
        "\n",
        "def checkEqual2(iterator):\n",
        "   print(iterator)\n",
        "#     return len(set(iterator)) <= 1\n",
        "  \n",
        "# Loop over each frame in the video\n",
        "while video_capture.isOpened():\n",
        "    success, frame = video_capture.read()\n",
        "    if not success:\n",
        "        print(\"couldn't read video\")\n",
        "        break\n",
        "\n",
        "    # Converting the image from BGR color used by OpenCV to RGB color\n",
        "    rgb_image = frame[:, :, ::-1]\n",
        "\n",
        "    results = model.detect([rgb_image], verbose=0)\n",
        "\n",
        "    # Mask R-CNN assumes we are running detection on multiple images.\n",
        "    # We only passed in one image to detect, so only grab the first result.\n",
        "    r = results[0]\n",
        "\n",
        "    # The r variable will now have the results of detection:\n",
        "    # - r['rois'] are the bounding box of each detected object\n",
        "    # - r['class_ids'] are the class id (type) of each detected object\n",
        "    # - r['scores'] are the confidence scores for each detection\n",
        "    # - r['masks'] are the object masks for each detected object (which gives you the object outline)\n",
        "    if parked_car_boxes is None:\n",
        "        # This is the first frame of video - assume all the cars detected are in parking spaces.\n",
        "        # Save the location of each car as a parking space box and go to the next frame of video.\n",
        "        parked_car_boxes = get_car_boxes(r['rois'], r['class_ids'])\n",
        "\n",
        "    else:\n",
        "        # Get where cars are currently located in the frame\n",
        "        car_boxes = get_car_boxes(r['rois'], r['class_ids'])\n",
        "\n",
        "        # See how much those cars overlap with the known parking spaces\n",
        "        overlaps = mrcnn.utils.compute_overlaps(parked_car_boxes, car_boxes)\n",
        "\n",
        "        # Assume no spaces are free until we find one that is free\n",
        "        free_space = False\n",
        "\n",
        "        # Loop through each known parking space box\n",
        "        for parking_area, overlap_areas in zip(parked_car_boxes, overlaps):\n",
        "\n",
        "            # For this parking space, find the max amount it was covered by any\n",
        "            # car that was detected in our image (doesn't really matter which car)\n",
        "            max_IoU_overlap = np.max(overlap_areas)\n",
        "\n",
        "            # Get the top-left and bottom-right coordinates of the parking area\n",
        "            y1, x1, y2, x2 = parking_area\n",
        "\n",
        "            # Check if the parking space is occupied by seeing if any car overlaps\n",
        "            # it by more than 0.15 using IoU\n",
        "            if max_IoU_overlap < 0.15:\n",
        "                # Parking space not occupied! Draw a green box around it\n",
        "                cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 3)\n",
        "                # Flag that we have seen at least one open space\n",
        "                free_space = True\n",
        "            else:\n",
        "                # Parking space is still occupied - draw a red box around it\n",
        "                cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 255), 1)\n",
        "\n",
        "            # Write the IoU measurement inside the box\n",
        "            font = cv2.FONT_HERSHEY_DUPLEX\n",
        "            cv2.putText(frame, f\"{max_IoU_overlap:0.2}\", (x1 + 6, y2 - 6), font, 0.3, (255, 255, 255))\n",
        "\n",
        "        # If at least one space was free, start counting frames\n",
        "        # This is so we don't alert based on one frame of a spot being open.\n",
        "        # This helps prevent the script triggered on one bad detection.\n",
        "        if free_space:\n",
        "            free_space_frames += 1\n",
        "        else:\n",
        "            # If no spots are free, reset the count\n",
        "            free_space_frames = 0\n",
        "\n",
        "        # If a space has been free for several frames, we are pretty sure it is really free!\n",
        "        if free_space_frames > 190:\n",
        "            # Write SPACE AVAILABLE!! at the top of the screen\n",
        "            font = cv2.FONT_HERSHEY_DUPLEX\n",
        "            cv2.putText(frame, f\"SPACE AVAILABLE!\", (10, 150), font, 3.0, (0, 255, 0), 2, cv2.FILLED)\n",
        "            \n",
        "            # If we haven't sent an SMS yet, sent it!\n",
        "            if not sms_sent:\n",
        "                print(\"SENDING SMS!!!\")\n",
        "                message = client.messages.create(\n",
        "                    body=\"Parking space available now!\",\n",
        "                    from_=twilio_phone_number,\n",
        "                    to=destination_phone_number\n",
        "                )\n",
        "                sms_sent = True\n",
        "                print(\"Hope you got the message on your phone\")\n",
        "\n",
        "        # Show the frame of video on the screen\n",
        "#         cv2.imshow('Video', frame)\n",
        "    #saving each frame\n",
        "    name = str(count) + \".jpg\"\n",
        "    name = os.path.join('./ak', name)\n",
        "    cv2.imwrite(name, frame)\n",
        "    count+=1\n",
        "    \n",
        "    #'q' to quit\n",
        "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
        "        break\n",
        "\n",
        "# Clean up\n",
        "print(\"Video finished\")\n",
        "video_capture.release()\n",
        "# cv2.destroyAllWindows()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HlhAEzjcXeED",
        "colab_type": "code",
        "outputId": "99d8b9bc-ab96-4873-d8cb-f88ea670cedf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "#create video including all the frames in the ak folder\n",
        "import glob\n",
        "\n",
        "images = list(glob.iglob(os.path.join('./ak', '*.*')))\n",
        "images = sorted(images, key=lambda x: float(os.path.split(x)[1][:-3]))\n",
        "\n",
        "# Get all image file paths to a list.\n",
        "# Sort the images by name index.\n",
        "# images = sorted(images, key=lambda x: float(os.path.split(x)[1][:-3]))\n",
        "\n",
        "def make_video(outvid, images=None, fps=30, size=None,\n",
        "               is_color=True, format=\"FMP4\"):\n",
        "    \"\"\"\n",
        "    Create a video from a list of images.\n",
        " \n",
        "    @param      outvid      output video\n",
        "    @param      images      list of images to use in the video\n",
        "    @param      fps         frame per second\n",
        "    @param      size        size of each frame\n",
        "    @param      is_color    color\n",
        "    @param      format      see http://www.fourcc.org/codecs.php\n",
        "    @return                 see http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_gui/py_video_display/py_video_display.html\n",
        "    \"\"\"\n",
        "    from cv2 import VideoWriter, VideoWriter_fourcc, imread, resize\n",
        "    fourcc = VideoWriter_fourcc(*format)\n",
        "    vid = None\n",
        "    for image in images:\n",
        "        if not os.path.exists(image):\n",
        "            raise FileNotFoundError(image)\n",
        "        img = imread(image)\n",
        "        if vid is None:\n",
        "            if size is None:\n",
        "                size = img.shape[1], img.shape[0]\n",
        "            vid = VideoWriter(outvid, fourcc, float(fps), size, is_color)\n",
        "        if size[0] != img.shape[1] and size[1] != img.shape[0]:\n",
        "            img = resize(img, size)\n",
        "        vid.write(img)\n",
        "    vid.release()\n",
        "    return vid\n",
        "  \n",
        "make_video('./vidout.mp4', images, fps=30)  \n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<VideoWriter 0x7f31190929f0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "metadata": {
        "id": "5uKtp4YvZtzP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# !cp -a Mask_RCNN/ drive/My\\ Drive/AK49/"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}