2D_convolution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "colab": {
      "name": "2D_convolution (1).ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "disturbed-arlington"
      },
      "source": [
        "# 2D convolution\n",
        "\n",
        "In this notebook a convolution applied to an image is demonstrated.<br> With the [opencv](https://docs.opencv.org) library, image processing can be done in python.\n",
        "\n",
        "A filter operation is a convolution of the image represented as matrix of numbers and a filter. There are several filter types available in the library. <br> In this notebook the the sobel filter as manual computing is presented.\n",
        "\n",
        "A detailed description to the Sobel filter with opencv can be found [here](https://docs.opencv.org/3.4/d2/d2c/tutorial_sobel_derivatives.html)."
      ],
      "id": "disturbed-arlington"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mature-collector"
      },
      "source": [
        "# load opencv library\n",
        "#\n",
        "import cv2"
      ],
      "id": "mature-collector",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "defined-gambling"
      },
      "source": [
        "# load numpy for matrix operations\n",
        "#\n",
        "import numpy as np"
      ],
      "id": "defined-gambling",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "committed-collective"
      },
      "source": [
        "# pyplot for displaying images\n",
        "#\n",
        "from matplotlib import pyplot as plt"
      ],
      "id": "committed-collective",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "located-kruger",
        "outputId": "3dec78b0-db8a-4b63-d11d-9426a3affca2"
      },
      "source": [
        "# load picture from pexels int current environment\n",
        "#\n",
        "# \n",
        "!curl -O --location \"https://images.pexels.com/photos/6243722/pexels-photo-6243722.jpeg?auto=compress&cs=tinysrgb&dpr=1&w=500\""
      ],
      "id": "located-kruger",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 96952  100 96952    0     0   265k      0 --:--:-- --:--:-- --:--:--  265k\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "controversial-moisture",
        "outputId": "2abcdc89-9aba-4b1f-a07c-612e65e357f6"
      },
      "source": [
        "# load original image\n",
        "\n",
        "# hint: check the environemnt for the correct image name\n",
        "image_1 = cv2.imread('pexels-photo-6243722.jpeg?auto=compress&cs=tinysrgb&dpr=1&w=500')\n",
        "\n",
        "# display shape of image_1\n",
        "#\n",
        "# nx: number of pixels in x-direction\n",
        "# ny: number of pixels in y-direction\n",
        "# nc: number of color channels (RGB = 3)\n",
        "\n",
        "[nx, ny, nc] = image_1.shape\n",
        "\n",
        "print(f'x: {nx}\\ty: {ny}\\tcolors: {nc}')"
      ],
      "id": "controversial-moisture",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x: 750\ty: 500\tcolors: 3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "democratic-testing"
      },
      "source": [
        "# !Note\n",
        "# Color image loaded by OpenCV is in BGR mode. But Matplotlib displays in RGB mode. \n",
        "# So color images will not be displayed correctly in Matplotlib if image is read with OpenCV. \n",
        "#\n",
        "# see https://stackoverflow.com/questions/15072736/extracting-a-region-from-an-image-using-slicing-in-python-opencv/15074748#15074748\n",
        "\n",
        "# convert BGR color image to RGB color image \n",
        "b,g,r = cv2.split(image_1)\n",
        "img_work = cv2.merge([r,g,b])"
      ],
      "id": "democratic-testing",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "competitive-folks"
      },
      "source": [
        "# use resize only if image is too big\n",
        "#\n",
        "# resize image to 600 x 600 px\n",
        "#\n",
        "# original_size = image_1.shape\n",
        "#\n",
        "# create ratio: new width / original width for a correct resizing\n",
        "#\n",
        "# r = 600 / original_size[1]\n",
        "# dim = (600, int(original_size[0]*r))\n",
        "#\n",
        "# img1_work = cv2.resize(image_1, dim)"
      ],
      "id": "competitive-folks",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "welcome-iraqi"
      },
      "source": [
        "### sobel filter\n",
        "The Sobel Operator is a discrete differentiation operator. It computes an approximation of the gradient of an image intensity function.\n",
        "The Sobel Operator combines Gaussian smoothing and differentiation. For example: sobel filtering with a kernel size of $3$ and  an image ***I*** .\n",
        "\n",
        "calculate two deviations:\n",
        "$$G_x = \\begin{bmatrix} -1 & 0 & +1  \\\\ -2 & 0 & +2 \\\\ -1 & 0 & +1 \\end{bmatrix} *  I$$\n",
        "\n",
        "$$G_y = \\begin{bmatrix} -1 & -2 & -1  \\\\ 0 & 0 & 0 \\\\ +1 & +2 & +1 \\end{bmatrix} * I$$\n",
        "\n",
        "At each point of the image we calculate an approximation of the gradient in that point by combining both results above:\n",
        "$$G = \\sqrt{G_x^2 + G_y^2} $$"
      ],
      "id": "welcome-iraqi"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "printable-brook"
      },
      "source": [
        "# gray image at first\n",
        "#\n",
        "# get color channels\n",
        "r_ch, g_ch, b_ch = img_work[:, :, 0], img_work[:, :, 1], img_work[:, :, 2]\n",
        "\n",
        "# take weights according to the opencv conversion documentation\n",
        "# to convert the color image to grayscale\n",
        "#\n",
        "# see: https://docs.opencv.org/3.4/de/d25/imgproc_color_conversions.html\n",
        "\n",
        "r_const = 0.299 \n",
        "g_const = 0.587 \n",
        "b_const = 0.114 \n",
        "\n",
        "grayscale_image =  b_const * b_ch + g_const * g_ch + r_const * r_ch"
      ],
      "id": "printable-brook",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tribal-notion"
      },
      "source": [
        "# define the Sobel operators as numpy matrices \n",
        "\n",
        "Sx = np.array([\n",
        "    [1.0, 0.0, -1.0],\n",
        "    [2.0, 0.0, -2.0],\n",
        "    [1.0, 0.0, -1.0]\n",
        "    ])\n",
        "\n",
        "Sy = np.array([\n",
        "    [1.0, 2.0, 1.0],\n",
        "    [0.0, 0.0, 0.0],\n",
        "    [-1.0, -2.0, -1.0]\n",
        "    ])\n"
      ],
      "id": "tribal-notion",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adaptive-prefix"
      },
      "source": [
        "# compute sobel filter output \n",
        "\n",
        "# pre-allocate arrays\n",
        "[nx, ny] = np.shape(grayscale_image)\n",
        "Gy = np.zeros(shape=(nx, ny))\n",
        "Gx = np.zeros(shape=(nx, ny))\n",
        "gradient = np.zeros(shape=(nx, ny))\n",
        "\n",
        "# kernel size\n",
        "k = 3\n",
        "\n",
        "for i in range(rows - 2):\n",
        "    for j in range(columns - 2):\n",
        "        gx = np.sum(np.multiply(Sx, grayscale_image[i:i + k, j:j + k]))  # x direction\n",
        "        gy = np.sum(np.multiply(Sy, grayscale_image[i:i + k, j:j + k]))  # y direction\n",
        "\n",
        "        Gx[i + 1, j + 1] = gx  # vertical edges\n",
        "        Gy[i + 1, j + 1] = gy  # horizontal edges\n",
        "        gradient[i + 1, j + 1] = np.sqrt(gx ** 2 + gy ** 2)\n"
      ],
      "id": "adaptive-prefix",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "final-halloween"
      },
      "source": [
        "# display images\n",
        "#\n",
        "fig, axes = plt.subplots(1,2, figsize=(10,10))\n",
        "\n",
        "plt.subplot(1,2,1),plt.imshow(img_work)\n",
        "plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "plt.subplot(1,2,2),plt.imshow(grayscale_image, 'gray')\n",
        "plt.title('Grayscale'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "plt.subplots(1,3, figsize=(10,10))\n",
        "\n",
        "plt.subplot(1,3,1),plt.imshow(Gx, 'gray')\n",
        "plt.title('Sobel_X'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "plt.subplot(1,3,2),plt.imshow(Gy, 'gray')\n",
        "plt.title('Sobel_Y'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "plt.subplot(1,3,3),plt.imshow(gradient, 'gray')\n",
        "plt.title('Sobel'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "plt.show()"
      ],
      "id": "final-halloween",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "communist-mining"
      },
      "source": [
        "### Alternative Sobel filtering with OpenCV library"
      ],
      "id": "communist-mining"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "corporate-interface"
      },
      "source": [
        "##### gaussian filter\n",
        "Gaussian filter blurs an image and reduces noise.<br>\n",
        "[GaussianBlur()](https://docs.opencv.org/3.4/d4/d86/group__imgproc__filter.html#gaabe8c836e97159a9193fb0b11ac52cf1)<br>\n",
        "[GaussianKernel](https://docs.opencv.org/3.4/d4/d86/group__imgproc__filter.html#gac05a120c1ae92a6060dd0db190a61afa)"
      ],
      "id": "corporate-interface"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "monetary-cologne"
      },
      "source": [
        "# # Remove noise by blurring with a Gaussian filter ( kernel size = 3 )\n",
        "# #\n",
        "# gs_kernel_size = (3,3)\n",
        "# sigma_X = 0\n",
        "# sigma_Y = 0 #(default), see GaussianKernel, https://docs.opencv.org/3.4/d4/d86/group__imgproc__filter.html#gac05a120c1ae92a6060dd0db190a61afa\n",
        "# #\n",
        "# blur_gs = cv2.GaussianBlur(img1_work, gs_kernel_size, sigma_X)\n",
        "# #"
      ],
      "id": "monetary-cologne",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "solid-return"
      },
      "source": [
        "# # sobel parameters\n",
        "# #\n",
        "# kernel_size = 3\n",
        "# dx = 1\n",
        "# dy = 1\n",
        "# #\n",
        "# sobelx = cv2.Sobel(img1_work, cv2.CV_64F, dx, dy, ksize=kernel_size)\n",
        "# sobely = cv2.Sobel(img1_work, cv2.CV_64F, dx, dy, ksize=kernel_size)\n",
        "# #"
      ],
      "id": "solid-return",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "closed-going"
      },
      "source": [
        "# # calculate absolute values, and converts the result to 8-bit.\n",
        "# #\n",
        "# abs_grad_x = cv2.convertScaleAbs(sobelx)\n",
        "# abs_grad_y = cv2.convertScaleAbs(sobely)\n",
        "# #"
      ],
      "id": "closed-going",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "incorporated-consistency"
      },
      "source": [
        "# # approximate the gradient by adding the single gradients\n",
        "# #\n",
        "# #\n",
        "# grad = cv2.addWeighted(abs_grad_x, 0.85, abs_grad_y, 0.85, 0)"
      ],
      "id": "incorporated-consistency",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blond-silly"
      },
      "source": [
        "# # display images\n",
        "# #\n",
        "# fig, axes = plt.subplots(1,2, figsize=(10,10))\n",
        "# #\n",
        "# plt.subplot(1,2,1),plt.imshow(img_work)\n",
        "# plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
        "# #\n",
        "# plt.subplot(1,2,2),plt.imshow(grayscale_image, 'gray')\n",
        "# plt.title('Grayscale'), plt.xticks([]), plt.yticks([])\n",
        "# #\n",
        "# plt.show()\n",
        "# #\n",
        "# #\n",
        "# plt.subplots(1,2, figsize=(10,10))\n",
        "# #\n",
        "# plt.subplot(1,2,1),plt.imshow(gx)\n",
        "# plt.title('Sobel X'), plt.xticks([]), plt.yticks([])\n",
        "# #\n",
        "# plt.subplot(1,2,2),plt.imshow(gy)\n",
        "# plt.title('Sobel Y'), plt.xticks([]), plt.yticks([])\n",
        "# #\n",
        "# plt.show()\n",
        "# #\n",
        "# plt.subplots(figsize=(10,10))\n",
        "# #\n",
        "# plt.subplot(1,2,1),plt.imshow(gradient, 'gray')\n",
        "# plt.title('Sobel'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "# plt.show()"
      ],
      "id": "blond-silly",
      "execution_count": null,
      "outputs": []
    }
  ]
}