WebScraping_KnowZe.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Update Shell"
      ],
      "metadata": {
        "id": "hBa2GgITmJdy"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tmwOpWu-mAoA",
        "outputId": "63920105-cc12-4e48-9425-45730ad54c6a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33m\r0% [Working]\u001b[0m\r            \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "\u001b[33m\r0% [Waiting for headers] [Waiting for headers] [1 InRelease 3,626 B/3,626 B 100\u001b[0m\u001b[33m\r0% [Waiting for headers] [Waiting for headers] [Connected to ppa.launchpadconte\u001b[0m\r                                                                               \rHit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n",
            "Get:6 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease [18.1 kB]\n",
            "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Hit:9 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Get:10 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,016 kB]\n",
            "Hit:11 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,521 kB]\n",
            "Get:13 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy/main Sources [2,244 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,242 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,284 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,498 kB]\n",
            "Get:17 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy/main amd64 Packages [1,152 kB]\n",
            "Fetched 10.2 MB in 4s (2,632 kB/s)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "32 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "wget is already the newest version (1.21.2-2ubuntu1).\n",
            "curl is already the newest version (7.81.0-1ubuntu1.14).\n",
            "unzip is already the newest version (6.0-26ubuntu3.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 32 not upgraded.\n",
            "--2023-11-24 13:55:36--  http://archive.ubuntu.com/ubuntu/pool/main/libu/libu2f-host/libu2f-udev_1.1.4-1_all.deb\n",
            "Resolving archive.ubuntu.com (archive.ubuntu.com)... 185.125.190.36, 91.189.91.83, 91.189.91.82, ...\n",
            "Connecting to archive.ubuntu.com (archive.ubuntu.com)|185.125.190.36|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3708 (3.6K) [application/x-debian-package]\n",
            "Saving to: ‘libu2f-udev_1.1.4-1_all.deb’\n",
            "\n",
            "libu2f-udev_1.1.4-1 100%[===================>]   3.62K  --.-KB/s    in 0s      \n",
            "\n",
            "2023-11-24 13:55:36 (324 MB/s) - ‘libu2f-udev_1.1.4-1_all.deb’ saved [3708/3708]\n",
            "\n",
            "Selecting previously unselected package libu2f-udev.\n",
            "(Reading database ... 120880 files and directories currently installed.)\n",
            "Preparing to unpack libu2f-udev_1.1.4-1_all.deb ...\n",
            "Unpacking libu2f-udev (1.1.4-1) ...\n",
            "Setting up libu2f-udev (1.1.4-1) ...\n",
            "--2023-11-24 13:55:37--  https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n",
            "Resolving dl.google.com (dl.google.com)... 172.217.204.190, 172.217.204.93, 172.217.204.136, ...\n",
            "Connecting to dl.google.com (dl.google.com)|172.217.204.190|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 103884100 (99M) [application/x-debian-package]\n",
            "Saving to: ‘google-chrome-stable_current_amd64.deb’\n",
            "\n",
            "google-chrome-stabl 100%[===================>]  99.07M   381MB/s    in 0.3s    \n",
            "\n",
            "2023-11-24 13:55:38 (381 MB/s) - ‘google-chrome-stable_current_amd64.deb’ saved [103884100/103884100]\n",
            "\n",
            "Selecting previously unselected package google-chrome-stable.\n",
            "(Reading database ... 120884 files and directories currently installed.)\n",
            "Preparing to unpack google-chrome-stable_current_amd64.deb ...\n",
            "Unpacking google-chrome-stable (119.0.6045.159-1) ...\n",
            "\u001b[1mdpkg:\u001b[0m dependency problems prevent configuration of google-chrome-stable:\n",
            " google-chrome-stable depends on libvulkan1; however:\n",
            "  Package libvulkan1 is not installed.\n",
            "\n",
            "\u001b[1mdpkg:\u001b[0m error processing package google-chrome-stable (--install):\n",
            " dependency problems - leaving unconfigured\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Errors were encountered while processing:\n",
            " google-chrome-stable\n",
            "--2023-11-24 13:55:53--  https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chromedriver-linux64.zip\n",
            "Resolving edgedl.me.gvt1.com (edgedl.me.gvt1.com)... 34.104.35.123, 2600:1900:4110:86f::\n",
            "Connecting to edgedl.me.gvt1.com (edgedl.me.gvt1.com)|34.104.35.123|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 8530703 (8.1M) [application/octet-stream]\n",
            "Saving to: ‘/tmp/chromedriver-linux64.zip’\n",
            "\n",
            "chromedriver-linux6 100%[===================>]   8.13M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2023-11-24 13:55:53 (77.9 MB/s) - ‘/tmp/chromedriver-linux64.zip’ saved [8530703/8530703]\n",
            "\n",
            "unzip:  cannot find or open /tmp/chromedriver_linux64.zip, /tmp/chromedriver_linux64.zip.zip or /tmp/chromedriver_linux64.zip.ZIP.\n",
            "chmod: cannot access '/tmp/chromedriver': No such file or directory\n",
            "mv: cannot stat '/tmp/chromedriver': No such file or directory\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.15.2-py3-none-any.whl (10.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3[socks]<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from selenium) (2.0.7)\n",
            "Collecting trio~=0.17 (from selenium)\n",
            "  Downloading trio-0.23.1-py3-none-any.whl (448 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m448.3/448.3 kB\u001b[0m \u001b[31m31.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium)\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium) (2023.7.22)\n",
            "Requirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (23.1.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (3.4)\n",
            "Collecting outcome (from trio~=0.17->selenium)\n",
            "  Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: sniffio>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc9 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.1.3)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium)\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
            "Collecting h11<1,>=0.9.0 (from wsproto>=0.14->trio-websocket~=0.9->selenium)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: outcome, h11, wsproto, trio, trio-websocket, selenium\n",
            "Successfully installed h11-0.14.0 outcome-1.3.0.post0 selenium-4.15.2 trio-0.23.1 trio-websocket-0.11.1 wsproto-1.2.0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "%%shell\n",
        "sudo apt -y update\n",
        "sudo apt install -y wget curl unzip\n",
        "wget http://archive.ubuntu.com/ubuntu/pool/main/libu/libu2f-host/libu2f-udev_1.1.4-1_all.deb\n",
        "dpkg -i libu2f-udev_1.1.4-1_all.deb\n",
        "wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n",
        "dpkg -i google-chrome-stable_current_amd64.deb\n",
        "CHROME_DRIVER_VERSION=`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`\n",
        "wget -N https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chromedriver-linux64.zip -P /tmp/\n",
        "unzip -o /tmp/chromedriver_linux64.zip -d /tmp/\n",
        "chmod +x /tmp/chromedriver\n",
        "mv /tmp/chromedriver /usr/local/bin/chromedriver\n",
        "pip install selenium"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Install Chromer driver for control chrome browser"
      ],
      "metadata": {
        "id": "PbBRc6VxmrO_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromedriver-autoinstaller"
      ],
      "metadata": {
        "id": "3CBwxVDgmRgB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "from bs4 import BeautifulSoup\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.common.exceptions import NoSuchElementException\n",
        "from selenium.webdriver.chrome.service import Service\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.webdriver.support.ui import WebDriverWait"
      ],
      "metadata": {
        "id": "IQXncSvLmZ7i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Selenium For Driver"
      ],
      "metadata": {
        "id": "FFIcTvTbm-vy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def driversetup():\n",
        "    options = webdriver.ChromeOptions()\n",
        "    #run Selenium in headless mode\n",
        "    options.add_argument('--headless')\n",
        "    options.add_argument('--no-sandbox')\n",
        "    #overcome limited resource problems\n",
        "    options.add_argument('--disable-dev-shm-usage')\n",
        "    options.add_argument(\"lang=en\")\n",
        "    #open Browser in maximized mode\n",
        "    options.add_argument(\"start-maximized\")\n",
        "    #disable infobars\n",
        "    options.add_argument(\"disable-infobars\")\n",
        "    #disable extension\n",
        "    options.add_argument(\"--disable-extensions\")\n",
        "    options.add_argument(\"--incognito\")\n",
        "    options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
        "\n",
        "    driver = webdriver.Chrome(options=options)\n",
        "\n",
        "    driver.execute_script(\"Object.defineProperty(navigator, 'webdriver', {get: () => undefined});\")\n",
        "\n",
        "    return driver"
      ],
      "metadata": {
        "id": "gcXS2Sg-mcFv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "driver = driversetup()"
      ],
      "metadata": {
        "id": "nTQ4cg2ZmgsE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get Elements from Website\n",
        "Get URL and Preprocessing"
      ],
      "metadata": {
        "id": "wzDHxJjRnlrM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_element_by_classname(url, driver, byClassName):\n",
        "    driver.get(url)\n",
        "\n",
        "    # Mencari semua elemen menggunakan byClassName\n",
        "    title_elements = driver.find_elements(By.CLASS_NAME, byClassName)\n",
        "\n",
        "    # List untuk menyimpan teks dari elemen-elemen yang ditemukan\n",
        "    title_texts = []\n",
        "\n",
        "    # Iterasi melalui setiap elemen dan ambil teksnya\n",
        "    for element in title_elements:\n",
        "        # print(element.text)\n",
        "        title_texts.append(element.text)\n",
        "\n",
        "    driver.quit()\n",
        "\n",
        "    return title_texts\n",
        "\n",
        "driver = driversetup()\n",
        "url = 'https://indeks.kompas.com/terpopuler'\n",
        "byClassName = 'article__link'  # byClassName yang spesifik\n",
        "selected_element_text = get_element_by_classname(url, driver, byClassName)\n",
        "\n",
        "print(selected_element_text)"
      ],
      "metadata": {
        "id": "8YktixMBmlQK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "QKAVKmIonjrd"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}