webscrape_nairaland.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/saheedniyi02/naijaweb/blob/main/webscrape_nairaland.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JQGBPwreNzkj",
        "outputId": "e2674fd7-6570-406f-890d-394a6126a0c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting fake_useragent\n",
            "  Downloading fake_useragent-1.5.1-py3-none-any.whl.metadata (15 kB)\n",
            "Downloading fake_useragent-1.5.1-py3-none-any.whl (17 kB)\n",
            "Installing collected packages: fake_useragent\n",
            "Successfully installed fake_useragent-1.5.1\n"
          ]
        }
      ],
      "source": [
        "pip install fake_useragent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cx9mNiavNvzq"
      },
      "outputs": [],
      "source": [
        "import aiohttp\n",
        "import asyncio\n",
        "import itertools\n",
        "import tqdm\n",
        "import pandas as pd\n",
        "import pickle\n",
        "from fake_useragent import UserAgent\n",
        "from bs4 import BeautifulSoup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6lF_-q8cNtEZ"
      },
      "outputs": [],
      "source": [
        "ua = UserAgent()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fv1BcxxvPNOs"
      },
      "outputs": [],
      "source": [
        "async def fetch_posts_link(url):\n",
        "\n",
        "  try:\n",
        "    timeout = aiohttp.ClientTimeout(total=50)  # 50 seconds\n",
        "    async with aiohttp.ClientSession(timeout=timeout,headers={'User-Agent': ua.random}) as session:\n",
        "      async with session.get(url) as resp:\n",
        "        body = await resp.text()\n",
        "        print(resp.status_code)\n",
        "        soup = BeautifulSoup(body, 'html.parser')\n",
        "        try:\n",
        "          posts_info=soup.find(\"div\",class_=\"body\").find_all(\"table\")[2].tr.th.find_all(\"td\")\n",
        "        except AttributeError:\n",
        "          posts_info=soup.find(\"div\",class_=\"body\").find_all(\"table\")[1].tr.th.find_all(\"td\")\n",
        "        nairaland_domain=\"https://www.nairaland.com\"\n",
        "        links=[nairaland_domain+i.find(\"b\").a.get(\"href\") for i in posts_info]\n",
        "        return links\n",
        "  except:\n",
        "    return []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "teCY_cPJMMR8"
      },
      "outputs": [],
      "source": [
        "async def fetch_posts_link(url):\n",
        "    print(url)\n",
        "    try:\n",
        "      timeout = aiohttp.ClientTimeout(total=50)  # 50 seconds\n",
        "      async with aiohttp.ClientSession(timeout=timeout, headers={'User-Agent': ua.random}) as session:\n",
        "          async with session.get(url) as resp:\n",
        "              body = await resp.text()\n",
        "              print(resp.status)\n",
        "              soup = BeautifulSoup(body, 'html.parser')\n",
        "              nairaland_domain = \"https://www.nairaland.com\"\n",
        "              try:\n",
        "                  posts_info = soup.find(\"div\", class_=\"body\").find_all(\"table\")[2].tr.th.find_all(\"td\")\n",
        "              except (AttributeError, IndexError):  # Be specific about which errors you're catching\n",
        "                  posts_info = soup.find(\"div\", class_=\"body\").find_all(\"table\")[1].tr.th.find_all(\"td\")\n",
        "\n",
        "              # Ensure posts_info is not empty\n",
        "              if not posts_info:\n",
        "                  return []\n",
        "\n",
        "              links = [nairaland_domain + i.find(\"b\").a.get(\"href\") for i in posts_info if i.find(\"b\") and i.find(\"b\").a]\n",
        "              return links\n",
        "    except:\n",
        "        return []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cPApRWsbR-u5",
        "outputId": "4716c563-73a1-41f2-e843-f1f11fb3addd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://www.nairaland.com/career\n",
            "200\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['https://www.nairaland.com/8223336/leading-public-affairs-recruiters-phifer',\n",
              " 'https://www.nairaland.com/7710688/civil-servants-military-experience-delay',\n",
              " 'https://www.nairaland.com/8222716/sustainability-personnel-urgently-required',\n",
              " 'https://www.nairaland.com/5993259/coren-registration-thread',\n",
              " 'https://www.nairaland.com/8223203/hardship-civil-servants-go-work',\n",
              " 'https://www.nairaland.com/2481899/course-review-msc-global-energy',\n",
              " 'https://www.nairaland.com/8217674/design-powerpoint-slides-presentations-seminars',\n",
              " 'https://www.nairaland.com/8222688/40-without-achievement',\n",
              " 'https://www.nairaland.com/8223260/how-turn-passion-into-profit',\n",
              " 'https://www.nairaland.com/3049857/computer-science-vs-architecture',\n",
              " 'https://www.nairaland.com/5928680/thread-volunteering-opportunities-nigeria',\n",
              " 'https://www.nairaland.com/8223154/how-choose-right-kitchen-sink',\n",
              " 'https://www.nairaland.com/8204240/hire-me-website-development-projects',\n",
              " 'https://www.nairaland.com/8223133/best-trades-site-binance-account',\n",
              " 'https://www.nairaland.com/8223106/updated-monopoly-go-free-dice',\n",
              " 'https://www.nairaland.com/8223084/earn-30-50-weekly-it',\n",
              " 'https://www.nairaland.com/8223041/why-must-earn-dollars-how',\n",
              " 'https://www.nairaland.com/8221835/750k-lagos-1.1-million-abeokuta',\n",
              " 'https://www.nairaland.com/8213264/know-study-uk-family',\n",
              " 'https://www.nairaland.com/8213250/know-study-uk-family',\n",
              " 'https://www.nairaland.com/8108057/natural-sea-salt-nuts-seeds',\n",
              " 'https://www.nairaland.com/8222861/free-monopoly-go-dice-links',\n",
              " 'https://www.nairaland.com/8199629/auwalu-ahmed-dankade-returned-10000',\n",
              " 'https://www.nairaland.com/5597198/federal-civil-servant-received-new',\n",
              " 'https://www.nairaland.com/8221671/19-yr-old-nigerian-makes-300k-less',\n",
              " 'https://www.nairaland.com/8200783/need-powerpoint-slides-presentations-hit',\n",
              " 'https://www.nairaland.com/8222672/verified-salesforce-data-architect-dumps-questions',\n",
              " 'https://www.nairaland.com/8222621/betckey-quality-labels-every-time',\n",
              " 'https://www.nairaland.com/8222611/caregiver-lagos-ogun-state',\n",
              " 'https://www.nairaland.com/8222525/biggest-wealth-trap-avoid',\n",
              " 'https://www.nairaland.com/6700050/thoughts-nigerian-banks-salaries-2021',\n",
              " 'https://www.nairaland.com/8222444/job-vacancy',\n",
              " 'https://www.nairaland.com/8222290/future-medical-device-recruitment-trends',\n",
              " 'https://www.nairaland.com/8222203/10-utme-top-scorers-369',\n",
              " 'https://www.nairaland.com/8222112/what-should-know-new-nde',\n",
              " 'https://www.nairaland.com/98978/cfa-candidates-house',\n",
              " 'https://www.nairaland.com/8222072/nscdc-nasarawa-state-command-gets',\n",
              " 'https://www.nairaland.com/8149459/advanced-professional-writing-services-branding',\n",
              " 'https://www.nairaland.com/8143914/professional-resume-writer-available',\n",
              " 'https://www.nairaland.com/8221993/still-looking-job-thats-gonna',\n",
              " 'https://www.nairaland.com/8221959/us-marine-intense-simulated-combat',\n",
              " 'https://www.nairaland.com/8221765/make-money-online',\n",
              " 'https://www.nairaland.com/7984396/want-switch-nursing',\n",
              " 'https://www.nairaland.com/8143962/professional-designer-available',\n",
              " 'https://www.nairaland.com/8221713/devops-engineer-free-ci-cd',\n",
              " 'https://www.nairaland.com/8017131/autocad-bim-arcadia-v-ray-enscape',\n",
              " 'https://www.nairaland.com/8137713/network-administration-career',\n",
              " 'https://www.nairaland.com/8221631/sap-mm-specialist-free-materials',\n",
              " 'https://www.nairaland.com/948366/torn-msc-mba',\n",
              " 'https://www.nairaland.com/8029123/free-publicity-lagos-nigeria-photos',\n",
              " 'https://www.nairaland.com/8221343/best-bad-press-removal-services',\n",
              " 'https://www.nairaland.com/8221313/importance-tv-radio-appearances-laerryblue']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "await fetch_posts_link(\"https://www.nairaland.com/career\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hRdFeRTASm28"
      },
      "outputs": [],
      "source": [
        "async def fetch_posts(url):\n",
        "  try:\n",
        "    timeout = aiohttp.ClientTimeout(total=50)\n",
        "    async with aiohttp.ClientSession(timeout=timeout,headers={'User-Agent': ua.random}) as session:\n",
        "      async with session.get(url) as resp:\n",
        "        text=await resp.text()\n",
        "        if resp.status!=200:\n",
        "          print(resp.status)\n",
        "          return []\n",
        "        soup=BeautifulSoup(text,\"lxml\")\n",
        "        body=soup.find(\"div\",class_=\"body\")\n",
        "        post_=body.find(\"div\",class_=\"narrow\").text\n",
        "        #posts_lists=[post]\n",
        "        posts=body.find_all(\"table\",summary=\"posts\")[0].find_all(\"td\",class_=\"l w pd\")\n",
        "        posts_lists= [post.find(\"div\",class_=\"narrow\").text for post in posts[1:]]\n",
        "        posts_lists.append(post_)\n",
        "        return posts_lists\n",
        "  except:\n",
        "    return []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G1rI62ziBEvH"
      },
      "outputs": [],
      "source": [
        "dict_={'Politics': 'https://www.nairaland.com/politics',\n",
        " 'Crime': 'https://www.nairaland.com/crime',\n",
        " 'Romance': 'https://www.nairaland.com/romance',\n",
        " 'Jobs/Vacancies': 'https://www.nairaland.com/jobs',\n",
        " 'Career': 'https://www.nairaland.com/career',\n",
        " 'Business': 'https://www.nairaland.com/business',\n",
        " 'Investment': 'https://www.nairaland.com/investment',\n",
        " 'NYSC': 'https://www.nairaland.com/nysc',\n",
        " 'Education': 'https://www.nairaland.com/education',\n",
        " 'Autos': 'https://www.nairaland.com/autos',\n",
        " 'Car Talk': 'https://www.nairaland.com/cartalk',\n",
        " 'Properties': 'https://www.nairaland.com/properties',\n",
        " 'Health': 'https://www.nairaland.com/health',\n",
        " 'Travel': 'https://www.nairaland.com/travel',\n",
        " 'Family': 'https://www.nairaland.com/family',\n",
        " 'Culture': 'https://www.nairaland.com/culture',\n",
        " 'Religion': 'https://www.nairaland.com/religion',\n",
        " 'Food': 'https://www.nairaland.com/food',\n",
        " 'Diaries': 'https://www.nairaland.com/diaries',\n",
        " 'Nairaland Ads': 'https://www.nairaland.com/ads',\n",
        " 'Pets': 'https://www.nairaland.com/pets',\n",
        " 'Agriculture': 'https://www.nairaland.com/agriculture',\n",
        " 'Jokes Etc': 'https://www.nairaland.com/jokes',\n",
        " 'TV/Movies': 'https://www.nairaland.com/tv-movies',\n",
        " 'Music/Radio': 'https://www.nairaland.com/music-radio',\n",
        " 'Celebrities': 'https://www.nairaland.com/celebs',\n",
        " 'Fashion': 'https://www.nairaland.com/fashion',\n",
        " 'Events': 'https://www.nairaland.com/events',\n",
        " 'Sports': 'https://www.nairaland.com/sports',\n",
        " 'Gaming': 'https://www.nairaland.com/gaming',\n",
        " 'Forum Games': 'https://www.nairaland.com/forum-games',\n",
        " 'Literature': 'https://www.nairaland.com/literature',\n",
        " 'Programming': 'https://www.nairaland.com/programming',\n",
        " 'Webmasters': 'https://www.nairaland.com/webmasters',\n",
        " 'Computers': 'https://www.nairaland.com/computers',\n",
        " 'Phones': 'https://www.nairaland.com/phones',\n",
        " 'Art, Graphics & Video': 'https://www.nairaland.com/graphics-video',\n",
        " 'Technology Market': 'https://www.nairaland.com/techmarket'}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BACS1aMu_gFk"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "def get_section_links(section_name, number_of_pages=500):\n",
        "  #number_of_pages=606#number_of_pages_for_each_section\n",
        "  #gets the links to all the pages in each section\n",
        "  return [dict_[section_name]+f\"/{page_no}\" for page_no in range(number_of_pages)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5oUqzJeW5nz9"
      },
      "outputs": [],
      "source": [
        "posts=await fetch_posts(\"https://www.nairaland.com/8149459/advanced-professional-writing-services-branding\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IWucDgvxcF_k",
        "outputId": "e71ef964-f826-4e99-a22c-aa74b1bdb6d9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "len(posts)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0RMyHeCcBqML"
      },
      "outputs": [],
      "source": [
        "async def get_all_post_links(section, number_of_pages=500):\n",
        "    posts_links=[]\n",
        "    rate_limit=10\n",
        "    section_urls=get_section_links(section,number_of_pages)\n",
        "    for i in range(0, len(section_urls), rate_limit):\n",
        "            # Fetch the next batch of URLs\n",
        "            current_batch = section_urls[i:i + rate_limit]\n",
        "            tasks =  [fetch_posts_link(url) for url in current_batch]\n",
        "            # Execute the batch of requests\n",
        "            batch_results = await asyncio.gather(*tasks)\n",
        "            posts_links.extend(batch_results)\n",
        "\n",
        "            # Wait before making the next batch of requests if not the last batch\n",
        "            if i + rate_limit < len(section_urls):\n",
        "                await asyncio.sleep(4)\n",
        "\n",
        "    return posts_links"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ku6o0KfjaVnF"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6ih_ARH_dwCz"
      },
      "outputs": [],
      "source": [
        "\n",
        "async def get_all_posts(post_urls, section,df):\n",
        "\n",
        "    posts=[]\n",
        "    rate_limit=10\n",
        "    for i in tqdm.tqdm(range(0, len(post_urls), rate_limit)):\n",
        "        #print(i)\n",
        "        # Fetch the next batch of URLs\n",
        "        current_batch = post_urls[i:i + rate_limit]\n",
        "        tasks =  [fetch_posts(url) for url in current_batch]\n",
        "        # Execute the batch of requests\n",
        "        batch_results = await asyncio.gather(*tasks)\n",
        "        posts.extend(batch_results)\n",
        "\n",
        "        # Wait before making the next batch of requests if not the last batch\n",
        "        if i + rate_limit < len(post_urls):\n",
        "            await asyncio.sleep(5)\n",
        "        #print(batch_results)\n",
        "        #print(len(posts))\n",
        "        posts_list=list(itertools.chain(*posts))\n",
        "\n",
        "        posts_df=pd.DataFrame({\"text\":posts_list})\n",
        "        #print(len(posts))\n",
        "        df=pd.concat([df,posts_df])\n",
        "        posts=[]\n",
        "        #save df\n",
        "        if (i%1000)==0:\n",
        "            print(\"saving\")\n",
        "            df.to_csv(f\"/content/drive/MyDrive/nairaland_webtext/data_{section}.csv\")\n",
        "\n",
        "        #if i + rate_limit < len(post_urls):\n",
        "        #   await asyncio.sleep(3)\n",
        "\n",
        "\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4J-hJskS4efG"
      },
      "outputs": [],
      "source": [
        "section=\"Politics\"\n",
        "number_of_pages=16725\n",
        "df=pd.DataFrame()\n",
        "posts_links=await get_all_post_links(section,number_of_pages)\n",
        "posts_links=list(itertools.chain(*posts_links))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kS2d0TSrovTG"
      },
      "outputs": [],
      "source": [
        "def save_links(section, lists):\n",
        "    with open(f'/content/drive/MyDrive/nairaland_webtext/links/links_{section}.pkl', 'wb') as f:\n",
        "        pickle.dump(lists, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4NvYT3ZD-njI"
      },
      "outputs": [],
      "source": [
        "def open_links(section):\n",
        "    with open(f'/content/drive/MyDrive/nairaland_webtext/links/links_{section}.pkl', 'rb') as f:\n",
        "        return pickle.load( f)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "save_links(section, posts_links)"
      ],
      "metadata": {
        "id": "z1iHqrYgVzKw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LQBplClHDyxQ",
        "outputId": "7b89187d-63fe-4c84-f342-84f0838cccd0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "966747\n"
          ]
        }
      ],
      "source": [
        "print(len(posts_links))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_SowxVICY_V",
        "outputId": "9b050880-95b0-4ec0-a528-21ad8a6ac653"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/3500 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  3%|▎         | 100/3500 [09:01<5:04:05,  5.37s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 153/3500 [13:51<4:59:02,  5.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  6%|▌         | 200/3500 [18:04<4:55:45,  5.38s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  9%|▊         | 300/3500 [27:05<4:44:54,  5.34s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 11%|█▏        | 400/3500 [36:08<4:38:08,  5.38s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 12%|█▏        | 433/3500 [39:12<4:37:51,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 14%|█▍        | 500/3500 [45:14<4:33:45,  5.48s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 17%|█▋        | 600/3500 [54:20<4:21:25,  5.41s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 20%|██        | 700/3500 [1:03:25<4:11:01,  5.38s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 23%|██▎       | 800/3500 [1:12:29<4:04:40,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 26%|██▌       | 900/3500 [1:21:33<3:55:02,  5.42s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 29%|██▊       | 1000/3500 [1:30:38<3:45:00,  5.40s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 31%|███▏      | 1100/3500 [1:39:43<3:34:28,  5.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 34%|███▍      | 1200/3500 [1:48:47<3:25:01,  5.35s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 37%|███▋      | 1300/3500 [1:57:53<3:17:22,  5.38s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 37%|███▋      | 1303/3500 [1:58:16<3:57:38,  6.49s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 40%|████      | 1400/3500 [2:06:58<3:08:36,  5.39s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 43%|████▎     | 1500/3500 [2:16:04<2:58:49,  5.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 46%|████▌     | 1600/3500 [2:25:10<2:49:51,  5.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 49%|████▊     | 1700/3500 [2:34:19<2:40:52,  5.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 51%|█████▏    | 1800/3500 [2:43:27<2:33:53,  5.43s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 54%|█████▍    | 1900/3500 [2:52:38<2:25:20,  5.45s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 57%|█████▋    | 2000/3500 [3:01:46<2:14:36,  5.38s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 60%|██████    | 2100/3500 [3:10:57<2:06:12,  5.41s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 2200/3500 [3:20:04<1:57:30,  5.42s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 66%|██████▌   | 2300/3500 [3:29:12<1:48:25,  5.42s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 69%|██████▊   | 2400/3500 [3:38:23<1:40:21,  5.47s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 71%|███████▏  | 2500/3500 [3:47:31<1:29:11,  5.35s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 74%|███████▍  | 2600/3500 [3:56:39<1:21:21,  5.42s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 77%|███████▋  | 2700/3500 [4:05:47<1:12:23,  5.43s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 2800/3500 [4:15:00<1:03:05,  5.41s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 82%|████████▏ | 2864/3500 [4:21:07<58:01,  5.47s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 83%|████████▎ | 2900/3500 [4:24:22<54:52,  5.49s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 86%|████████▌ | 3000/3500 [4:33:39<45:21,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 87%|████████▋ | 3062/3500 [4:39:27<39:25,  5.40s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 89%|████████▊ | 3100/3500 [4:42:53<36:05,  5.41s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 89%|████████▉ | 3121/3500 [4:44:57<34:23,  5.45s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 91%|█████████▏| 3200/3500 [4:52:06<27:12,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 92%|█████████▏| 3204/3500 [4:52:36<31:13,  6.33s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 93%|█████████▎| 3254/3500 [4:57:07<22:06,  5.39s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 3300/3500 [5:01:17<17:57,  5.39s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 97%|█████████▋| 3400/3500 [5:10:30<09:04,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saving\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 98%|█████████▊| 3417/3500 [5:12:12<07:31,  5.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "429\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 3500/3500 [5:19:44<00:00,  5.48s/it]\n"
          ]
        }
      ],
      "source": [
        "df=await get_all_posts(posts_links, section,df)\n",
        "df.to_csv(f\"/content/drive/MyDrive/nairaland_webtext/data_{section}.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "t-zjSejg1K6u",
        "outputId": "640ad7d4-a768-4649-aff5-5632021c4ca1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: []\n",
              "Index: []"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a3218ce8-040c-456c-8eb4-023d1c93173d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a3218ce8-040c-456c-8eb4-023d1c93173d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a3218ce8-040c-456c-8eb4-023d1c93173d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a3218ce8-040c-456c-8eb4-023d1c93173d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_fd404e47-1122-4799-afcc-30bae40958e9\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_fd404e47-1122-4799-afcc-30bae40958e9 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 0,\n  \"fields\": []\n}"
            }
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "df"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1RYraPCcLQHSx_ZNB-mqS8-19OWJ7f8Tn",
      "authorship_tag": "ABX9TyP3d8+UMwE1FUESj1WCEFhG",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}