Glassdoor Job Designation Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request, urllib.parse, urllib.error\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.glassdoor.co.in/Job/india-product-manager-jobs-SRCH_IL.0,5_IN115_KO6,21.htm'\n",
    "\n",
    "ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'\n",
    "\n",
    "request = urllib.request.Request(url,headers={'User-Agent': ua})\n",
    "response = urllib.request.urlopen(request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = response.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(html, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_link = []\n",
    "for link in soup.select('a[href]'):\n",
    "    if 'jobListing' in str(link.get('href')):\n",
    "        all_link.append('https://www.glassdoor.co.in' + link.get('href'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "jobUrl = all_link[2]\n",
    "\n",
    "jobRequest = urllib.request.Request(jobUrl,headers={'User-Agent': ua})\n",
    "jobHTML = urllib.request.urlopen(jobRequest).read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "job_soup = BeautifulSoup(jobHTML, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Product Manager'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post = job_soup.find('div', {'class' : 'jobViewJobTitleWrap'}).getText()\n",
    "post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Senior Product Manager\n",
      "Senior Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "SR. PRODUCT MANAGER - TECH\n",
      "SR. PRODUCT MANAGER - TECH\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Sr Product Manager\n",
      "Sr Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Growth Manager\n",
      "Product Growth Manager\n",
      "Senior Product Manager\n",
      "Senior Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n",
      "Product Manager\n"
     ]
    }
   ],
   "source": [
    "for links in all_link:\n",
    "    job_request = urllib.request.Request(links,headers={'User-Agent': ua})\n",
    "    job_HTML = urllib.request.urlopen(job_request).read()\n",
    "    job_data = BeautifulSoup(job_HTML, \"lxml\")\n",
    "    for designation in job_data.find_all('div', {'class' : 'jobViewJobTitleWrap'}):\n",
    "        print(designation.getText())\n",
    "        \n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}