UC Berkeley Majors & Minors Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Native imports\n",
    "import sys\n",
    "import re\n",
    "import os\n",
    "import os.path\n",
    "import json\n",
    "from pprint import pprint\n",
    "import urllib.request\n",
    "import urllib.parse\n",
    "\n",
    "# 3rd-party scraping/parsing imports\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import dateutil.parser as dt\n",
    "import demjson\n",
    "\n",
    "# 3rd-party data science imports\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "# Used to make the plots bigger\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = (8, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_html_page(url):\n",
    "    return requests.get(url).text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNDERGRAD_PROGRAMS_URL = 'http://guide.berkeley.edu/undergraduate/degree-programs/'\n",
    "GRAD_PROGRAMS_URL = 'http://guide.berkeley.edu/graduate/degree-programs/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Undergraduate Programs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = fetch_html_page(UNDERGRAD_PROGRAMS_URL)\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_all_programs(soup):\n",
    "    for element in soup.find_all('script'):\n",
    "        if len(element.contents) > 0 and 'allProgData' in element.contents[0]:\n",
    "            json_str = element.contents[0].split(' = ')[1]\n",
    "            json_str = json_str.strip()[:-1]\n",
    "\n",
    "            raw_program_data = demjson.decode(json_str)\n",
    "            processed_program_data = list(raw_program_data.values())\n",
    "            for program in processed_program_data:\n",
    "                del program['id']\n",
    "                del program['url']\n",
    "            \n",
    "            return processed_program_data\n",
    "        \n",
    "def fetch_filter_info(soup, debug=False):\n",
    "    filter_info = {}\n",
    "    \n",
    "    for filter_group in soup.find_all('div', class_='filter-group'):\n",
    "        filter_cat = filter_group.previous_sibling.get_text()\n",
    "        \n",
    "        if debug:\n",
    "            print('Filter Category:', filter_cat)\n",
    "        \n",
    "        filter_info[filter_cat] = {}\n",
    "        \n",
    "        for filter_ in filter_group.find_all('label'):\n",
    "            filter_internal_name = filter_.attrs['for']\n",
    "            filter_real_name = filter_.get_text().strip()\n",
    "            \n",
    "            if debug:\n",
    "                print(f'  {filter_real_name} ({filter_internal_name})')\n",
    "            \n",
    "            filter_info[filter_cat][filter_internal_name] = filter_real_name\n",
    "    \n",
    "    return filter_info\n",
    "\n",
    "def synthesize_program_dataset_undergrad(programs, filters, keep_major_minor_only=True):\n",
    "    program_dataset = {}\n",
    "    \n",
    "    selected_filter_cat = list(filters.values())[0]\n",
    "    \n",
    "    for program in programs:\n",
    "        for (filter_key, filter_name) in selected_filter_cat.items():\n",
    "            filter_name = filter_name.lower()\n",
    "            \n",
    "            if filter_key in program['filters']:\n",
    "                if keep_major_minor_only and filter_name not in ['majors', 'minors']:\n",
    "                    continue\n",
    "                \n",
    "                if filter_name not in program_dataset:\n",
    "                    program_dataset[filter_name] = []\n",
    "                \n",
    "                program_dataset[filter_name] += [program['name']]\n",
    "    \n",
    "    return program_dataset\n",
    "\n",
    "all_programs = fetch_all_programs(soup)\n",
    "all_filters = fetch_filter_info(soup)\n",
    "final_dataset = synthesize_program_dataset_undergrad(all_programs, all_filters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "majors_list_mongo = [{'_id': i, 'major': major} for (i, major) in enumerate(final_dataset['majors'])]\n",
    "minors_list_mongo = [{'_id': i, 'minor': minor} for (i, minor) in enumerate(final_dataset['minors'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('exported/majors.json', 'w') as fp:\n",
    "    json.dump(majors_list_mongo, fp)\n",
    "    \n",
    "with open('exported/minors.json', 'w') as fp:\n",
    "    json.dump(minors_list_mongo, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}