public-diversity-in-eecs-department.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Diversity in EECS at UC Berkeley: Part 1\n",
    "This notebook contains the code for the article at https://shomil.me/eecs-diversity/.\n",
    "<br>\n",
    "<br>\n",
    "All data for this project has been sourced from Cal Answers (http://calanswers.berkeley.edu). If you're interested in downloading the data, please shoot me an email (shomil@berkeley.edu) and I'll help walk you through it. \n",
    "<br>\n",
    "<br>\n",
    "The data files should be stored in the `/data/eecs-admissions/` folder in three sub-folders: `applied/`, `admitted/`, and `committed/`. Each category should contain a list of CSV files – these files come directly from Cal Answers.\n",
    "<br>\n",
    "<br>\n",
    "Recommended: use pipenv to install & run this notebook (`pipenv install && pipenv run jupyter notebook` is all you should need to take care of all of the packages required for these visualizations)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import admissions, export, helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import glob\n",
    "from tabulate import tabulate\n",
    "import chart_studio\n",
    "import chart_studio.plotly as py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the Academic Year\n",
    "ACADEMIC_YEAR = '2020-21'\n",
    "\n",
    "# Choose included majors.\n",
    "INCLUDED_MAJORS = ['Electrical Eng & Comp Sci', 'L&S Computer Science']\n",
    "\n",
    "# For treemaps, set an overarching bucket name.\n",
    "TOP_LEVEL= f'All SIR\\'ed Students in EECS/L&S CS (Academic Year: {ACADEMIC_YEAR})'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If this is set to true, graphs/tables will be exported to EXPORT_FOLDER.\n",
    "SHOULD_WRITE = False\n",
    "EXPORT_FOLDER = 'exported_graphs/'\n",
    "\n",
    "# If this is set to true, graphs will be published to Plotly Chart Studio.\n",
    "SHOULD_PUBLISH = False\n",
    "PUBLISH_PREFIX = 'eecs-diversity-'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "export.setup_chart_studio()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns: \n",
      " • ('Admitted')\n",
      " • ('Applied')\n",
      " • ('SIRed')\n",
      " • Academic Division\n",
      " • Academic Yr\n",
      " • Admit Rate\n",
      " • Applicant Headcounts (renamed to: Headcount)\n",
      " • College/School\n",
      " • Department\n",
      " • Derived Residency (renamed to: Residency)\n",
      " • Gender\n",
      " • High School API Rank\n",
      " • Income Range Amt 2 - Parent (renamed to: Family Income)\n",
      " • Intended Major\n",
      " • LCFF+ Flg\n",
      " • Neither Parent has 4 Year College Degree\n",
      " • Neither Parent has Attended College (renamed to: First Generation Student)\n",
      " • Prior School Type\n",
      " • Short Ethnic Desc (renamed to: Ethnicity L3)\n",
      " • Ucb Level1 Ethnic Rollup Desc (renamed to: Ethnicity L1)\n",
      " • Ucb Level2 Ethnic Rollup Desc (renamed to: Ethnicity L2)\n",
      " • Yield Rate\n",
      "\n",
      "Years Present: 2000-01, 2001-02, 2002-03, 2003-04, 2004-05, 2005-06, 2006-07, 2007-08, 2008-09, 2009-10, 2010-11, 2011-12, 2012-13, 2013-14, 2014-15, 2015-16, 2016-17, 2017-18, 2018-19, 2019-20, 2020-21\n",
      "\n",
      "Total # Applied:  118885.0 students\n",
      "Total # Admitted: 16184.0 students\n",
      "Total # SIRed:    8519.0 students\n",
      "\n",
      "Latest Academic Year (Cross Check This Data with Public Information): 2020-21\n",
      "Total # Applied:  11617.0 students\n",
      "Total # Admitted: 1553.0 students\n",
      "Total # SIRed:    859.0 students\n"
     ]
    }
   ],
   "source": [
    "applied, admitted, committed = admissions.load_admissions_df(majors=INCLUDED_MAJORS)\n",
    "frames = {'Applied': applied, 'Admitted': admitted, 'SIR\\'ed': committed}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Historical Look"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot Applications, Acceptances, and SIR's over time.\n",
    "fig = go.Figure()\n",
    "for name, df in frames.items():\n",
    "    data = df.groupby('Academic Yr').sum().reset_index()\n",
    "    fig.add_trace(go.Scatter(\n",
    "        x=data['Academic Yr'],\n",
    "        y=data['Headcount'],\n",
    "        mode='lines+markers',\n",
    "        name=name\n",
    "    ))\n",
    "\n",
    "helpers.style_figure(fig, \n",
    "                     title='Number of Applied, Admitted, and SIR\\'ed Students over Time', \n",
    "                     x_title='Academic Year', \n",
    "                     y_title='Number of Students')\n",
    "\n",
    "if SHOULD_WRITE:\n",
    "    fig.write_html(EXPORT_FOLDER + 'graph1.html', include_plotlyjs='cdn')\n",
    "else:\n",
    "    fig.show()\n",
    "    \n",
    "if SHOULD_PUBLISH:\n",
    "    py.plot(fig, filename=PUBLISH_PREFIX + 'graph1', auto_open=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_line_graph(frames=frames,\n",
    "                        column='Gender',\n",
    "                        title='Gender Ratios over Time (Applications)',\n",
    "                        file_name='graph2',\n",
    "                        write=SHOULD_WRITE,\n",
    "                        write_folder=EXPORT_FOLDER,\n",
    "                        publish=SHOULD_PUBLISH,\n",
    "                        publish_prefix=PUBLISH_PREFIX,\n",
    "                        categories=['Applied', 'Admitted', 'SIR\\'ed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.write_table(frames=frames,\n",
    "                    category='Gender',\n",
    "                    title='Gender Breakdowns',\n",
    "                    file_name='table1.txt',\n",
    "                    year=ACADEMIC_YEAR,\n",
    "                    write=SHOULD_WRITE,\n",
    "                    write_folder=EXPORT_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_line_graph(frames=frames,\n",
    "                        column='Ethnicity L1',\n",
    "                        title='Ethnicity Ratios over Time (Applications)',\n",
    "                        file_name='graph3',\n",
    "                        write=SHOULD_WRITE,\n",
    "                        write_folder=EXPORT_FOLDER,\n",
    "                        publish=SHOULD_PUBLISH,\n",
    "                        publish_prefix=PUBLISH_PREFIX,\n",
    "                        categories=['Applied'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_line_graph(frames=frames,\n",
    "                        column='Ethnicity L1',\n",
    "                        title='Ethnicity Ratios over Time (Applications)',\n",
    "                        file_name='graph4',\n",
    "                        write=SHOULD_WRITE,\n",
    "                        write_folder=EXPORT_FOLDER,\n",
    "                        publish=SHOULD_PUBLISH,\n",
    "                        publish_prefix=PUBLISH_PREFIX,\n",
    "                        categories=['Admitted', 'SIR\\'ed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.write_table(frames=frames,\n",
    "                    category='Ethnicity L1',\n",
    "                    title='Ethnic Breakdowns (L1) for Applicant Pool of ' + ACADEMIC_YEAR,\n",
    "                    file_name='table2.txt',\n",
    "                    year=ACADEMIC_YEAR,\n",
    "                    write=SHOULD_WRITE,\n",
    "                    write_folder=EXPORT_FOLDER)\n",
    "\n",
    "helpers.write_table(frames=frames,\n",
    "                    category='Ethnicity L2',\n",
    "                    title='Ethnic Breakdowns (L2) for the Applicant Pool of ' + ACADEMIC_YEAR,\n",
    "                    file_name='table3.txt',\n",
    "                    year=ACADEMIC_YEAR,\n",
    "                    write=SHOULD_WRITE,\n",
    "                    write_folder=EXPORT_FOLDER)\n",
    "\n",
    "helpers.write_table(frames=frames,\n",
    "                    category='Ethnicity L3',\n",
    "                    title='Ethnic Breakdowns (L3) for the Applicant Pool of ' + ACADEMIC_YEAR,\n",
    "                    file_name='table4.txt',\n",
    "                    year=ACADEMIC_YEAR,\n",
    "                    write=SHOULD_WRITE,\n",
    "                    write_folder=EXPORT_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_line_graph(frames=frames,\n",
    "                        column='Family Income',\n",
    "                        title='Family Income Breakdown over Time (Applications)',\n",
    "                        file_name='graph5',\n",
    "                        write=SHOULD_WRITE,\n",
    "                        write_folder=EXPORT_FOLDER,\n",
    "                        publish=SHOULD_PUBLISH,\n",
    "                        publish_prefix=PUBLISH_PREFIX,\n",
    "                        categories=['Applied'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_line_graph(frames=frames,\n",
    "                        column='Family Income',\n",
    "                        title='Family Income Breakdown over Time (Admitted & SIR\\'ed Students)',\n",
    "                        file_name='graph6',\n",
    "                        write=SHOULD_WRITE,\n",
    "                        write_folder=EXPORT_FOLDER,\n",
    "                        publish=SHOULD_PUBLISH,\n",
    "                        publish_prefix=PUBLISH_PREFIX,\n",
    "                        categories=['Admitted', 'SIR\\'ed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.write_table(frames=frames,\n",
    "                    category='Family Income',\n",
    "                    title='Family Income Breakdowns for the Applicant Pool of ' + ACADEMIC_YEAR,\n",
    "                    file_name='table5.txt',\n",
    "                    write=SHOULD_WRITE,\n",
    "                    year=ACADEMIC_YEAR,\n",
    "                    write_folder=EXPORT_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Gender/Ethnicity Breakdown (Shaded by Headcount)',\n",
    "                     path=['Ethnicity L3', 'Gender'],\n",
    "                     file_name='graph7',\n",
    "                     color_col='Headcount',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Gender/Ethnicity Breakdown (Shaded by Admit Rate)',\n",
    "                     path=['Ethnicity L3', 'Gender'],\n",
    "                     file_name='graph8',\n",
    "                     color_col='Admit Rate',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Family Income/Ethnicity Breakdown (Shaded by Headcount)',\n",
    "                     path=['Family Income', 'Ethnicity L3'],\n",
    "                     file_name='graph9',\n",
    "                     color_col='Headcount',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Family Income/Ethnicity Breakdown (Shaded by Admit Rate)',\n",
    "                     path=['Family Income', 'Ethnicity L3'],\n",
    "                     file_name='graph10',\n",
    "                     color_col='Admit Rate',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Comparing Ethnicity & First Generation College Students (Shaded by Admit Rate)',\n",
    "                     path=['Ethnicity L3', 'First Generation Student'],\n",
    "                     file_name='graph11',\n",
    "                     color_col='Admit Rate',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "helpers.plot_treemap(data=committed,\n",
    "                     title='Stacking Demographic Inequalities (Shaded by Headcount)',\n",
    "                     path=['Gender', 'Ethnicity L3', 'Family Income', 'First Generation Student'],\n",
    "                     file_name='graph12',\n",
    "                     color_col='Headcount',\n",
    "                     write=SHOULD_WRITE,\n",
    "                     publish=SHOULD_PUBLISH,\n",
    "                     top_level=TOP_LEVEL,\n",
    "                     year=ACADEMIC_YEAR,\n",
    "                     write_folder=EXPORT_FOLDER,\n",
    "                     publish_prefix=PUBLISH_PREFIX)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}