explore-jhu-time-series.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring COVID-19 time series data from Johns Hopkins University"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook analyzes the data in the [CSSEGISandData/COVID-19](https://github.com/CSSEGISandData/COVID-19) github repo. This assumes you have cloned the repo at the local path `~/github/CSSEGISandData/COVID-19`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = os.path.expanduser(\"~/github/CSSEGISandData/COVID-19/\"\n",
    "                             \"csse_covid_19_data/csse_covid_19_time_series\")\n",
    "def read_csv(basename):\n",
    "    return pd.read_csv(os.path.join(dirname, basename), header=0)\n",
    "us_cases_df = read_csv(\"time_series_covid19_confirmed_US.csv\")\n",
    "us_deaths_df = read_csv(\"time_series_covid19_deaths_US.csv\")\n",
    "global_cases_df = read_csv(\"time_series_covid19_confirmed_global.csv\")\n",
    "global_deaths_df = read_csv(\"time_series_covid19_deaths_global.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_cases_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_deaths_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_cases_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_deaths_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(us_cases_df.columns[6:])\n",
    "print(us_deaths_df.columns[6:])\n",
    "print(global_cases_df.columns)\n",
    "print(global_deaths_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_torch(df, first_column):\n",
    "    df = df[df.columns[first_column:]]\n",
    "    return torch.from_numpy(df.to_numpy()).float()\n",
    "\n",
    "case_data = torch.cat([to_torch(us_cases_df, first_column=11),\n",
    "                       to_torch(global_cases_df, first_column=4)])\n",
    "print(case_data.shape)\n",
    "case_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "death_data = torch.cat([to_torch(us_deaths_df, first_column=12),\n",
    "                        to_torch(global_deaths_df, first_column=4)])\n",
    "print(death_data.shape)\n",
    "death_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "gisaid_stats = pickle.load(open(\"results/gisaid.stats.pkl\", \"rb\"))\n",
    "print(gisaid_stats.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, count in gisaid_stats[\"location\"].most_common(30):\n",
    "    print(f\"{count}\\t{name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gisaid_locations = {tuple(part.strip() for part in key.lower().split(\"/\")[1:])\n",
    "                    for key in gisaid_stats[\"location\"]}\n",
    "gisaid_locations = {loc for loc in gisaid_locations if len(loc) >= 1}\n",
    "print(len(gisaid_locations))\n",
    "print(sorted(gisaid_locations)[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jhu_locations = set()\n",
    "for i, row in us_cases_df[[\"Country_Region\", \"Province_State\", \"Admin2\"]].iterrows():\n",
    "    a, b, c = row\n",
    "    if isinstance(c, str):\n",
    "        jhu_locations.add((a.lower(), b.lower(), c.lower()))\n",
    "    else:\n",
    "        jhu_locations.add((a.lower(), b.lower()))\n",
    "for i, row in global_cases_df[[\"Country/Region\", \"Province/State\"]].iterrows():\n",
    "    a, b = row\n",
    "    if isinstance(b, str):\n",
    "        jhu_locations.add((a.lower(), b.lower()))\n",
    "    else:\n",
    "        jhu_locations.add((a.lower(),))\n",
    "assert len(jhu_locations) == len(us_cases_df) + len(global_cases_df)\n",
    "print(len(jhu_locations))\n",
    "print(sorted(jhu_locations)[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(gisaid_locations & jhu_locations))\n",
    "print(len(gisaid_locations - jhu_locations))\n",
    "print(len(jhu_locations - gisaid_locations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc = {loc[0] for loc in gisaid_locations}\n",
    "jc = {loc[0] for loc in jhu_locations}\n",
    "print(gc - jc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{row for row in jhu_locations if any(\"serrat\" in col for col in row)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyrocov.geo import GISAID_TO_JHU\n",
    "\n",
    "for country in gc - jc:\n",
    "    c = GISAID_TO_JHU[country]\n",
    "    assert c is None or isinstance(c, tuple), c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Joining with population data from UN\n",
    "\n",
    "`WPP2019_TotalPopulationBySex.csv` was downloaded from https://population.un.org/wpp/Download/Standard/CSV/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/WPP2019_TotalPopulationBySex.csv\", header=0)\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df[\"Time\"] == 2020]\n",
    "df = df[df[\"Variant\"] == \"High\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uc = {name.lower() for name in df[\"Location\"].to_list()}\n",
    "jc - uc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{row for row in uc if \"myanmar\" in row}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyrocov.geo import JHU_TO_UN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in jc:\n",
    "    if c not in uc:\n",
    "        c2 = JHU_TO_UN[c]\n",
    "        if c2 is not None:\n",
    "            assert c2 in uc, (c, c2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}