data-wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aacfd99d",
   "metadata": {},
   "source": [
    "# Downloading FEMA NFIP Data for New York City"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbc9148c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing libraries\n",
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import requests\n",
    "import json\n",
    "import time\n",
    "\n",
    "plt.rcParams['savefig.facecolor'] = 'white'\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f54d766c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "printing packages and versions:\n",
      "\n",
      "Python implementation: CPython\n",
      "Python version       : 3.8.13\n",
      "IPython version      : 8.4.0\n",
      "\n",
      "numpy     : 1.23.1\n",
      "pandas    : 1.4.3\n",
      "geopandas : 0.11.1\n",
      "matplotlib: 3.5.2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('printing packages and versions:\\n')\n",
    "\n",
    "%reload_ext watermark\n",
    "%watermark -v -p numpy,pandas,geopandas,matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c52bee1c",
   "metadata": {},
   "source": [
    "We utilize FEMA's [OpenFEMA API](https://www.fema.gov/about/openfema/api) to fetch data by county. The county codes for New York City are below."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ce72c47",
   "metadata": {},
   "source": [
    "County FIPS Codes for New York City:\n",
    "\n",
    "- The Bronx is Bronx County - FIPS 36005  \n",
    "- Brooklyn is Kings County - FIPS 36047  \n",
    "- Manhattan is New York County - FIPS 36061  \n",
    "- Queens is Queens County - FIPS 36081  \n",
    "- Staten Island is Richmond County - FIPS 36085"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfb7a4ed",
   "metadata": {},
   "source": [
    "# The OpenFEMA API\n",
    "We write a function to fetch data utilizing FEMA's OpenFEMA's API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "37adebaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set constant parameters\n",
    "BASE_URL = 'https://www.fema.gov/api/open/v2/'\n",
    "FORMAT_PARAM = '$format=json'\n",
    "METADATA_PARAM = '&$metadata=off'\n",
    "FILTER_PARAM = '&$filter=countyCode%20eq%20%27{}%27'\n",
    "SKIP_PARAM = '&$skip={}'\n",
    "TOP_PARAM = '&$top=10000'\n",
    "\n",
    "def get_api_url(dataset, county_fips, skip):\n",
    "    \"\"\"Generate the API URL for the given dataset, county FIPS, and skip value.\"\"\"\n",
    "    url_base = f'{BASE_URL}FimaNfip{dataset.capitalize()}?'\n",
    "    return f'{url_base}{FORMAT_PARAM}{METADATA_PARAM}{FILTER_PARAM}{SKIP_PARAM}{TOP_PARAM}'.format(county_fips, skip)\n",
    "\n",
    "def make_request(url):\n",
    "    \"\"\"Make an HTTP request and handle potential errors.\"\"\"\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "        response.raise_for_status()\n",
    "        return response.json()\n",
    "    except requests.RequestException as e:\n",
    "        print(f\"Error making request: {e}\")\n",
    "        return None\n",
    "\n",
    "def download_data(dataset, county_fips):\n",
    "    \"\"\"\n",
    "    Download FEMA NFIP policies or claims data for a specific county and save it to a JSON file.\n",
    "\n",
    "    Parameters:\n",
    "    - county_fips (str): County FIPS code.\n",
    "    - dataset (str): Either 'policies' or 'claims'.\n",
    "\n",
    "    Returns:\n",
    "    - None\n",
    "    \"\"\"\n",
    "    if dataset not in {'policies', 'claims'}:\n",
    "        raise ValueError(\"Invalid dataset. Pass either 'policies' or 'claims' to the dataset argument\")\n",
    "\n",
    "    if not isinstance(county_fips, str):\n",
    "        raise ValueError(\"County FIPS code must be passed as a string\")\n",
    "\n",
    "    result_list = []\n",
    "    skip = 0\n",
    "\n",
    "    # we'll add print statements to see what's happening\n",
    "    print(f\"Dataset: {dataset}\")\n",
    "    print(f'County FIPS: {county_fips}\\n------------')\n",
    "\n",
    "    while True:\n",
    "        print(f'Skip number: {skip:,}')\n",
    "\n",
    "        # make HTTP request and handle JSON response\n",
    "        url = get_api_url(dataset, county_fips, skip)\n",
    "        data = make_request(url)\n",
    "\n",
    "        dataset_name = f\"FimaNfip{dataset.capitalize()}\"\n",
    "\n",
    "        # check if the response is empty or contains an error\n",
    "        if not data or dataset_name not in data:\n",
    "            break\n",
    "\n",
    "        # process JSON data and extend the result list\n",
    "        result_list.extend(data[dataset_name])\n",
    "\n",
    "        rows = len(data[dataset_name])\n",
    "        print(f'Number of rows: {rows:,}')\n",
    "        if rows < 10000:\n",
    "            break\n",
    "\n",
    "        print(f'Result list length: {len(result_list):,}')\n",
    "        skip += 10000\n",
    "        time.sleep(5)\n",
    "\n",
    "    print(f'\\nLength of the full result list: {len(result_list):,}\\n')\n",
    "\n",
    "    # write the result list to a JSON file\n",
    "    output_filename = f\"data/{dataset}-{county_fips}.json\"\n",
    "    with open(output_filename, 'w') as json_file:\n",
    "        json.dump(result_list, json_file, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bab3069",
   "metadata": {},
   "source": [
    "# Download Data by Dataset and County FIPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e942e3b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: policies\n",
      "County FIPS: 36005\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 20,000\n",
      "Skip number: 20,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 30,000\n",
      "Skip number: 30,000\n",
      "Number of rows: 2,487\n",
      "\n",
      "Length of the full result list: 32,487\n",
      "\n",
      "CPU times: user 4.77 s, sys: 435 ms, total: 5.21 s\n",
      "Wall time: 34.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('policies', '36005')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fdcaccec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: claims\n",
      "County FIPS: 36005\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 1,922\n",
      "\n",
      "Length of the full result list: 1,922\n",
      "\n",
      "CPU times: user 287 ms, sys: 22.4 ms, total: 309 ms\n",
      "Wall time: 2.48 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('claims', '36005')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0b4d000b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: policies\n",
      "County FIPS: 36047\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 20,000\n",
      "Skip number: 20,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 30,000\n",
      "Skip number: 30,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 40,000\n",
      "Skip number: 40,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 50,000\n",
      "Skip number: 50,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 60,000\n",
      "Skip number: 60,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 70,000\n",
      "Skip number: 70,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 80,000\n",
      "Skip number: 80,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 90,000\n",
      "Skip number: 90,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 100,000\n",
      "Skip number: 100,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 110,000\n",
      "Skip number: 110,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 120,000\n",
      "Skip number: 120,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 130,000\n",
      "Skip number: 130,000\n",
      "Number of rows: 9,385\n",
      "\n",
      "Length of the full result list: 139,385\n",
      "\n",
      "CPU times: user 20.1 s, sys: 1.85 s, total: 22 s\n",
      "Wall time: 2min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('policies', '36047')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "09b8e642",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: claims\n",
      "County FIPS: 36047\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 6,239\n",
      "\n",
      "Length of the full result list: 6,239\n",
      "\n",
      "CPU times: user 893 ms, sys: 66.4 ms, total: 960 ms\n",
      "Wall time: 4.26 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('claims', '36047')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "392bed8b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: policies\n",
      "County FIPS: 36061\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 20,000\n",
      "Skip number: 20,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 30,000\n",
      "Skip number: 30,000\n",
      "Number of rows: 4,061\n",
      "\n",
      "Length of the full result list: 34,061\n",
      "\n",
      "CPU times: user 4.83 s, sys: 397 ms, total: 5.23 s\n",
      "Wall time: 33.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('policies', '36061')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "31150495",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: claims\n",
      "County FIPS: 36061\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 1,475\n",
      "\n",
      "Length of the full result list: 1,475\n",
      "\n",
      "CPU times: user 202 ms, sys: 14.9 ms, total: 217 ms\n",
      "Wall time: 2.05 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('claims', '36061')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6a864928",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: policies\n",
      "County FIPS: 36081\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 20,000\n",
      "Skip number: 20,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 30,000\n",
      "Skip number: 30,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 40,000\n",
      "Skip number: 40,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 50,000\n",
      "Skip number: 50,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 60,000\n",
      "Skip number: 60,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 70,000\n",
      "Skip number: 70,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 80,000\n",
      "Skip number: 80,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 90,000\n",
      "Skip number: 90,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 100,000\n",
      "Skip number: 100,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 110,000\n",
      "Skip number: 110,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 120,000\n",
      "Skip number: 120,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 130,000\n",
      "Skip number: 130,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 140,000\n",
      "Skip number: 140,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 150,000\n",
      "Skip number: 150,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 160,000\n",
      "Skip number: 160,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 170,000\n",
      "Skip number: 170,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 180,000\n",
      "Skip number: 180,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 190,000\n",
      "Skip number: 190,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 200,000\n",
      "Skip number: 200,000\n",
      "Number of rows: 3,657\n",
      "\n",
      "Length of the full result list: 203,657\n",
      "\n",
      "CPU times: user 30.7 s, sys: 2.78 s, total: 33.4 s\n",
      "Wall time: 3min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('policies', '36081')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cbeb28d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: claims\n",
      "County FIPS: 36081\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 4,787\n",
      "\n",
      "Length of the full result list: 14,787\n",
      "\n",
      "CPU times: user 1.95 s, sys: 145 ms, total: 2.09 s\n",
      "Wall time: 13.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('claims', '36081')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "79a0055f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: policies\n",
      "County FIPS: 36085\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 20,000\n",
      "Skip number: 20,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 30,000\n",
      "Skip number: 30,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 40,000\n",
      "Skip number: 40,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 50,000\n",
      "Skip number: 50,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 60,000\n",
      "Skip number: 60,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 70,000\n",
      "Skip number: 70,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 80,000\n",
      "Skip number: 80,000\n",
      "Number of rows: 10,000\n",
      "Result list length: 90,000\n",
      "Skip number: 90,000\n",
      "Number of rows: 6,688\n",
      "\n",
      "Length of the full result list: 96,688\n",
      "\n",
      "CPU times: user 14.2 s, sys: 1.27 s, total: 15.4 s\n",
      "Wall time: 1min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('policies', '36085')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03ea6def",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: claims\n",
      "County FIPS: 36085\n",
      "------------\n",
      "Skip number: 0\n",
      "Number of rows: 10,000\n",
      "Result list length: 10,000\n",
      "Skip number: 10,000\n",
      "Number of rows: 9,410\n",
      "\n",
      "Length of the full result list: 19,410\n",
      "\n",
      "CPU times: user 2.6 s, sys: 202 ms, total: 2.8 s\n",
      "Wall time: 19.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_data('claims', '36085')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f6d0a7b",
   "metadata": {},
   "source": [
    "# Preview Full NYC NFIP Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efe7c36b",
   "metadata": {},
   "source": [
    "## Policies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5b4a2da0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of data: (506278, 81)\n",
      "CPU times: user 31.8 s, sys: 4.42 s, total: 36.2 s\n",
      "Wall time: 36.4 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>censusBlockGroupFips</th>\n",
       "      <th>censusTract</th>\n",
       "      <th>countyCode</th>\n",
       "      <th>reportedZipCode</th>\n",
       "      <th>reportedCity</th>\n",
       "      <th>propertyState</th>\n",
       "      <th>femaRegion</th>\n",
       "      <th>...</th>\n",
       "      <th>elevatedBuildingIndicator</th>\n",
       "      <th>contentsDeductibleCode</th>\n",
       "      <th>buildingDeductibleCode</th>\n",
       "      <th>crsClassCode</th>\n",
       "      <th>construction</th>\n",
       "      <th>condominiumCoverageTypeCode</th>\n",
       "      <th>cancellationDateOfFloodPolicy</th>\n",
       "      <th>basementEnclosureCrawlspaceType</th>\n",
       "      <th>baseFloodElevation</th>\n",
       "      <th>agricultureStructureIndicator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6cdba47d-1e40-4680-8ca5-f2f4a61cb174</td>\n",
       "      <td>-74.0</td>\n",
       "      <td>40.7</td>\n",
       "      <td>3.606101e+11</td>\n",
       "      <td>3.606101e+10</td>\n",
       "      <td>36061</td>\n",
       "      <td>10011</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f07086ce-f925-431e-ab70-fe1d9ab4e653</td>\n",
       "      <td>-74.0</td>\n",
       "      <td>40.7</td>\n",
       "      <td>3.606101e+11</td>\n",
       "      <td>3.606101e+10</td>\n",
       "      <td>36061</td>\n",
       "      <td>10011</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1ded539c-1bdd-4f20-bb3d-314a92a9f9f1</td>\n",
       "      <td>-74.0</td>\n",
       "      <td>40.7</td>\n",
       "      <td>3.606101e+11</td>\n",
       "      <td>3.606101e+10</td>\n",
       "      <td>36061</td>\n",
       "      <td>10012</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b3eb3d0a-446d-4493-bcba-b48dabc701aa</td>\n",
       "      <td>-73.9</td>\n",
       "      <td>40.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36061</td>\n",
       "      <td></td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>MS</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0b1069b8-6b2b-4cb1-9ecd-af9aa8bd2170</td>\n",
       "      <td>-74.0</td>\n",
       "      <td>40.7</td>\n",
       "      <td>3.606101e+11</td>\n",
       "      <td>3.606101e+10</td>\n",
       "      <td>36061</td>\n",
       "      <td>10001</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  longitude  latitude  \\\n",
       "0  6cdba47d-1e40-4680-8ca5-f2f4a61cb174      -74.0      40.7   \n",
       "1  f07086ce-f925-431e-ab70-fe1d9ab4e653      -74.0      40.7   \n",
       "2  1ded539c-1bdd-4f20-bb3d-314a92a9f9f1      -74.0      40.7   \n",
       "3  b3eb3d0a-446d-4493-bcba-b48dabc701aa      -73.9      40.8   \n",
       "4  0b1069b8-6b2b-4cb1-9ecd-af9aa8bd2170      -74.0      40.7   \n",
       "\n",
       "   censusBlockGroupFips   censusTract  countyCode reportedZipCode  \\\n",
       "0          3.606101e+11  3.606101e+10       36061           10011   \n",
       "1          3.606101e+11  3.606101e+10       36061           10011   \n",
       "2          3.606101e+11  3.606101e+10       36061           10012   \n",
       "3                   NaN           NaN       36061                   \n",
       "4          3.606101e+11  3.606101e+10       36061           10001   \n",
       "\n",
       "            reportedCity propertyState  femaRegion  ...  \\\n",
       "0  Currently Unavailable            NY         2.0  ...   \n",
       "1  Currently Unavailable            NY         2.0  ...   \n",
       "2  Currently Unavailable            NY         2.0  ...   \n",
       "3  Currently Unavailable            MS         4.0  ...   \n",
       "4  Currently Unavailable            NY         2.0  ...   \n",
       "\n",
       "  elevatedBuildingIndicator contentsDeductibleCode  buildingDeductibleCode  \\\n",
       "0                     False                      0                       0   \n",
       "1                     False                      0                       0   \n",
       "2                     False                      0                       0   \n",
       "3                     False                      1                       1   \n",
       "4                     False                      1                       0   \n",
       "\n",
       "   crsClassCode construction  condominiumCoverageTypeCode  \\\n",
       "0           NaN        False                            N   \n",
       "1           NaN        False                            N   \n",
       "2           NaN        False                            N   \n",
       "3           NaN        False                            N   \n",
       "4           NaN        False                            N   \n",
       "\n",
       "   cancellationDateOfFloodPolicy  basementEnclosureCrawlspaceType  \\\n",
       "0                           None                              NaN   \n",
       "1                           None                              NaN   \n",
       "2                           None                              2.0   \n",
       "3                           None                              NaN   \n",
       "4                           None                              2.0   \n",
       "\n",
       "   baseFloodElevation  agricultureStructureIndicator  \n",
       "0                 NaN                          False  \n",
       "1                 NaN                          False  \n",
       "2                 NaN                          False  \n",
       "3                 NaN                          False  \n",
       "4                 NaN                          False  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "path = 'data/'\n",
    "all_files = glob.glob(os.path.join(path, \"policies-*.json\"))\n",
    "\n",
    "# concat all counties into one dataframe\n",
    "df = pd.concat((pd.read_json(file) for file in all_files), ignore_index=True)\n",
    "\n",
    "# reverse column order\n",
    "df = df.loc[:, df.columns[::-1]]\n",
    "\n",
    "# examine data\n",
    "print('shape of data: {}'.format(df.shape))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "97611139",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 506278 entries, 0 to 506277\n",
      "Data columns (total 81 columns):\n",
      " #   Column                                  Non-Null Count   Dtype  \n",
      "---  ------                                  --------------   -----  \n",
      " 0   id                                      506278 non-null  object \n",
      " 1   longitude                               506278 non-null  float64\n",
      " 2   latitude                                506278 non-null  float64\n",
      " 3   censusBlockGroupFips                    503883 non-null  float64\n",
      " 4   censusTract                             503883 non-null  float64\n",
      " 5   countyCode                              506278 non-null  int64  \n",
      " 6   reportedZipCode                         506278 non-null  object \n",
      " 7   reportedCity                            506278 non-null  object \n",
      " 8   propertyState                           506278 non-null  object \n",
      " 9   femaRegion                              506277 non-null  float64\n",
      " 10  floodZoneCurrent                        375333 non-null  object \n",
      " 11  mapPanelSuffix                          489431 non-null  object \n",
      " 12  mapPanelNumber                          487939 non-null  float64\n",
      " 13  programTypeIndicator                    506278 non-null  bool   \n",
      " 14  nfipCommunityName                       389977 non-null  object \n",
      " 15  nfipCommunityNumberCurrent              375359 non-null  float64\n",
      " 16  nfipRatedCommunityNumber                506278 non-null  int64  \n",
      " 17  grandfatheringTypeCode                  482689 non-null  float64\n",
      " 18  mandatoryPurchaseFlag                   506278 non-null  bool   \n",
      " 19  disasterAssistanceCoverageRequiredCode  385156 non-null  float64\n",
      " 20  stateOwnedIndicator                     506278 non-null  bool   \n",
      " 21  tenantIndicator                         506278 non-null  bool   \n",
      " 22  rentalPropertyIndicator                 506278 non-null  bool   \n",
      " 23  propertyPurchaseDate                    163471 non-null  object \n",
      " 24  endorsementEffectiveDate                415741 non-null  object \n",
      " 25  rolloverTransferCode                    504785 non-null  object \n",
      " 26  waitingPeriodType                       340782 non-null  object \n",
      " 27  floodproofedIndicator                   506278 non-null  bool   \n",
      " 28  postFirmVzoneIndicator                  506278 non-null  bool   \n",
      " 29  insuranceToValueCode                    254331 non-null  float64\n",
      " 30  buildingDescriptionCode                 406212 non-null  float64\n",
      " 31  enclosureTypeCode                       185863 non-null  object \n",
      " 32  AdditionalContentsRate                  432441 non-null  float64\n",
      " 33  basicContentsRate                       432441 non-null  float64\n",
      " 34  additionalBuildingRate                  432536 non-null  float64\n",
      " 35  basicBuildingRate                       432536 non-null  float64\n",
      " 36  buildingReplacementCost                 491627 non-null  float64\n",
      " 37  premiumPaymentIndicator                 238135 non-null  float64\n",
      " 38  communityProbationSurcharge             444970 non-null  float64\n",
      " 39  reserveFundAssessment                   506278 non-null  int64  \n",
      " 40  iccPremium                              501816 non-null  float64\n",
      " 41  subsidizedRateType                      281831 non-null  object \n",
      " 42  cancellationVoidanceReasonCode          13657 non-null   object \n",
      " 43  totalInsurancePremiumOfThePolicy        506278 non-null  int64  \n",
      " 44  totalContentsInsuranceCoverage          506263 non-null  float64\n",
      " 45  totalBuildingInsuranceCoverage          506278 non-null  int64  \n",
      " 46  smallBusinessIndicatorBuilding          506278 non-null  bool   \n",
      " 47  regularEmergencyProgramIndicator        506278 non-null  object \n",
      " 48  rateMethod                              506276 non-null  object \n",
      " 49  primaryResidenceIndicator               506278 non-null  bool   \n",
      " 50  postFIRMConstructionIndicator           506278 non-null  bool   \n",
      " 51  policyTermIndicator                     506278 non-null  int64  \n",
      " 52  policyTerminationDate                   506278 non-null  object \n",
      " 53  policyEffectiveDate                     506278 non-null  object \n",
      " 54  policyCount                             506278 non-null  int64  \n",
      " 55  policyCost                              506278 non-null  int64  \n",
      " 56  originalNBDate                          506146 non-null  object \n",
      " 57  originalConstructionDate                506114 non-null  object \n",
      " 58  occupancyType                           506278 non-null  int64  \n",
      " 59  obstructionType                         62818 non-null   float64\n",
      " 60  numberOfFloorsInInsuredBuilding         503938 non-null  float64\n",
      " 61  nonProfitIndicator                      506278 non-null  bool   \n",
      " 62  lowestFloorElevation                    111387 non-null  float64\n",
      " 63  lowestAdjacentGrade                     110567 non-null  float64\n",
      " 64  locationOfContents                      339937 non-null  float64\n",
      " 65  houseOfWorshipIndicator                 506278 non-null  bool   \n",
      " 66  hfiaaSurcharge                          506278 non-null  int64  \n",
      " 67  ratedFloodZone                          504080 non-null  object \n",
      " 68  federalPolicyFee                        506278 non-null  int64  \n",
      " 69  elevationDifference                     111439 non-null  float64\n",
      " 70  elevationCertificateIndicator           185924 non-null  object \n",
      " 71  elevatedBuildingIndicator               506278 non-null  bool   \n",
      " 72  contentsDeductibleCode                  417425 non-null  object \n",
      " 73  buildingDeductibleCode                  491189 non-null  object \n",
      " 74  crsClassCode                            19 non-null      float64\n",
      " 75  construction                            506278 non-null  bool   \n",
      " 76  condominiumCoverageTypeCode             506137 non-null  object \n",
      " 77  cancellationDateOfFloodPolicy           13657 non-null   object \n",
      " 78  basementEnclosureCrawlspaceType         400634 non-null  float64\n",
      " 79  baseFloodElevation                      111115 non-null  float64\n",
      " 80  agricultureStructureIndicator           506278 non-null  bool   \n",
      "dtypes: bool(15), float64(29), int64(11), object(26)\n",
      "memory usage: 262.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a286c826",
   "metadata": {},
   "source": [
    "## Claims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cf9de75c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of data: (43833, 73)\n",
      "CPU times: user 2.7 s, sys: 290 ms, total: 2.99 s\n",
      "Wall time: 3.01 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>censusBlockGroupFips</th>\n",
       "      <th>censusTract</th>\n",
       "      <th>countyCode</th>\n",
       "      <th>reportedZipCode</th>\n",
       "      <th>reportedCity</th>\n",
       "      <th>state</th>\n",
       "      <th>rentalPropertyIndicator</th>\n",
       "      <th>...</th>\n",
       "      <th>baseFloodElevation</th>\n",
       "      <th>elevationDifference</th>\n",
       "      <th>elevationCertificateIndicator</th>\n",
       "      <th>elevatedBuildingIndicator</th>\n",
       "      <th>dateOfLoss</th>\n",
       "      <th>crsClassificationCode</th>\n",
       "      <th>policyCount</th>\n",
       "      <th>basementEnclosureCrawlspaceType</th>\n",
       "      <th>asOfDate</th>\n",
       "      <th>agricultureStructureIndicator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7279caf2-e180-474f-9072-ce5d6382278a</td>\n",
       "      <td>-73.8</td>\n",
       "      <td>40.8</td>\n",
       "      <td>3.600501e+11</td>\n",
       "      <td>3.600501e+10</td>\n",
       "      <td>36005</td>\n",
       "      <td>10465</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-09-02T00:00:00.000Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-05-19T15:56:42.000Z</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e5f8240c-06d7-48f9-b148-82040a1e5261</td>\n",
       "      <td>-73.8</td>\n",
       "      <td>40.8</td>\n",
       "      <td>3.600501e+11</td>\n",
       "      <td>3.600501e+10</td>\n",
       "      <td>36005</td>\n",
       "      <td>10465</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>2012-10-29T00:00:00.000Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-22T16:55:53.000Z</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abc053a4-1d37-420c-8fba-a2c97ef52dda</td>\n",
       "      <td>-73.8</td>\n",
       "      <td>40.8</td>\n",
       "      <td>3.600502e+11</td>\n",
       "      <td>3.600502e+10</td>\n",
       "      <td>36005</td>\n",
       "      <td>10465</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017-11-22T00:00:00.000Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-22T16:55:53.000Z</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7764e429-a818-48e4-ba0a-99bb50a67f5c</td>\n",
       "      <td>-73.8</td>\n",
       "      <td>40.8</td>\n",
       "      <td>3.600501e+11</td>\n",
       "      <td>3.600501e+10</td>\n",
       "      <td>36005</td>\n",
       "      <td>10465</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>2012-10-29T00:00:00.000Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020-01-22T16:55:53.000Z</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a5e6a2e2-4798-47b1-bfcd-cdae0cc1d11b</td>\n",
       "      <td>-73.8</td>\n",
       "      <td>40.9</td>\n",
       "      <td>3.600505e+11</td>\n",
       "      <td>3.600505e+10</td>\n",
       "      <td>36005</td>\n",
       "      <td>10464</td>\n",
       "      <td>Currently Unavailable</td>\n",
       "      <td>NY</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>2012-10-29T00:00:00.000Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-22T16:55:53.000Z</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 73 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  longitude  latitude  \\\n",
       "0  7279caf2-e180-474f-9072-ce5d6382278a      -73.8      40.8   \n",
       "1  e5f8240c-06d7-48f9-b148-82040a1e5261      -73.8      40.8   \n",
       "2  abc053a4-1d37-420c-8fba-a2c97ef52dda      -73.8      40.8   \n",
       "3  7764e429-a818-48e4-ba0a-99bb50a67f5c      -73.8      40.8   \n",
       "4  a5e6a2e2-4798-47b1-bfcd-cdae0cc1d11b      -73.8      40.9   \n",
       "\n",
       "   censusBlockGroupFips   censusTract  countyCode reportedZipCode  \\\n",
       "0          3.600501e+11  3.600501e+10       36005           10465   \n",
       "1          3.600501e+11  3.600501e+10       36005           10465   \n",
       "2          3.600502e+11  3.600502e+10       36005           10465   \n",
       "3          3.600501e+11  3.600501e+10       36005           10465   \n",
       "4          3.600505e+11  3.600505e+10       36005           10464   \n",
       "\n",
       "            reportedCity state  rentalPropertyIndicator  ...  \\\n",
       "0  Currently Unavailable    NY                    False  ...   \n",
       "1  Currently Unavailable    NY                    False  ...   \n",
       "2  Currently Unavailable    NY                    False  ...   \n",
       "3  Currently Unavailable    NY                    False  ...   \n",
       "4  Currently Unavailable    NY                    False  ...   \n",
       "\n",
       "   baseFloodElevation elevationDifference  elevationCertificateIndicator  \\\n",
       "0                 NaN                 NaN                            NaN   \n",
       "1                 NaN                 NaN                            NaN   \n",
       "2                 NaN                 NaN                            1.0   \n",
       "3                 NaN                 NaN                            NaN   \n",
       "4                 NaN                 NaN                            NaN   \n",
       "\n",
       "   elevatedBuildingIndicator                dateOfLoss  crsClassificationCode  \\\n",
       "0                      False  2021-09-02T00:00:00.000Z                    NaN   \n",
       "1                      False  2012-10-29T00:00:00.000Z                    NaN   \n",
       "2                      False  2017-11-22T00:00:00.000Z                    NaN   \n",
       "3                      False  2012-10-29T00:00:00.000Z                    NaN   \n",
       "4                      False  2012-10-29T00:00:00.000Z                    NaN   \n",
       "\n",
       "   policyCount  basementEnclosureCrawlspaceType                  asOfDate  \\\n",
       "0            1                              0.0  2022-05-19T15:56:42.000Z   \n",
       "1            1                              1.0  2020-01-22T16:55:53.000Z   \n",
       "2            1                              1.0  2020-01-22T16:55:53.000Z   \n",
       "3            1                              2.0  2020-01-22T16:55:53.000Z   \n",
       "4            1                              1.0  2020-01-22T16:55:53.000Z   \n",
       "\n",
       "   agricultureStructureIndicator  \n",
       "0                          False  \n",
       "1                          False  \n",
       "2                          False  \n",
       "3                          False  \n",
       "4                          False  \n",
       "\n",
       "[5 rows x 73 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "path = 'data/'\n",
    "all_files = glob.glob(os.path.join(path, \"claims-*.json\"))\n",
    "\n",
    "# concat all counties into one dataframe\n",
    "df = pd.concat((pd.read_json(file) for file in all_files), ignore_index=True)\n",
    "\n",
    "# reverse column order\n",
    "df = df.loc[:, df.columns[::-1]]\n",
    "\n",
    "# examine data\n",
    "print('shape of data: {}'.format(df.shape))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "491f0057",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 43833 entries, 0 to 43832\n",
      "Data columns (total 73 columns):\n",
      " #   Column                                      Non-Null Count  Dtype  \n",
      "---  ------                                      --------------  -----  \n",
      " 0   id                                          43833 non-null  object \n",
      " 1   longitude                                   43833 non-null  float64\n",
      " 2   latitude                                    43833 non-null  float64\n",
      " 3   censusBlockGroupFips                        43416 non-null  float64\n",
      " 4   censusTract                                 43416 non-null  float64\n",
      " 5   countyCode                                  43833 non-null  int64  \n",
      " 6   reportedZipCode                             43833 non-null  object \n",
      " 7   reportedCity                                43833 non-null  object \n",
      " 8   state                                       43833 non-null  object \n",
      " 9   rentalPropertyIndicator                     43833 non-null  bool   \n",
      " 10  buildingDescriptionCode                     17404 non-null  float64\n",
      " 11  floodZoneCurrent                            11647 non-null  object \n",
      " 12  waterDepth                                  41645 non-null  float64\n",
      " 13  stateOwnedIndicator                         43833 non-null  bool   \n",
      " 14  replacementCostBasis                        40144 non-null  object \n",
      " 15  contentsReplacementCost                     19618 non-null  float64\n",
      " 16  buildingReplacementCost                     34434 non-null  float64\n",
      " 17  numberOfUnits                               43815 non-null  float64\n",
      " 18  nonPaymentReasonBuilding                    8656 non-null   float64\n",
      " 19  nonPaymentReasonContents                    11170 non-null  float64\n",
      " 20  nfipCommunityName                           11919 non-null  object \n",
      " 21  nfipCommunityNumberCurrent                  11648 non-null  float64\n",
      " 22  nfipRatedCommunityNumber                    43833 non-null  int64  \n",
      " 23  netIccPaymentAmount                         43833 non-null  float64\n",
      " 24  iccCoverage                                 23700 non-null  float64\n",
      " 25  floodEvent                                  29498 non-null  object \n",
      " 26  floodproofedIndicator                       43833 non-null  bool   \n",
      " 27  floodWaterDuration                          41381 non-null  float64\n",
      " 28  floodCharacteristicsIndicator               133 non-null    float64\n",
      " 29  ficoNumber                                  27874 non-null  float64\n",
      " 30  eventDesignationNumber                      1184 non-null   object \n",
      " 31  disasterAssistanceCoverageRequired          38292 non-null  float64\n",
      " 32  contentsPropertyValue                       19618 non-null  float64\n",
      " 33  netContentsPaymentAmount                    43833 non-null  float64\n",
      " 34  contentsDeductibleCode                      25750 non-null  object \n",
      " 35  contentsDamageAmount                        19618 non-null  float64\n",
      " 36  condominiumCoverageTypeCode                 41669 non-null  object \n",
      " 37  causeOfDamage                               43577 non-null  object \n",
      " 38  buildingPropertyValue                       34434 non-null  float64\n",
      " 39  netBuildingPaymentAmount                    43833 non-null  float64\n",
      " 40  buildingDeductibleCode                      29275 non-null  object \n",
      " 41  buildingDamageAmount                        34434 non-null  float64\n",
      " 42  primaryResidenceIndicator                   43833 non-null  bool   \n",
      " 43  yearOfLoss                                  43833 non-null  int64  \n",
      " 44  totalContentsInsuranceCoverage              43812 non-null  float64\n",
      " 45  totalBuildingInsuranceCoverage              43812 non-null  float64\n",
      " 46  smallBusinessIndicatorBuilding              43833 non-null  bool   \n",
      " 47  rateMethod                                  40215 non-null  object \n",
      " 48  postFIRMConstructionIndicator               43833 non-null  bool   \n",
      " 49  amountPaidOnIncreasedCostOfComplianceClaim  36710 non-null  float64\n",
      " 50  amountPaidOnContentsClaim                   36710 non-null  float64\n",
      " 51  amountPaidOnBuildingClaim                   36710 non-null  float64\n",
      " 52  originalNBDate                              43833 non-null  object \n",
      " 53  originalConstructionDate                    43829 non-null  object \n",
      " 54  occupancyType                               43805 non-null  float64\n",
      " 55  obstructionType                             20441 non-null  float64\n",
      " 56  nonProfitIndicator                          43833 non-null  bool   \n",
      " 57  numberOfFloorsInTheInsuredBuilding          43753 non-null  float64\n",
      " 58  lowestFloorElevation                        5846 non-null   float64\n",
      " 59  lowestAdjacentGrade                         5696 non-null   float64\n",
      " 60  locationOfContents                          28059 non-null  float64\n",
      " 61  houseWorship                                43833 non-null  bool   \n",
      " 62  ratedFloodZone                              28947 non-null  object \n",
      " 63  baseFloodElevation                          5859 non-null   float64\n",
      " 64  elevationDifference                         5910 non-null   float64\n",
      " 65  elevationCertificateIndicator               12865 non-null  object \n",
      " 66  elevatedBuildingIndicator                   43833 non-null  bool   \n",
      " 67  dateOfLoss                                  43833 non-null  object \n",
      " 68  crsClassificationCode                       1 non-null      float64\n",
      " 69  policyCount                                 43833 non-null  int64  \n",
      " 70  basementEnclosureCrawlspaceType             25040 non-null  float64\n",
      " 71  asOfDate                                    43833 non-null  object \n",
      " 72  agricultureStructureIndicator               43833 non-null  bool   \n",
      "dtypes: bool(10), float64(39), int64(4), object(20)\n",
      "memory usage: 21.5+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b05753",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}