End of Life Linear Model Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import time\n",
    "import importlib\n",
    "import sparse\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import Utils.dbutils as dbutils\n",
    "import Utils.data_utils as data_utils\n",
    "import Generators.CohortGenerator as CohortGenerator\n",
    "import Generators.FeatureGenerator as FeatureGenerator\n",
    "import config\n",
    "local_imports = (\n",
    "    dbutils,\n",
    "    data_utils,\n",
    "    CohortGenerator,\n",
    "    FeatureGenerator,\n",
    "    config\n",
    ")\n",
    "for i in local_imports:\n",
    "    i = importlib.reload(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohort, Outcome and Feature Collection\n",
    "\n",
    "### 1. Set up a connection to the OMOP CDM database\n",
    "\n",
    "Parameters for connection to be specified in ./config.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executed 1 SQLs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kodialam/.local/lib/python3.5/site-packages/sqlalchemy/dialects/postgresql/base.py:3285: SAWarning: Skipped unsupported reflection of expression-based index idx__loc_source_value\n",
      "  \"expression-based index %s\" % idx_name\n"
     ]
    }
   ],
   "source": [
    "## database connection parameters\n",
    "# username = config.PG_USERNAME #we use peer authentication so don't need use vars, but in theory would pass them into config_path\n",
    "# password = config.PG_PASSWORD\n",
    "database_name = config.DB_NAME\n",
    "config_path = 'postgresql://{database_name}'.format(\n",
    "    database_name = database_name\n",
    ")\n",
    "connect_args = {\"host\": '/var/run/postgresql/'} # connect_args to pass to sqlalchemy create_engine function\n",
    "\n",
    "# schemas \n",
    "schema_name = 'eol_test' # all created tables will be created using this schema\n",
    "cdm_schema_name = config.OMOP_CDM_SCHEMA # the name of the schema housing your OMOP CDM tables\n",
    "\n",
    "# caching\n",
    "reset_schema = False # if true, rebuild all data from scratch\n",
    "\n",
    "# set up database, reset schemas as needed\n",
    "db = dbutils.Database(config_path, schema_name, connect_args, cdm_schema_name)\n",
    "if reset_schema:\n",
    "    db.execute(\n",
    "        'drop schema if exists {} cascade'.format(schema_name)\n",
    "    )\n",
    "db.execute(\n",
    "    'create schema if not exists {}'.format(schema_name)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Example Query of an OMOP-standard Database\n",
    "Wiki page of expected tables [here](https://github.com/OHDSI/CommonDataModel/wiki)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_name</th>\n",
       "      <th>concept_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pure hypercholesterolemia</td>\n",
       "      <td>ICD9CM: 272.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Benign prostatic hypertrophy with outflow obst...</td>\n",
       "      <td>ICD9CM: 600.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Acute pharyngitis</td>\n",
       "      <td>ICD9CM: 462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cellulitis and abscess of buttock</td>\n",
       "      <td>ICD9CM: 682.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>No matching concept</td>\n",
       "      <td>ICD9CM: 786.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        concept_name    concept_code\n",
       "0                          Pure hypercholesterolemia   ICD9CM: 272.0\n",
       "1  Benign prostatic hypertrophy with outflow obst...  ICD9CM: 600.01\n",
       "2                                  Acute pharyngitis     ICD9CM: 462\n",
       "3                  Cellulitis and abscess of buttock   ICD9CM: 682.5\n",
       "4                                No matching concept   ICD9CM: 786.4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sql = \"\"\"\n",
    "    select \n",
    "        b.concept_name as concept_name,\n",
    "        a.condition_source_value as concept_code\n",
    "    from\n",
    "        {omop_schema}.condition_occurrence a\n",
    "    inner join\n",
    "        {omop_schema}.concept b\n",
    "    on \n",
    "        a.condition_concept_id = b.concept_id\n",
    "    limit 100\n",
    "\"\"\".format(\n",
    "    omop_schema=config.OMOP_CDM_SCHEMA\n",
    ")\n",
    "output_df = db.query(sql)\n",
    "display(output_df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a Cohort for the End of Life Prediction Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create a Cohort Object that can be constructed as specified by SQL File & Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cohort_name = '__eol_cohort'\n",
    "if config.CDM_VERSION == 'v5.3.1':\n",
    "    cohort_script_path = config.SQL_PATH_COHORTS + '/gen_EOL_cohort_v531.sql'\n",
    "else:\n",
    "    cohort_script_path = config.SQL_PATH_COHORTS + '/gen_EOL_cohort.sql'\n",
    "\n",
    "# cohort parameters  \n",
    "params = {\n",
    "          'cohort_table_name'     : cohort_name,\n",
    "          'schema_name'           : schema_name,\n",
    "          'aux_data_schema'       : config.CDM_AUX_SCHEMA,\n",
    "          'training_start_date'   : '2016-01-01',\n",
    "          'training_end_date'     : '2017-01-01',\n",
    "          'gap'                   : '3 months',\n",
    "          'outcome_window'        : '6 months'\n",
    "         }\n",
    "\n",
    "cohort = CohortGenerator.Cohort(\n",
    "    schema_name=schema_name,\n",
    "    cohort_table_name=cohort_name,\n",
    "    cohort_generation_script=cohort_script_path,\n",
    "    cohort_generation_kwargs=params,\n",
    "    outcome_col_name='y'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Build Cohort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Regenerating Table (replace=True)\n",
      "Regenerated Cohort in 22.463050603866577 seconds\n"
     ]
    }
   ],
   "source": [
    "cohort.build(db, replace=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get a Time Series of Features for Cohort Members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "featureSet = FeatureGenerator.FeatureSet(db)\n",
    "featureSet.add_default_features(\n",
    "    ['drugs','conditions','procedures','specialty'],\n",
    "    schema_name,\n",
    "    cohort_name\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data loaded to buffer in 1445.19 seconds\n",
      "Got Unique Concepts and Timestamps in 160.23 seconds\n",
      "Created Index Mappings in 0.05 seconds\n",
      "136795\n",
      "Generated Sparse Representation of Data in 517.52 seconds\n",
      "CPU times: user 10min 4s, sys: 2min 29s, total: 12min 34s\n",
      "Wall time: 35min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Build the Feature Set by executing SQL queries and reading into sparse matrices\n",
    "cache_data_path = '/tmp/cache_data_eol_test'\n",
    "featureSet.build(cohort, from_cached=False, cache_file=cache_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "outcomes_filt, feature_matrix_3d_transpose, remap, good_feature_names = \\\n",
    "    FeatureGenerator.postprocess_feature_matrix(cohort, featureSet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a Model to Predict End of Life using this data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform Data to get a vector of counts over 1, 6, 12, 24 and infinity (represented by 10000 days) month windows for each Patient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14min 58s, sys: 2min 40s, total: 17min 38s\n",
      "Wall time: 17min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "feature_matrix_counts, feature_names = data_utils.window_data(\n",
    "    window_lengths = [30, 180, 365, 730, 10000],\n",
    "    feature_matrix = feature_matrix_3d_transpose,\n",
    "    all_feature_names = good_feature_names,\n",
    "    cohort = cohort,\n",
    "    featureSet = featureSet\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set up a standard sklearn modelling pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve, average_precision_score\n",
    "\n",
    "lr = LogisticRegression(\n",
    "    class_weight='balanced', C=0.02,\n",
    "    penalty='l1', fit_intercept=True,\n",
    "    solver='liblinear', random_state=0,\n",
    "    verbose=0, max_iter = 200, tol=1e-1\n",
    ")\n",
    "\n",
    "def sparse_ufunc(f):\n",
    "    def wrapper(*a, **k):\n",
    "        X = a[0]\n",
    "        if not scipy.sparse.isspmatrix(X):\n",
    "            raise ValueError\n",
    "        X2 = X.copy()\n",
    "        X2.data = f(X2.data, *(a[1:]), **k)\n",
    "        return X2\n",
    "    return wrapper\n",
    "\n",
    "@sparse_ufunc\n",
    "def tr_func(X, kwarg=1):\n",
    "    return np.clip(X, 0, kwarg)\n",
    "\n",
    "func = FunctionTransformer(\n",
    "    func=tr_func,\n",
    "    accept_sparse=True,\n",
    "    validate=True,\n",
    "    kw_args={'kwarg': 1}\n",
    ")\n",
    "\n",
    "# The classifier will transform each data point using func, which here takes a count vector to a binary vector\n",
    "# Then, it will use logistic regression to classify the transformed data\n",
    "clf = Pipeline([\n",
    "    ('func',func),\n",
    "    ('lr', lr)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate, Train and Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Test AUC: 0.80\n"
     ]
    }
   ],
   "source": [
    "indices = range(len(outcomes_filt))\n",
    "X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(\n",
    "    feature_matrix_counts.T, outcomes_filt, indices,\n",
    "    test_size=0.2, random_state=1\n",
    ")\n",
    "clf.fit(X_train, y_train)\n",
    "pred = clf.predict_proba(X_test)[:, 1]\n",
    "print('Model Test AUC: {0:.2f}'.format(roc_auc_score(y_test, pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model : RoC and Precision-Recall Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd803X+wPHXuyldFFqggOyCIEMEQaZyqOBAQDhFFOXEjQs9RUTLeZ7neQIOHMApOM/JKZ6/U8GJ4B6goLJlU0qhlO6d5PP745O2oXSk0DRN+34+Hnm0+eY73knTvPPZYoxBKaWUqkhIoANQSilVt2miUEopVSlNFEoppSqliUIppVSlNFEopZSqlCYKpZRSldJEEQREZLKIfBLoOAJNRDqKSLaIOGrxmvEiYkQktLau6U8iskFEzjqG4/Q9WIaIXC0iX/u478si8pC/Y/IXTRTVJCK7RCTP84GV7HkDRPvzmsaY140x5/nzGnWR57U+p/i+MWaPMSbaGOMKZFyB4klYXY/nHMaYk40xq6q4zlHJ0V/vQc+Hrcvz/+R9a+vDsT59+Ho9n7VltseJSKGI7DqOp9AgaKI4NhcaY6KBU4F+QEKA4zkmgfyWXF++oVeHvt4V+s7zBcD7luSH60SJSG+v+1cAO/1wnXpHE8VxMMYkAx9jEwYAIhIuIo+JyB4ROSAiz4pIpNfj40VknYhkish2ERnl2R4jIi+IyH4R2SciDxVXsXgXcUXkGRF5zDsOEfmfiEz3/N5WRN4RkRQR2Skit3vt94CILBWR10QkE7i67HPyxPGK5/jdInKfiIR4xfGNiCwQkQwR2SwiI8scW9lz+EZEnhCRVOABETlRRD4XkVQROSQir4tIrGf/V4GOwPueb5gzy37TFZFVIvIPz3mzROQTEYnzimeK5zmkishfy5ZQyjzvSBF53LN/hoh87f13AyZ7/qaHROQvXscNEpHvRCTd87wXiEiY1+NGRG4Vkd+B3z3bnhKRvZ73wE8i8gev/R0iMsvz3sjyPN5BRL707PKL5/W4zLP/WM/7KV1EvhWRPl7n2iUi94jIr0COiIR6vwae2Nd44jggIvM8hxZfK91zraFSpppFRE4WkU9F5LDn2Fnlva7HS0R6ev7O6WKrzcYdx+leBa7yuj8FeMXX64lICxF5z/N6/QicWObYHl6vyRYRufQ4Yq1bjDF6q8YN2AWc4/m9PfAb8JTX408A7wHNgSbA+8Bsz2ODgAzgXGySbgf08Dz2LrAIaAy0An4EbvQ8djXwtef34cBeQDz3mwF5QFvPOX8C7gfCgC7ADuB8z74PAEXAHz37Rpbz/F4B/ueJPR7YClznFYcTuBNoBFzmeT7NfXwOTuA2IBSIBLp6XotwoCX2A+rJ8l5rz/14wAChnvurgO3ASZ7zrQLmeB7rBWQDwzyvxWOe535OBX/XhZ7j2wEO4HRPXMXXfM5zjb5AAdDTc9xpwBDPc4oHNgF3eJ3XAJ9i3w+Rnm1/Alp4jrkLSAYiPI/djX1PdQfEc70WXufq6nXufsBBYLAn5qs8r1m41+u3Dujgde2S1xT4DrjS83s0MKS817mc92ATYL8n9gjP/cHH+P9Uct5yHmsEbANmef6GI4AsoLvn8ZeBh3y4RvHzicf+7zg874/NwDnALh+vtwR4C/v+7g3s83pNGnvOfY3n79oPOAT0qk6sdfUW8ACC7eb5R8v2vIEMsAKI9TwmQA5wotf+Q4Gdnt8XAU+Uc87W2A+fSK9tlwMrPb97/5MKsAcY7rl/A/C55/fBwJ4y504AXvL8/gDwZSXPzQEUFr+5PdtuBFZ5xZGEJ0l5tv0IXOnjc9hT0bU9+/wRWFvmta4qUdzn9fgtwEee3+8H3vR6LMrz3I5KFNikmQf0Leex4mu2L/OcJ1XwHO4A3vW6b4ARVTzvtOJrA1uA8RXsVzZRPAP8o8w+W4AzvV6/a8t5/xYnii+BvwNxFTznihLF5d5/p+P8f7oa+wUi3eu23fPYH7BJNMRr/zeBBzy/v0z1EkUo8BlwPjAH+AtHJooKr4f93yjC88XO89jDXq/JZcBXZa67CPhbdWKtq7e6XG9Zl/3RGPOZiJwJvAHEYd/gLbEfSD+JSPG+gn2Tgf1mt7yc83XCfpvZ73VcCPYbyhGMMUZElmD/Wb/E1rO+5nWetiKS7nWIA/jK6/5R5/QS54ljt9e23dhv2cX2Gc873+vxtj4+hyOuLSKtgaew/6BNPPunVRJfeZK9fs/FfjPGE1PJ9YwxuZ4qr/LEYb8Zb6/udUTkJGAeMAD7tw/Fluq8lX3eM4DrPDEaoKknBrDvkcri8NYJuEpEbvPaFuY5b7nXLuM64EFgs4jsBP5ujPnAh+v6FKOIdAQ2Ft83tl2vPN8bY4aVs70tsNcY4/baVvb9WF2vYJPT6dj33Uk+Xq8l9m+7t8xjxToBg8v874Viq7uCnrZRHAdjzBfYbwrFbQaHsN9MTzbGxHpuMV7/IHspU6/ptb0A+82u+LimxpiTK7j0m8AlItIJW4p4x+s8O73OEWuMaWKMGe0ddiVP6RD2W1Mnr20dsUXsYu3EKxN4Hk/y8TmUvfbDnm2nGGOaYqtkpJL9q2M/tmoQsG0Q2Oqe8hwC8in/b1OVZ7BVGN08z2EWRz4H8HoenvaImcClQDNjTCy2+q74mIreI+XZC/yzzN87yhjzZnnXLssY87sx5nJsNeFcYKmINK7sGK/rdqkqOFPaSy26kiRRmSSgg3jayDzKvh+r6x1gDLDDGLOnGtdLwZZ8OpR5rNhe4Isyf4toY8zNxxFrnaGJ4vg9CZwrIn0930SeA54QkVYAItJORM737PsCcI2IjBSREM9jPYwx+4FPgMdFpKnnsRM9JZajGGPWYj/cngc+NsYUf4v5EcjyNGBGehpGe4vIQF+eiLHdTt8C/ikiTTyJaDqlJRawHyq3i0gjEZkI9ASWV/c5eDTBVuNliEg7bP28twP48IFUgaXAhSJyuqdx+QGO/gAHwPN3exGYJ7YzgMPTgBvuw3WaAJlAtoj0AKr6YGiC/cBJAUJF5H5siaLY88A/RKSbWH1EpDjBlX09ngNuEpHBnn0bi8gYEWniQ9yIyJ9EpKXn+Re/h9ye2NxU/Np/ALQRkTvEdt5oIiKDfblmNf2ALb3N9LzfzgIuxLYVFHOISITXLay8ExUzxuRg2x6ur871PP8b/8V2wogSkV4c2TD+AXCSiFzpObaRiAwUkZ7H9MzrGE0Ux8kYk4Itzt7v2XQPtkHse7E9iz7DNkxijPkR29j1BPZb5BeUfnufgq022IitflkKtKnk0m9g61ff8IrFBYzF9sLaSWkyianGU7oN286yA/jac/4XvR7/AejmOfc/gUuMMcVVOtV9Dn8H+mNfi2XYf0Rvs4H7PD1QZlTjOWCM2eB5LkuwpYtsbMNvQQWHzMA2Iq8GDmO/Yfvy/zEDW/2Xhf3g/k8V+38MfITtJLAbW5Lxrs6Yh03Wn2AT0AvYRnSwye7fntfjUmPMGmwb1QLs672NcnqyVWIUsEFEsrFVgJOMMXnGmFzs3/Ybz7WGeB9kjMnCdkK4EFsl9ztwdjWuW9ZQOXocxUBjTKHnGhdg32//AqYYYzZ7HXsvthRffPu8qosZY9YYY46qOvPhetOwVY7J2JqEl7yOzQLOAyZhSybJ2PeQL1826rzinjNKVUlErgaur6A+uU4TOygyHVtFtDPQ8SgVTLREoeotEbnQU03QGNuO9Bu2149Sqho0Uaj6bDy2GiAJW102yWgRWqlq06onpZRSldIShVJKqUoF3YC7uLg4Ex8fH+gwlFIqqPz000+HjDEtj+XYoEsU8fHxrFmzJtBhKKVUUBGR3VXvVT6telJKKVUpTRRKKaUqpYlCKaVUpTRRKKWUqpQmCqWUUpXSRKGUUqpSfksUIvKiiBwUkfUVPC4i8rSIbBORX0Wkv79iUUopdez8WaJ4GTuNcUUuwM6/0w2Yil0ARimlVA0rLHQd1/F+G3BnjPlSROIr2WU88IpnkrbvRSRWRNp4FsBRSinlxRjIzAS3u+J9iorg4EEI9fpkf/TRT9i4MRmIOOa1MQI5MrsdRy7YkujZdlSiEJGp2FIHHTt2LPuwUkrVO04npKWVJobERMjPh8LCyo8rKICcnNL7UVGtWLPmRyC80bHGEhRTeBhjFgOLAQYMGKDT3Sqlglp2NqSnH7nN6YTU1KP3K/BakzE11SaO6EpWIN+zJ4WkpP2MGdMHgEsv7cspp8Rz442PlbsUsC8CmSj2ceRC5e05vkXTlVKqzigstB/+eXk2KYR4tQgfPgyHDoGrTNNBURFkZR25LSoKwj2VRi1bQrNm5V8vP7+IF174klde+RaHQxg8uD0dOjQHhDZtYo/ruQQyUbwHTBORJcBgIEPbJ5RSwcoYWwJITbVVRkVF9gZHlwzAJoQWLY5MIKGh0KoVhIVV79rffPM7jzyynH37bDFl/PjTiImJrOIo3/ktUYjIm8BZQJyIJAJ/AxoBGGOeBZYDo7ELwucC1/grFqWUOl5FRbYUkJwMjcqp7S8osLfCQtvoXNyeEBFhSw7Nmh3ZyNyihX3seBw8mMnjj3/MihUbAejWrTUJCWPo06dDFUdWjz97PV1exeMGuNVf11dKqeoy5sheRfn5ttooOdnez8qyDcXZ2eUf73LZc0RFQZs2EBkJcswtA1WbO3c5X3yxhYiIRtx001lMmjSE0NCaH/UQFI3ZSilVlstVWrVTVmGhbQeozoe0222PKdtukJVVmjCMsQ3JJ5xQfvVQSMiRpQZ/cDrdJcngttvOITTUwZ13nscJJ8T47ZqaKJRSdUZOztEf1MWSk0s/+J1O+62+okQBkJtrP+Crw+WyicG73cDthubNoWNH/5cQKpOdnc+//vU5e/YcZv78yYgI8fFxzJ070e/X1kShlAo4txvWrrUf/GUbfb338e5S6nbbHkUVNfy6XBATU9pjyFfH0pjsT8YYPvtsI48//hGHDmXjcAhbtiTTo0ebWotBE4VSqlYZr5FQbrftJZSYaD/09+2zyaJx4/KPa9QImjYt3RYZCQ6H/2MOlMTEwzzyyId8++02APr0aU9Cwli6dWtdq3FoolBK+dXBg6VVQFlZNiF4y88vbSA2Bnr2rP0Y66JXX/2WZ59dSUGBkyZNIrjttnP44x/7ExJS+3VfmiiUUjXOGNtGsGVLabfSYvn5NlkU1/UbY6t6YmMrHkzWEOXnF1FQ4GT06D7cccd5NG9eTjGrlmiiUErVqPx82LDBtjW4XLB/v+2F1NpTWxIRYRuGQ3Q1nCOkpeWwe3cqp55q57O76qozOO20ePr37xTgyDRRKKWOk9NpSwz79tmuocXTUKSm2qTRuDF06RK43kJ1ndtteO+9tTz99Kc4HCEsXTqNmJhIwsJC60SSAE0USjVYLpdtTDbGftBXNn21t6wsW63kLTvbtjNkZtr7TqetSoqP1wRRmW3bDjJ79gf88oudSHvw4C7k5xfV6PQbNUEThVL1VHHVD9i5h7ynp87Ott/2i5NDQUHFo43Lk5d3dKN048Z2NHJ4ePlTXKhSeXmFPPfcF7z++ve4XG5atGjM9OmjOO+8k5E6mFk1UShVD/36q/0gdzpLtx0+fGTX1MJCmyxCQ23CCA+HJk18O39MDHToUL+7pvrTPfe8zbffbkMEJk4cwC23jKRJk+Oc+MmPNFEoVU9kZtqSwZ49NkHs2WPvF08853ZDXFzp/iEhthSgjcq1b8qUM0hNzSYhYQy9e7cPdDhV0kShVBDJyYGMjNL7hw+XjmR2OkvXMkhPt9t1TELgOZ1u/vOfH9i/P50ZMy4AYMCAeF59dWpAxkQcC00USgUBp9P2Jtq82TY8e1chea91YIwdixAVBZ07ByZWVWr9+n08/PAHbN1qp5+96KLTOPHEVgBBkyRAE4VSdYoxttRw4EBpQ3NRkd1W3BidmmrbCIpnKY2JsTcRrUaqK7Ky8lm4cAXvvLMGY6BNmxhmzhxdkiSCjSYKpQIoNdUOSCv+gM/Ls6WDoiJbrVTMewK8qCg7zbWqmz7+eD3z5n1EamoODkcIf/rTUK6/fjiRkXVopsFq0kShVC3LyrK3/Z6Ff/fvP7KrqdNpu7VGRx85pUVUlJYYgsH3328nNTWHvn07kJAwhq5da3cCP3/QRKFULShuQ8jOhp07bUnC5bIN0y6X7WpaXJXkcOg4hGBSWOjk4MEs2re3Wf3Pfz6Xfv06MnbsqUHVDlEZTRRK1ZCCAltdlJNz5Gjk9HRblVTcAJ2WZvdr0QLatrVdVOvgGCvlg9WrdzJ79jJCQoQ337yJRo0cxMZGMW5cv0CHVqM0UShVTZmZsHfvkYPNcnNtMigstI97D3QDmzyMsaUGp9P2SIqou+OrVBVSU7N58slP+fDDXwGIj4/jwIHMklJFfaOJQqkyjLHLbiYllV8FVFRk11goO99RcRfWsDA7sM27PSEuThNDfeB2G9599ycWLFhBVlY+4eGhXHvtH5gy5QwaNaq/w9Q1USjl4XLZEsHGjXZqi7S0iuc/crnsvEbeS2aK2GkwtBqp/pox4z98+eUWAIYOPZF77hlN+/bNAxyV/2miUA3Wvn2lYxO82xGys21PpNBQaNeu/FKFw6E9kBqiESN6sGHDPmbMGMU55/SqkxP4+YMmCtVguN02CbjdsH17aRVSsYICexOxpYXY2MDFquqGL77YwsGDmUycOBCAMWP6cvbZPWncODzAkdUuTRSqwVi/3o5fKCqy9w8csL+3bWvvx8RoDyRlJSdn8OijH/LFF1sIC3MwdGhX2rdvhog0uCQBmihUPVa8HGfxB7/bbRup8/LsYLaoqNIkoRSA0+liyZIfWLRoFXl5RTRuHMbNN4+gTZuYQIcWUJooVL1S3PNo40b7+86dpYv3gG2D6N5d2xfU0X77LZGHH/6A338/AMA55/Ri+vTzadWqaYAjCzxNFCroZWXZrqxFRaXjGdxuux5DaCicdFLpvpogVEWeeWYlv/9+gHbtYrn77tEMG9Yt0CHVGZooVFDbs8dWJ6Wl2SThctmurWFhts2hTZtAR6jqKmMMOTmFREfbNoeZMy9g2bJfuO664URE6Bwq3jRRqKBjjK1W2r3bJoikJNubqVUrO45BJ89TVdm16xBz5y5HBBYuvBIRIT4+jltvHRno0OokTRSqznO77ZgHY2yJ4fBhuy03t3RQ3Ekn6frNqmoFBU5efvkrXn75G4qKXMTERJKUlE67dvVz6o2aoolC1Tkul+2ZlJJiE0J6uk0GxaOkvZf8jI6Gbt00Saiqff/9dubOXc7evXahj3HjTuX2288lNjYqwJHVfX5NFCIyCngKcADPG2PmlHm8I/BvINazz73GmOX+jEnVbQUFdrxDbq4tQaSl2e25ubY6qUULW73UurVOxa18Y4zhwQff4/331wHQpUtLEhLG0K9fpwBHFjz8lihExAEsBM4FEoHVIvKeMWaj1273AW8ZY54RkV7AciDeXzGpus3ptEkiPd1OoREZaZNDs2Y2QURGBjpCFYxEhLZtYwkPD+WGG85k8uSh9XoCP3/wZ4liELDNGLMDQESWAOMB70RhgOJOyjFAkh/jUXVITo5dvCcl5ejt6el2hHTHjoGJTQW/LVuSOXQoizPOsF1cr7rqDEaP7qNtEcfIn4miHbDX634iMLjMPg8An4jIbUBj4JzyTiQiU4GpAB310yMoJSWVrvKWlWV/z8iwbREZGUfuGx2tI6bVscnJKWDRolUsWfIDMTGRLF06jZiYSMLCQjVJHIdAN2ZfDrxsjHlcRIYCr4pIb2OM23snY8xiYDHAgAEDTADiVMcoPx+2bLE/Dxwo3V5YaLe1aAFduth2B6WOlTGGVas289hjH3HgQCYhIcKoUacQGqr9pGuCPxPFPqCD1/32nm3ergNGARhjvhORCCAOOIgKeocP2yk00tPt7wUFpSWFkBBbvaTjHdTx2r8/nUce+ZCvvtoKQK9ebZk1ayw9euhoy5riz0SxGugmIp2xCWIScEWZffYAI4GXRaQnEAGUqbVWwaJ44Z+0NPszJcVO452RYZNC5846M6uqWcYYZs58i02b9tO4cTi33jqCCRMG4HDoN5Ca5LdEYYxxisg04GNs19cXjTEbRORBYI0x5j3gLuA5EbkT27B9tTFGq5aCjMtlZ2rNz7f3s7NtoigqskkiPl57LKma5XYbQkIEEeHPfz6Pd95Zw113nU9cXJNAh1YvSbB9Lg8YMMCsWbMm0GEobDLIzoYdO2y1kveKcTExdr6lpk2PXC5UqeORnp7LggWfAXDffeMCHE3wSEqCceNitxqT3v1Yjg90Y7YKQk6nLUEUFtqBcIWFtqE6IgJ69NDqJVXzjDEsW/YLTz75KenpuTRq5OCGG86idWudArw2aKJQ1ZKWZksQWVm2/aGw0E7C1749NNFSv/KDnTtTmD17GT//vBuA006LJyFhjCaJWqSJQlXJGJsUEhNtaaK4F1NkpO3aqiUI5Q/GGJ59diX//vc3OJ1uYmOjuOOO8xgzpg+ib7papYlCVaqwEH77zf5MTbVJAmw315iGvTqk8jMR4eDBLJxONxdd1J9p084hJkZ7RQSCJgpVrsOHS9eXzsiwjWEOh+3iGhER6OhUfZWSkkV6ei7durUG4M9/Ppfx4/tx6qk6I0MgaaJQJbzXekhMtIkiP9+WJpo3tzO2KuUPLpebpUvX8K9/fU6rVk14442baNTIQWxslCaJOkAThSqxbp3txZSXZ9sl0tOhXTs7vYZ2cVX+snnzfh5++AM2brRzgrZv34mcnAJdJ6IO8SlRiEgY0NEYs83P8agAycqyDdV79tikEBVlk4T2ZFL+kp1dwLPPfs5bb63G7Ta0bt2UGTNGcdZZPbSxuo6pMlGIyBhgHhAGdBaRU4G/GWMu8ndwyv+cTlvdtHevneLbGNuTSSl/MsYwdepLbN16AIdDmDx5CFOnnkXjxjo7ZF3kS4niQez04CsBjDHrRKSrX6NStaJ4NbmcHHvLzLRTfCvlbyLCFVcM4e231zBr1li6dz8h0CGpSviSKIqMMellioLBNe+HOkpeHmzcaAfQFY+qbtlSu7wq/ygqcvH6698REiJMmXIGAGPG9OWCC/roBH5BwJdEsUlELgVCPDPB3g5879+wlL+43baa6dAhmyRSUyEuzt6U8oe1a3cze/YyduxIISzMwZgxfWnRIhoRweHQtohg4EuimAbcD7iB/2Jng53lz6CU/6xbZ6uZDh+2DditW9uur0rVtPT0XJ566lPef38dAB07Nueee8bQooXWbwYbXxLF+caYe4B7ijeIyMXYpKGCyN69tkRx4IAdH3HiidrtVdU8Ywzvv7+Op576lIyMPBo1cnD11cO4+uphhIdrj/xg5Mtf7T6OTgp/KWebqsNcLjtf0+7dNkn07BnoiFR99uGHv5GRkcfAgZ25557RxMdr3WYwqzBRiMj52GVK24nIPK+HmmKroVSQ8O7dlJdnpwJXqibl5xeRnZ1PXFwTRIR77hnNxo1JXHDBKTomoh6orERxEFgP5AMbvLZnAff6MyhVc7Zvtw3Wubl2Wo7mzXW2V1Wzvvnmdx55ZDnt2jVj4cIrERHi4+O0FFGPVJgojDFrgbUi8roxJr8WY1I1ZMcO27tp3z6bKFq21N5NquYcPJjJ449/zIoVGwGIigonIyNPp96oh3xpo2gnIv8EegEl84YaY07yW1TquOXk2CSRnGyTRPfuEKLd1VUNcLncvP32ap555nNycgqJjGzEjTeezaRJgwkN1TdZfeRLongZeAh4DLgAuAYdcFenFRTA5s02WWRlQbdumiRUzXC7DVOnvswvv+wF4KyzejBjxihOOEFHatZnviSKKGPMxyLymDFmO3CfiKwB/urn2NQxyMyErVvtGhL799vqplDtkahqSEiIMGTIiSQnZzBz5mjOPLN7oENStcCXj5ACEQkBtovITcA+QOcUrYNycmxJIi0NUlKgVSto0SLQUalgZozh00834HCEMHJkLwCuuuoMJk8eSlSUDsJpKHxJFHcCjbFTd/wTiAGu9WdQ6tgUVzelpEDHjtC4caAjUsEsMfEwc+Ys5/vvt9OsWRQDB3amadNIwsJCdaBmA1NlojDG/OD5NQu4EkBE2vkzKFU9u3bZLrAul23AjorSJKGOXWGhk1de+ZaXXvqKggInTZtGcMstI4iO1jVwG6pKE4WIDATaAV8bYw6JyMnYqTxGAO1rIT5VhYICmyR27bIjro3RUdfq2K1Zs4s5c5axa9chAEaP7sMdd5xH8+b6zaMhq2xk9mxgAvALtgH7A+AWYC5wU+2EpyqTkmITRFaWTRhduujcTerYuVxu5s61SaJTpxYkJIxhwIDOgQ5L1QGVlSjGA32NMXki0hzYC5xijNlRO6Gp8uTn23aIXbugqMjOAnv4sO3dpElCVZfbbSgsdBIR0QiHI4R77x3D2rW7mTLlDMLCtLucsip7J+QbY/IAjDGHRWSrJonAcbvh998hOxvS0+22lBS7lGnnznbhIaWqY9u2Azz88DLi41tw//3jATjttHhOOy0+sIGpOqeyRNFFRIpniBXsetklM8YaYy72a2SqxN69pVODHzhgJ/aLiYFmzXRKDlV9eXmFPPfcF7z++ve4XG6SktLIzMyjadPIQIem6qjKEsWEMvcX+DMQdaTERNvukJ5uSw3JybYtIiJCp+NQx+7LL7fwyCMfkpycgQhMnDiAW24ZSZMmWiRVFatsUsAVtRmIKnX4sJ3ILznZ3s/NtV1fu3XTUdbq2DidbhISlrJy5SYATjrpBGbNGkvv3trTXVVNP3bqkNxcOy14YaEtSWRlQfv2tnpJ2yDU8QgNDSE6OpyoqDBuuulsLr10kE7gp3zm10QhIqOApwAH8LwxZk45+1wKPICdaPAXY8wV/oypLioqsg3VeXl2+o3Dh+22tm2hiU6Woo7R+vWJAPTubYc8/fnP53LTTWfTunXTQIalgpDPiUJEwo0xBdXY3wEsBM4FEoHVIvKeMWaj1z7dgATgDGNMmoi08j30+uPXX+1kfqmptsG6ZUvbUO1wBDoyFYyysvJZsOAz/vtHkp6hAAAgAElEQVTfn4iPj+ONN26iUSOHrhOhjlmVZU8RGSQivwG/e+73FZH5Ppx7ELDNGLPDGFMILMGOzfB2A7DQGJMGYIw5WK3o64HkZNv+cOCAbaDu3t1WNWmSUNVljOGjj37jkksW8M47PxESEsLw4d1xuXTlYnV8fClRPA2MBf4PwBjzi4ic7cNx7bCD9IolAoPL7HMSgIh8g62eesAY85EP5w56LhesW2d7NB06ZH926xboqFSw2rMnlTlzlvPjj3aoU9++HUhIGEvXrg2ykK5qmC+JIsQYs7vMAumuGrx+N+As7NxRX4rIKcaYdO+dRGQqMBWgY8eONXTpwNq0yQ6e27/fJokuXQIdkQpWTqeLm29+hQMHMomJieS2285h3Lh+hITo4uiqZviSKPaKyCDAeNodbgO2+nDcPqCD1/32nm3eEoEfjDFFwE4R2YpNHKu9dzLGLAYWAwwYMCDoV9dzu0vHSLhcOomfOjbGGESE0FAHN988gp9+2sXtt59Ds2Y6gZ+qWb70j7sZmA50BA4AQzzbqrIa6CYinUUkDJgEvFdmn//DliYQkThsVVS9nSbE7bZdXteuLV2mVEsSqrpSU7P561/f5YUXvizZNnZsX/72t/GaJJRf+FKicBpjJlX3xMYYp4hMAz7Gtj+8aIzZICIPAmuMMe95HjtPRDZiq7PuNsakVvdaweDwYdi50/ZqKiy0A+qaNdOJ/JTv3G7Du+/+xIIFK8jKyqdJkwguv3wIjRuHBzo0Vc/5kihWi8gW4D/Af40xWb6e3BizHFheZtv9Xr8bbGlluq/nDCbFCwkVFdleTfv327WsHQ7bBVbnaVK+2ro1mdmzl/Hbb3ZsxOmnd2XmzNGaJFSt8GWFuxNF5HRs1dHfRWQdsMQYs8Tv0QWxXbtsksjLsyOunU7bJqEzvarqcDpdLFiwgjff/B6XyxAXF82MGaMYObIXZTqYKOU3Po3hN8Z8a4y5HegPZAKv+zWqIJeRYacA373b3rKy7BgJTRKquhyOELZsScbtNlx22SCWLr2Vc845WZOEqlVVlihEJBo7UG4S0BP4H3C6n+MKatu32zaJvDyd6VVVX3JyBi6Xm3btmiEiJCSMITu7gF692gY6NNVA+dJGsR54H3jEGPOVn+MJetnZdt3q7GzbBqFJQvnK6XTx5ps/sGjRKvr0ac/ChVciInTs2CLQoakGzpdE0cUYo3MAVCItzS4uVFRk7xeXJurJ2EBVC379dS+zZy/j998PANC0aST5+UVERmq3OBV4FSYKEXncGHMX8I6IHDXITVe4sw4dst1eDx2ypYgCz7SJbdpoaUJVLTMzj/nzP+Pdd38GoF27WGbOHM0ZZ+h8LqruqKxE8R/PT13ZrgL5+TZJHDxoSxVt2tglSrWdUfmisNDJFVcsIjk5g9DQEK688nSuu244ERGNAh2aUkeobIW7Hz2/9jTGHJEsPAPpGvwKeBs32lJEWhrEx0OkLjmsqiEsLJTx4/uxevVO7r13DF26tAx0SEqVy5fKkWvL2XZdTQcSTIyBPXtsm8Thw3ZxIU0SqioFBU4WLVrJRx/9VrLtmmv+wKJFV2mSUHVaZW0Ul2G7xHYWkf96PdQESC//qIYhMdHeUlNto3WPHoGOSNV133+/nblzl7N372GaN2/MWWf1ICKikS5HqoJCZW0UPwKp2FlfF3ptzwLW+jOouu7gQVuSyMiwk/ppm4SqyKFD2TzxxMd8/PF6ALp0acmsWWO1HUIFlcraKHYCO4HPai+cum//fjupX3o6nHiiTuqnyudyufnvf39i4cIVZGcXEB4eyg03nMnkyUNp1EiXL1TBpbKqpy+MMWeKSBrg3T1WsPP5Nfd7dHWMywVJSbY0AZokVMXcbsN//vMj2dkFnHFGN2bOvIB27ZoFOiyljkllVU/Fy53qHKceO3fadSQyMnQwnTpaTk4BbrehSZMIGjVycN99F5Kams2IET11biYV1CpsSfMajd0BcBhjXMBQ4Eagwa2OkpVV2i4B0LjBvQKqIsYYPv98ExMnLuSJJz4u2X7qqR11lldVL/gyhcf/AQNF5ETgJeAD4A1grD8Dq2u2boXMzNIGbKUAkpLSefTRD/nqK7s68PbtKRQUOAkP9+VfS6ng4Mu72W2MKRKRi4H5xpinRaRB9nrKyLBjJsJ1rZgGz+l08dpr3/Hcc19QUOCkceNwbr11BBMmDMDh0C6vqn7xaSlUEZkIXAn80bOtQfXtO3DANmTn50M3nYKnwcvPL+Lqq59n27aDAJx3Xm+mTz+PuLgmAY5MKf/wJVFcC9yCnWZ8h4h0Bt70b1h1S2KiXYjIGLuMqWrYIiIa0bNnW/Lzi7j33jEMGXJioENSyq98WQp1vYjcDnQVkR7ANmPMP/0fWt1QvIxpWhp06hToaFQgGGNYtuwX2rdvzqmn2u5ud911PqGhDh04pxoEX1a4+wPwKrAPO4biBBG50hjzjb+Dqws2bbIT/wFERQU2FlX7du5MYfbsZfz88246d47jjTduolEjB9HRuqatajh8qXp6AhhtjNkIICI9sYljgD8Dqwuysmx106FDdvpw1XDk5xfx4otf8cor3+B0umnWLIqrrx6mczOpBsmXRBFWnCQAjDGbRKRBjEneutW2TRQVQVtdrrjB+Pbbbcydu5x9+9IAuOii/kybdg4xMTpFsGqYfEkUP4vIs8BrnvuTaQCTAqak2LaJw4e1baIhyc0t5P773yU9PZcTT2xFQsKYknYJpRoqXxLFTcDtwEzP/a+A+X6LqI7Ys8dO/GeMtk3Udy6XG2MMoaEOoqLCmDFjFAcOZDJ58hBCQ7Wbm1KVJgoROQU4EXjXGPNI7YQUeMWT/mVmQjOdx61e27QpiYcf/oAzz+zO9defCcCoUacEOCql6pbKZo+dhV3J7mfsFB4PGmNerLXIAuTAAdi1yyaLggKdrqO+ys4u4NlnP+ett1bjdhtycgo8jdVaglCqrMpKFJOBPsaYHBFpCSwH6nWiMKZ05Tptm6ifjDGsWLGRxx77iEOHsnE4hMmTh3DjjWdrklCqApUligJjTA6AMSZFROp9v8D16+00HZmZ0K6dtk3UNzk5BSQkLOXbb7cB0Lt3OxISxtK9+wkBjkypuq2yRNHFa61sAU70XjvbGHOxXyMLgMJCSE62vZ2aNg10NKqmRUWFUVTkIjo6nGnTzuHii08jJESnAFeqKpUligll7i/wZyCBduiQ/VlYqBP/1Sc//7ybuLhoOnZsgYhw//3jCAsLpUWL6ECHplTQqGzN7BW1GUig7d5tSxMuF4TqUgJBLz09l6ee+pT331/HoEGdWbjwSkSENm1iAx2aUkFHPxKxI6/dbts+od1hg5vbbXj//XU8/fSnZGTk0aiRg1NP7YjLZQgN1WompY6FXxOFiIwCngIcwPPGmDkV7DcBWAoMNMas8WdM5dm+3XaFzcvTnk7BbPv2g8yZs4y1a/cAMGhQZ+65ZwydOrUIcGRKBTefE4WIhBtjCqqxvwNYCJwLJAKrReQ973mjPPs1Af4M/ODruWvS4cN28r+UlOJ4AhGFOl7Z2flcc80L5OYW0rx5Y+688zxGjTpF16tWqgb4Ms34IOAFIAboKCJ9geuNMbdVcegg7NoVOzznWQKMBzaW2e8fwFzg7mrGftwKCmDnTjh4EHJy4KSTajsCdbyMMYgI0dERTJlyBikpmdx660iaNtUJ/JSqKb6MjXgaGAukAhhjfgHO9uG4dsBer/uJnm0lRKQ/0MEYs6yyE4nIVBFZIyJrUoq/+teA9evtfE5padC5s65eF0wOHsxk5sy3WL7815Jt1133BxISxmqSUKqG+VL1FGKM2V2mCO863gt7BvDNA66ual9jzGJgMcCAAQPM8V4bbOM12FHYzZpBhK5DExScTjdvv/0jzzyzktzcQrZs2c+oUafgcIRoNZNSfuJLotjrqX4ynnaH24CtPhy3D+jgdb+9Z1uxJkBvYJXnH/wE4D0RGVcbDdqbNtnqpsJCnc8pWGzYsI/Zs5exefN+AM46qwczZozC4aj3kwYoFVC+JIqbsdVPHYEDwGeebVVZDXQTkc7YBDEJuKL4QWNMBhBXfF9EVgEzaqvXU36+rXZyOLQBu67Lyyvk6ac/Y+nS1RgDJ5wQw913X8CZZ3YPdGhKNQhVJgpjzEHsh3y1GGOcIjIN+BjbPfZFY8wGEXkQWGOMea/a0daQn3+203Tk5urKdcHA4Qjhxx93EBIiXHHFUKZOPZPIyAaxyKJSdYIvvZ6eA45qFzDGTK3qWGPMcuyss97b7q9g37OqOl9NyMiwo6937LCzxUbrTA51UmLiYaKjI4iNjSIsLJQHH7yI8PBQunZtHejQlGpwfKl6+szr9wjgIo7szRRUtm2zYydcLujZM9DRqLIKC5288sq3vPTSV4wadQp//es4AE4+uV0VRyql/MWXqqf/eN8XkVeBr/0WkR8ZT7koOxtatgxsLOpoa9bsYs6cZezaZWdodLncuFxubaxWKsCOZQqPzkBQlv937rTTdOTlQceOgY5GFTt8OIennvqUZct+AaBTpxYkJIxhwIDOAY5MKQW+tVGkUdpGEQIcBu71Z1D+4HbbgXXp6fZ+iH5JrRPS03OZOHEhGRl5hIU5uPbaPzBlyhmEhel8lUrVFZX+N4od4NCX0vEPbmNMjQx4q2379tlxE5mZWpqoS2Jjoxg+vDsHD2Zy771j6NCheaBDUkqVUWmiMMYYEVlujOldWwH5S36+ndvJ7YbGjQMdTcOVl1fIc899wbBhJ9G/v52q9957xxAW5tCR1UrVUb5UwKwTkX5+j8TPMjPtKOxInQYoYL78cgsTJ/6LV175ljlzluF228JpeHioJgml6rAKSxQiEmqMcQL9sFOEbwdysOtnG2NM/1qK8bgVFdnusGlput5EICQnZ/D44x+xcuVmALp3P4FZs8bqetVKBYnKqp5+BPoD42opFr/JzLTJAiAqKrCxNCROp5slS35g0aKV5OUVERUVxs03n83EiYMIDdXeBEoFi8oShQAYY7bXUix+k5xs2ydU7crJKeDll78mL6+IESN6ctddo2jdummgw1JKVVNliaKliEyv6EFjzDw/xOMX+fm2x1Oo9rj0u6ysfMLDQwkLCyUmJpJZs8YSFuZg2DBdFUqpYFVZ+d8BRGOnAy/vFhSK153IzYXWQTlMMDgYY/joo9+YMGEBr7zyTcn2ESN6apJQKshV9h17vzHmwVqLxE/WrrWliaIiaKq1Hn6xe3cqc+cu48cfdwLw8897SpYoVUoFvyrbKIKZMfZ24ICuYOcPBQVO/v3vr3nppa8pKnIRExPJ7befy4UXnqpJQql6pLJEMbLWovCTpCRbkigo0Jlia9qhQ9lMnfoSe/YcBuDCC0/lz38+l9hY7VamVH1TYaIwxhyuzUD8ITnZjp1QNa9Fi8a0bh2DwxHCvfeO4bTT4gMdklLKT+ptP6CMDPszJwfi4irfV1XN7Ta8++5PDBjQmU6dWiAiPPTQxTRtGkmjRo5Ah6eU8qN6myi2bbNtEwUF0KVLoKMJblu3JjN79jJ++y2RQYM6s3DhlYgILVro8oBKNQT1NlEAZGVB+/aBjiJ45eYWsnjxKt5883tcLkPLlk24+OIBgQ5LKVXL6mWi8B6FHR4euDiC2apVm3n00Q85cCCTkBDhsssGcfPNI4iO1hdUqYamXiaK9evtAkVOp47GPhYHD2Yya9ZSCgtd9OzZhoSEsfTq1TbQYSmlAqTefoxmZtqxE7qSnW+cThcORwgiQqtWTbnllhGEhjqYOHGgrlmtVANX7z4B8vPtz6IiaNUqsLEEi19+2cuf/rSY5ct/Ldn2pz+dzqRJgzVJKKXqX4kiKclWO+kiRVXLyMhjwYLPePfdnwFYunQNo0f30VHVSqkj1LtEAbY0EREB+nlXPmMMy5f/ypNPfkJaWi6hoSFMmXIG1177B00SSqmj1LtEkZZmV7PT3k7lS03N5i9/eYc1a3YB0L9/JxISxtC5c8vABqaUqrPqVaIoKLBJIjtbR2NXpEmTCA4dyiY2Noo77jiXMWP6ailCKVWpepUo0tJs20RREcTGBjqauuP777fTo0cbYmOjCAsLZc6cicTFResEfkopn9SrLi379tkFipR16FAWs2a9w7RprzF//mcl27t2baVJQinls3pVogBb7RQTE+goAsvlcvPf//7EggUryMkpIDw8lE6dWuhiQkqpY1LvEoXLBS1aBDqKwNm8eT8PP/wBGzcmATBsWDdmzhxN27ZaF6eUOjb1JlHk59vV7Iq7xjZESUnpXHXVc7hchlatmjBjxgWcfXYPLUUopY6LXxOFiIwCngIcwPPGmDllHp8OXA84gRTgWmPM7mO51q5ddg0Kt7vhzu/Utm0sF154KlFR4dx441k0bqx9hJVSx89vjdki4gAWAhcAvYDLRaRXmd3WAgOMMX2ApcAjx3q9nBxbqmhISSIpKZ0773yTn37aVbLtL3+5kOnTz9ckoZSqMf78WB0EbDPG7AAQkSXAeGBj8Q7GmJVe+38P/Ol4LlhQAE2bHs8ZgoPT6eK1177juee+oKDASXp6Li+9dB2AVjMppWqcPxNFO2Cv1/1EYHAl+18HfFjeAyIyFZgK0LFjxwpPUFgIrVtXO86gsm7dHh5++AN27EgB4LzzejN9+nkBjkopVZ/ViYoaEfkTMAA4s7zHjTGLgcUAAwYMMGUfP3zYrj3hdNbfhuzMzDyeeupT/ve/tQC0b9+Me+8dw5AhJwY4MqVUfefPRLEP6OB1v71n2xFE5BzgL8CZxpiCso/7YudOu+xpfeZ2G774YguhoSFcffUwrr56GBERjQIdllKqAfBnolgNdBORztgEMQm4wnsHEekHLAJGGWMOHstFkpNtSSItDZo0Od6Q65Zduw7Rtm0sYWGhxMZG8dBDF3PCCTHEx+tEVkqp2uO3Xk/GGCcwDfgY2AS8ZYzZICIPisg4z26PAtHA2yKyTkTeq+519u2zVU8FBdCuXY2FH1D5+UUsXLiCSZOe4ZVXvinZPmTIiZoklFK1zq9tFMaY5cDyMtvu9/r9nJq4Tn4+NGtWP9af+Pbbbcydu4x9+9IBSE/XyauUUoFVJxqzj5fbDVFBPsddSkoWjz/+EZ99ZnsPd+3aioSEsfTt26GKI5VSyr+COlFkZNj2iYKC4O7ttHt3KlOmLCYnp5CIiEZMnXomV1wxhNBQR6BDU0qp4E4USUl2tli3G8LCAh3NsevYsTm9erUjIqIRM2deQJs2OoGfUqruCOpEkZtrSxONgqyXaHZ2AYsWreSSSwbSqVMLRIR58yYRGRnE2U4pVW8FdaIAO34iWKYVN8awYsVGHnvsIw4dymbXrkPMn29nLdEkoZSqq4I2Ubhc9hYsy54mJqbxyCPL+fbbbQCcckp7brutRjp9KaWUXwVtonC77foTULe7xRYVuXj11W954YUvKShw0qRJBNOmjeSii04jJKQOB66UUh5BmyiKikoTRV124EAGzz//BYWFLi644BTuuOM8WrSIDnRYSinls6BNFHl5tmtsXZSZmUeTJhGICO3bN+euu0bRoUNzBg3qEujQlFKq2vw2hYe/paXVvUThdhv+97+1XHTRfJYv/7Vk+4QJAzRJKKWCVtCWKIqKbPfYumL79oPMmbOMtWv3AHYqjjFj+gY4KqWUOn5Bmyhyc20bRbNmgY0jP7+I55//gldf/Q6Xy03z5o2ZPv18zj+/d2ADU0qpGhK0iQLsinaBHJG9e3cqt932GklJ6YjAhAmnceutI2naNDJwQSmlVA0L6kRRVARxAZx1u02bGMLCQjnppNYkJIzllFPaBy4YVQcVERKSiEh+ne7CreoPY8CYCNzu9kDNTVkRlImisLB0sF1tliicTjfvvLOG88/vTWxsFGFhocyfP5mWLZsSGhq0/QKUn4SEJNKqVRNiYuIRzRSqFhhjyMhI5eDBRNzuzjV23qD8dMvOLh1HEVpLqW79+n1cddVzPProh8yf/1nJ9jZtYjVJqHKJ5BMT00KThKo1IuJ5z+XX6HmDskSRmmoXK6oN2dn5LFz4OUuXrsYYOOGEGM48s3vtXFwFNRE0SahaJyI1XtUZlImioMCOoXD4cbkGYwyffLKBefM+JjU1G4cjhMmTh3DDDWfqBH6q2tau9c95+/Xzz3mV8haUdSZFRTZR+LN9YuvWA/zlL++QmppNnz4deP31qdx++7maJNQx27KlZm++aNrUwdChpzJwYG8mTryQ9PT0ksc2btzA6NEj6NevO337dmPOnH9gvObF+eSTD/nDHwZw2mm9OP30fiQk3FXTL8lx++WXtdxyy3WBDqNSjz02mz59utKvX3c+++zjcvdZuXIFZ5zRn6FDT+Xcc4exfbudPLSgoIApUy6jT5+unHXWYHbv3gXA+vW/ceONV9fSMwjSEkXxhIA13T7hcrlxOGzu7N79BK64YghdurRk3Lh+OoGfqhHda6jW0tdEERkZyXffrQNg6tSrWLx4ITNn/oW8vDwuu2wcTz75DCNHnkdubi6TJ09g8eJ/ceONt7Jhw3ruumsaS5cuo3v3HrhcLl58cXHNBO/hdDoJPc5/4kcffZh77rmvVq9ZHZs2bWTp0iWsXr2B/fuTuPDCc1i3biuOMtUhd955M0uW/I8ePXqyePG/eOSRh1i06GX+/e8XiI1txq+/buPtt5fw17/ewyuv/IfevU9h375E9u7dQ4cOHf3+PIKyRBEaaqufImtwuMKaNTu57LJn+Pnn3SXbpk8/nz/+sb8mCVUvDBo0lKSkfQC89dYbDBlyBiNHngdAVFQUjz++gHnz5gDw5JOPcPfdf6F79x4AOBwObrjh5qPOmZ2dzU03XcOgQacweHAf/u//3gGgdevSiS/ffXdpybffG2+8mttvv4mzzhrMfffNpFev+CNKOX37duPAgQOkpKRwxRUTGD58IMOHD+S777456tpZWVls2PArp5xiZ0BYs+ZHRowYyumn92PkyNPZutVm09dee5lLLx3H6NEjGDNmpOf5Pcrw4QMZPLgPDz30t5JzTpr0R4YNO40BA06ukcS4bNn/uOSSSYSHhxMf35kuXbqyZs2PR+0nImRlZQKQmZlBmzZtS46fPPkqAC666BJWrVpRUuq74IILWbp0yXHH6IugLFE4nbZEURPrZB8+nMNTT33KsmW/APD669/Rv3+n4z+xUnWIy+Vi1aoVXHWVrabZtGkDp5562hH7dOlyIjk52WRmZrJx43puv73qqqa5c/9B06Yx/PjjbwCkpaVVeUxSUiIrVnyLw+HA5XLx/vvvcuWV17B69Q906NCJ1q1bc801VzBt2p2cfvow9u7dw/jx5/Pzz5uOOM/PP6+hZ8/SGRBOOqkHn3zyFaGhoaxc+RkPPDCLN96wiWvdup/5/vtfad68OStWfMK2bb/zxRc/Yozh0kvH8fXXXzJs2HD+9a8Xad68OXl5eQwfPpDx4yfQoszKaPfccydffrnyqOd1ySWTuOuue8s8130MGjSk5H67du1LkrW3BQueZ8KE0URERNKkSVNWrvy+5Pj27TsAEBoaSkxMDKmpqcTFxdG//wDmzZvDnXfOrPI1P15BmSjAJorjadm3E/j9zPz5n5GZmU9YmINrrx3OlCmn11yQSgVYXl4eQ4eeSlLSPrp378mIEefW6PlXrvyMl18u/VbbzIc5dS66aGJJ1cuECZcxZ86DXHnlNSxduoQJEy4rOe/mzRtLjsnKyiQ7O5vo6NKSyoED+4mLa1lyPzMzg6lTr2L79t8REYqKikoeGzHiXJo3bw7AihWf8Pnnn3D66bYnQE5ONtu3/86wYcN55pmnef/9dwHYt28v27f/flSimDv3Cd9enGpYsOAJ3nlnOQMHDubJJx8lIWE6Cxc+X+kxLVu2Yv/+pBqPpTxBmSiMOb7pO/btS+Ovf32XX3/dC8CQISdyzz2j6dCheQ1GqVTgFbdR5ObmMn78+SxatJBbbrmdHj168c03Xx6x786dO2jcOJqmTZvSs+fJrF37U0m1TnV5dwsuKDiyL3tUVOOS3wcPHsqOHdtISUnhgw/+j5kzbXuD2+1m5crviaik2iAiIvKIc//jH39l+PCzWbLkXXbv3sUFF5xV7jWNMdx1VwLXXXfjEef78stVrFr1GZ9//h1RUVGMGnUW+eX0w69OiaJt23YkJu4tub9vXyJt27Y7Yp+UlBTWr/+FgQMHAzZ5/vGPo444vl279jidTjIyMkoSV35+PpE1Wf9eiaBrozDm+BuzGzcOZ8+eVFq0iObhhycwf/5kTRKqVtRmjydvUVFRPPbY08yf/zhOp5PLLpvMd999zcqVdvBoXl4ed999O3fcYasx7rjjbh577GF+/30rYD+4n3/+2aPOO2LEuSxevLDkfnHVU6tWrdm8eRNut5v33nu3wrhEhAsvvIiEhOl0796z5ENw5MjzePbZ+SX7/frruqOO7d69Jzt2bCu5n5GRUfIh/NprL1d4zXPOOZ9XX32R7OxswFbvHDx4kMzMDGJjmxEVFcWWLZtZvfr7co+fO/cJvvtu3VG3skkCYPTocSxduoSCggJ27drJ9u2/M2DAoCP2adasGRkZGSWv9eeff0r37j1Ljn/99X8Dtq3nzDNHlCThbdu2HlH15k9BWaJwu6t/zHffbeO00+IJCwslNjaKefMm0aVLS6Kja6ChQykf1FSPp2PVt28/evfuw9tvv8nll1/JkiX/Y8aM25g+/VZcLheTJl3JTTdNA6B37z7Mnfsk11xzObm5uYgIF1ww9qhzzpx5H9On38rAgb1xOBwkJPyN8eMv5u9/n8PEiWOJi2tJv34DyMnJrjCuCRMuY/jwgSxa9HLJtkcffZrp029l8OA+OJ1OzjhjOE8/fWSi6t69BxkZGWRlZdGkSRPuvHMmN954FY888hDnn8JpcfUAAArMSURBVD+mwuuNHHkemzdvYsSIoQBER0fz/POvce65o3jhhWfp378nJ53UnYEDh1R4Dl/16nUyF198KQMG9CI0NJR58xaWVLtdfPFoFi58njZt2rJgwXNMnjyBkJAQYmOb8cwzLwJw1VXXcf31V9KnT1eaNWt+RDXfl1+uZNSoip9nTRITDOuJeunff4CZP38NO3dCz55V75+cnMFjj33EqlWbuemms7n++uH+D1IpIDR0E127+vAmVcdswYIniI5uwtVXXx/oUGpVQUEBo0adyaeffl1ud99t2zbhdJa+95KSYNy42K3GpB/T15Wgq3pyu31b2c7pdPPaa98xceJCVq3aTFRUGDExOv23UvXJ9dffTHh4eKDDqHV79+7h73+fU2tjQoKu6snttjPHVtbj6bffEpk9+wO2bj0AwIgRPZkxYxStWjWtpSiVUrUhIiKCyy+/MtBh1LquXbvRtWu3Wrte0CUKl8uWKEIqKAutX5/Itde+gDHQtm0sM2dewLBhJ9VukEpRvDaA0YkBVa0yxlDTLQpBlyiMscugNm5c/uMnn9yOoUO70r37CVx33XAiImpu8Q6lqsOYCDIyUnWqcVVritejMKZmO+kEZaLIzoZunlLXnj2pzJv3MXfeeT6dOtl/yCefvEKn3VAB53a35+DBRFJSUnSFO1UrjlzhruYEZaIAcLudLF78NS+//DWFhS7CwkJ55JFLATRJqDqiUY2uMqZUoPi115OIjBKRLSKyTUSOGo0iIuEi8h/P4z+ISHxV5zQGNmzYwaRJz7J48RcUFrq48MJTmTXr6D7eSimljp/fShQi4gAWAucCicBqEXnPGLPRa7frgDRjTFcRmQTMBS6r7Lz796czZ86rAHTuHEdCwlidxE8ppfzIn1VPg4BtxpgdACKyBBgPeCeK8cADnt+XAgtEREwlowCzs/No1CiUyy4bzrhxp9OokYO9eyvaWyml1PEuHe23kdkicgkwyhhzvef+lcBgY8w0r33We/ZJ9Nzf7tnnUJlzTQWmeu71gaY7/BJ00CmIhfD0qvdrCPS1KKWvRSl9LUpldTTGFXUsRwZFY7YxZjGwGEBE1hiTPiDAIdUJ9rXI09cCfS286WtRSl+LUiKy5liP9Wdj9j6gg9f99p5t5e4jIqFADJDqx5iUUkpVkz8TxWqgm4h0FpEwYBLwXpl93gOu8vx+CfB5Ze0TSimlap/fqp6MMU4RmQZ8DDiAF40xG0TkQWCNMeY94AXgVRHZBhzGJpOq1OwK78FNX4tS+lqU0teilL4WpY75tQi6acaVUkrVrqCbZlwppVTt0kShlFKqUnU2Ufhj+o9g5cNrMV1ENsr/t3euMVZVZxh+Xq1UrIZWiY2X1tF4RQWk1GBNqhRrlKYYlYAEtDT2Ir0YrfZHg0ab9kcTL4lI7aitARLRFitKEFuNGcWSGZVWGaz1LjGkpvCDEKOoLb79sdbA6fTMOXtG2GfPme9JdrL32uvynS/77G+vtfZ+l9Qr6QlJbfupejNf1OS7WJIlte2rkUV8IWlWvjb+Lml52TaWRYH/yBcldUl6Pv9PprfCzr2NpHskbcnfqNU7L0mLsp96JU0qVHHSLq/WRpr8fgM4BhgFbADG9cvzA6Az718C/L7VdrfQF1OBA/L+gpHsi5zvIGAt0ANMbrXdLbwujgOeBz6Xjw9ttd0t9MVdwIK8Pw7Y1Gq795IvvgpMAl4c4Px04FFAwBTgmSL1VrVHsUv+w/ZHQJ/8Ry0XAEvz/gPANLWn6H9TX9jusv1+PuwhfbPSjhS5LgB+QdIN+4TCBZWmiC++C/za9jYA21tKtrEsivjCQN8Sl2OAf5ZoX2nYXkt6g3QgLgCWOdEDfFbSYc3qrWqgOAKoVXDanNPq5rH9H2A7cEgp1pVLEV/UcjnpiaEdaeqL3JX+gu1HyjSsBRS5Lo4Hjpe0TlKPpPNKs65civjiRmCepM3AGuDH5ZhWOQZ7PwGGiYRHUAxJ84DJwFmttqUVSNoHuBWY32JTqsKnSMNPZ5N6mWslnWp7JGofzQGW2L5F0hmk77dOsf1xqw0bDlS1RxHyH7sp4gsknQMsBGbY/rAk28qmmS8OAk4BnpS0iTQGu6pNJ7SLXBebgVW2/237LeBVUuBoN4r44nLgDwC2u4H9gbGlWFctCt1P+lPVQBHyH7tp6gtJpwF3koJEu45DQxNf2N5ue6ztDtsdpPmaGbaHLIZWYYr8Rx4i9SaQNJY0FNWOystFfPE2MA1A0kmkQLG1VCurwSrgsvz20xRgu+13mhWq5NCT9578x7CjoC9uAg4EVuT5/Ldtz2iZ0XuJgr4YERT0xZ+BcyW9BOwEfmq77XrdBX1xDXC3pKtJE9vz2/HBUtJ9pIeDsXk+5gZgPwDbnaT5menA68D7wLcL1duGvgqCIAj2IFUdegqCIAgqQgSKIAiCoCERKIIgCIKGRKAIgiAIGhKBIgiCIGhIBIqgckjaKemFmq2jQd6OgZQyB9nmk1l9dEOWvDhhCHVcIemyvD9f0uE1534radwetvM5SRMLlLlK0gGftO1g5BKBIqgiO2xPrNk2ldTuXNsTSGKTNw22sO1O28vy4Xzg8Jpz37H90h6xcredd1DMzquACBTBkIlAEQwLcs/haUl/y9tX6uQ5WdKzuRfSK+m4nD6vJv1OSfs2aW4tcGwuOy2vYbAxa/1/Oqf/SrvXALk5p90o6VpJM0maW/fmNkfnnsDk3OvYdXPPPY/FQ7SzmxpBN0m/kbReae2Jn+e0K0kBq0tSV047V1J39uMKSQc2aScY4USgCKrI6Jphp5U5bQvwdduTgNnAojrlrgBusz2RdKPenOUaZgNn5vSdwNwm7X8T2Chpf2AJMNv2qSQlgwWSDgEuBE62PR74ZW1h2w8A60lP/hNt76g5/cdcto/ZwP1DtPM8kkxHHwttTwbGA2dJGm97EUlSe6rtqVnK4zrgnOzL9cBPmrQTjHAqKeERjHh25JtlLfsBi/OY/E6SblF/uoGFko4EHrT9mqRpwJeA57K8yWhS0KnHvZJ2AJtIMtQnAG/ZfjWfXwr8EFhMWuvid5JWA6uL/jDbWyW9mXV2XgNOBNblegdj5yiSbEutn2ZJ+h7pf30YaYGe3n5lp+T0dbmdUSS/BcGARKAIhgtXA/8CJpB6wv+3KJHt5ZKeAb4BrJH0fdJKXktt/6xAG3NrBQQlHVwvU9YWOp0kMjcT+BHwtUH8lvuBWcDLwErbVrprF7YT+CtpfuJ24CJJRwPXAl+2vU3SEpLwXX8EPG57ziDsDUY4MfQUDBfGAO/k9QMuJYm//Q+SjgHezMMtD5OGYJ4AZko6NOc5WMXXFH8F6JB0bD6+FHgqj+mPsb2GFMAm1Cn7Lkn2vB4rSSuNzSEFDQZrZxa0ux6YIulE0upt7wHbJX0eOH8AW3qAM/t+k6TPSKrXOwuCXUSgCIYLdwDfkrSBNFzzXp08s4AXJb1AWpdiWX7T6DrgMUm9wOOkYZmm2P6ApK65QtJG4GOgk3TTXZ3r+wv1x/iXAJ19k9n96t0G/AM4yvazOW3Qdua5j1tIqrAbSOtjvwwsJw1n9XEX8CdJXba3kt7Iui+3003yZxAMSKjHBkEQBA2JHkUQBEHQkAgUQRAEQUMiUARBEAQNiUARBEEQNCQCRRAEQdCQCBRBEARBQyJQBEEQBA35L4AYQzVVhFJ9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, _ = roc_curve(y_test, pred)\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.fill_between(fpr, tpr, color='b', alpha = 0.2,\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc_score(y_test, pred))\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic - EoL Model')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VfWd//HXhyQkhH1VWRQUFJFFEBHcqlipVsVOq1LbutVqa8dfO3aqXZ2q3ZzOtPNrq/05ztiqVSsuo2LVOl1QqhY1VERZFFCEALLJFiFk+/z++Jyb3IQQLiE3yQ3v5+NxH7nLued870ly3vf7/Z7v95i7IyIisied2roAIiLSvikoRESkSQoKERFpkoJCRESapKAQEZEmKShERKRJCgppkpktNLPT9rLMoWZWZmZ5rVSsrDOzFWb20eT+TWZ2X1uXScK+/D7M7Dkz+0K2y9TRKShyVHIg25kcoNeZ2d1m1q2lt+Pux7j7c3tZZqW7d3P36pbefnJQqEw+5xYze8nMprT0dg40DfZr6rYlw/dmdPA1s9PMzM3ssQbPj0uef66ZxZdWpqDIbee5ezdgAjAR+G7DBSzk+u95ZvI5+wGzgYfbuDwtzszy22CzM5OAT916ZWEbG4ApZtY37bnLgLezsC3Jklw/gAjg7quBZ4DRUPuN74dm9iKwAzjczHqa2V1mttbMVpvZD9KbiszsKjNbbGbbzWyRmU1Ink9vgplkZiVmti2pxfwseX5o8g0xP3k80MxmmdkHZrbMzK5K285NZvaQmd2bbGuhmU3M8HNWAfcDg8ysf9o6zzWz+Wk1jrFprw0xs/8xsw1mtsnMbkueP8LM/pI8t9HM7jezZh0ozez8ZPvbzGy5mZ3VcN+lffb7GuyzK81sJfAXM3vGzK5tsO7XzeyTyf2RZvbHZL++ZWYXNae8GX6mE83sVTPbmvw8sZmrqgAeBz6drDcPmEH8HjPanpkNM7Pnk7+XPxJfGNLfOzn5vW9J9tdpzSyr7IGCogMwsyHAx4HX0p6+BLga6A68B9wNVAHDgfHANOALyfsvBG4CLgV6ANOBTY1s6ufAz929B3AE8NAeivQgUAoMBC4AfmRmU9Nen54s0wuYBdyW4efsnJRxE7A5eW488Gvgi0Bf4D+BWWZWmByUfp98/qHAoGS7AAb8OCnj0cCQZB/sEzObBNwLXJ98nlOBFfuwio8k2/8Y8Dvg4rR1jwIOA54ys67AH4EHgAHEgfdXyTItysz6AE8BvyD26c+SMvRt8o17di/xe4P4nG8Ca/Zhew8A84iA+D5RI0m9d1Dy3h8AfYCvA4+mf5GQ/aegyG2PW7QrvwA8D/wo7bW73X1h8i28DxEk/+TuH7r7euA/SL7lEYHxE3d/1cMyd3+vke1VAsPNrJ+7l7n73IYLJKF1EvANdy939/nAf1N3oAB4wd2fTvo0fguM28vnvCj5nDuBq4ALks8FEYb/6e4vu3u1u98D7AImA5OIILg++dzl7v4CQPIZ/+juu9x9A3Fw+sheytGYK4FfJ+uqcffV7r5kH95/U1K2ncBjwLFmdljy2meB/3H3XcC5wAp3/427V7n7a8CjwIXNKHPKRcm38NRtdvL8OcBSd/9tsq3fAUuA85qzEXd/CehjZkcRfwf3Nlhkj9szs0OB44Ebk9/VHODJtPd+Dng6+Xuqcfc/AiXE37u0EAVFbvuEu/dy98Pc/cvJwSZlVdr9w4ACYG3qoEB88x6QvD4EWJ7B9q4EjgSWJM0D5zayzEDgA3ffnvbce8S3+ZT30+7vAIrMLN/MPmt1HavPpC3zUNJ+fhDxbfS4Bp/tn9MPeMnnGZj8fC8tVGqZ2UFm9qBFM9w24D4aNGlkKNN9tye1v6dknz1FXYBfTF0TzWHACQ0+52eBgxuu0MxOSduPC5vY9kPJ30/qdnry/EDid5au4e9wX/0WuBY4nQjEdE1tbyCw2d0/bPBaymHAhQ32y8nAIftRVmmgLTrQpHWkTwu8iviW3a+xg2by+hF7XaH7UuBii87xTwKPNNIcsYb49tg9LSwOBVZnsP77adB23eD1jWZ2NVBiZg+4+9qk7D909x82XN7i7KhDzSy/kc/9I2IfjXH3D8zsE2TYBNZAU/vuQ6A47fFuB3Xq/54gmp++Z2ZzgCKi8z61nefd/cy9Fcjd/wrszxlwa4gDcLpDgT/sxzp/CywD7nX3HWaW6fbWAr3NrGtaWBxK3X5bBfzW3a9CskY1igNAckD9X+CnZtbDzDolnbmpppb/Br5uZsdZGJ7W/FHLzD5nZv3dvQZInUpZ02Bbq4CXgB+bWZFFx/KVxDf2lvgsbwHPAjckT/0X8CUzOyEpe1czO8fMugOvEAeaW5Pni8zspOR93YEyYGvSzn19M4t0F3CFmZ2R7NdBZjYyeW0+8GkzK7DosL8gg/U9TRw0byHOSkrt398DR5rZJcn6CszseDM7upnl3lsZjjSzzyQ1vRnAqKQMKfnJ/kzdCppaobu/SzTtfWdftpc0gZYAN5tZZzM7mfpNYPcRTVQfM7O8pCynmdngZn962Y2C4sBxKdAZWER0BD9CUj1394eBHxKdhtuJs1T6NLKOs4CFZlZGdGx/ukFzV8rFROfxGqKZ4Xvu/qcW/Cz/BlxtZgPcvYTot7gt+VzLgMsBkj6Q84gO/JVEB/uMZB03E6cVbyWae/6nOQVx91eAK4g+n61EX1EqZG8kahubk+09kMH6diVl+Wj68kntbBrRLLWGaL77V6CwOeVOzLD64yjKkn26iegT+WfixIEbgHPdfWPae/8f0WeUuv0mg8/2gruvaeT5vW3vM8AJwAfA90jr40i+mJwPfJs4FXcVEfo6trUg04WLRESkKUpdERFpUlaDwszOshgYtMzMvtnI65dbDISan9w0J4uISDuTtbOeLAY73Q6cSbQNv2pms9x9UYNFZ7r7tbutQERE2oVs1igmAcvc/R13ryBGxJ6fxe2JiEgWZHMcxSDqD/oqJc5caOhTZnYqMUnYdclZDPUk585fDdC5c/fjBg06kooKcIe8POjXDzqpt0VEZI/mzZu30d2bNbVJWw+4exL4nbvvMrMvAvcAUxsu5O53AncCDBky0W++uYTqahg4EN59Fy69FLp0ad2Ci4jkEjNrbFqejGTze/hqYnqDlME0GJ3r7puS88YhBn2lT83QpLw8qD+4U0REsiGbQfEqMMJiiuDOxEChWekLmFn6fCzTgcWZrFhDP0REWk/Wmp7cvcpibv1ngTxihs2FZnYLUOLus4CvmNl0YvrrD0hG1IqISPuR1T4Kd3+amMcl/bl/Sbv/LeBb+75eNTvlksrKSkpLSykvL2/rooh0eEVFRQwePJiCgian39onbd2ZLQeA0tJSunfvztChQzElvEjWuDubNm2itLSUYcOGtdh6dVKpZF15eTl9+/ZVSIhkmZnRt2/fFq+9KyikVSgkRFpHNv7XFBQiItIk9VFIq5s7F7Zs2ftymerVCyZPbnqZvLw8xowZQ1VVFUcffTT33HMPxcXF9Za5++67uf766xk0aBDl5eV88Ytf5Lrrrmu5graQ1157jdtuu4277rqrrYvSbsyaNYtFixbxzW/uNvcoACUlJdx777384he/aNVyuTtf/epXefrppykuLubuu+9mwoQJuy33ne98h3vvvZfNmzdTVlZW+/yuXbu49NJLmTdvHn379mXmzJkMHTqUN954g5/+9KfcfffdrfI5crJGoXEUuW3LFujfv+VumYROly5dmD9/Pm+++SadO3fmjjvuaHS5GTNmMH/+fF588UV++MMfsmrVbjPK7JeqqsauRLtvfvSjH/GVr3ylBUrTuJYoY2tvf/r06XsMCYCJEye2ekgAPPPMMyxdupSlS5dy5513cs011zS63Hnnnccrr7yy2/N33XUXvXv3ZtmyZVx33XV84xvfAGDMmDGUlpaycuXKrJY/JSeDQmR/nHLKKSxbtqzJZfr27cvw4cNZu3btbq+VlZVxxRVXMGbMGMaOHcujjz4KQLdudZepfuSRR7j88ssBuPzyy/nSl77ECSecwA033MDQoUPZkpZuI0aMYN26dWzYsIFPfepTHH/88Rx//PG8+OKLu217+/btLFiwgHHjxgHwyiuvMGXKFMaPH8+JJ57IW2+9BcDkyZNZuHBh7ftOO+00SkpK+PDDD/n85z/PpEmTGD9+PE888QQQtanp06czdepUzjjjDMrKyjjjjDOYMGECY8aMqV0O4Pvf/z5HHXUUJ598MhdffDH//u//DsDy5cs566yzOO644zjllFNYsmTJbuW/6aabuOSSS5gyZQojRozgv/7rvwB47rnnOOWUU5g+fTqjRo0C4L777mPSpEkce+yxfPGLX6S6uhqAP/zhD0yYMIFx48Zxxhln1Jb/2mtjEuqHH36Y0aNHM27cOE499dTa9Z977rkAfPDBB3ziE59g7NixTJ48mQULFtSW7fOf/zynnXYahx9+eIsEyxNPPMGll16KmTF58mS2bNnS6N/U5MmTOeSQQxp9/2WXXQbABRdcwJ///GdSF5s777zzePDBB/e7jJnIyaYn1SikuaqqqnjmmWc466yzmlxu5cqVlJeXM3bs2N1e+/73v0/Pnj154403ANi8efNet1taWspLL71EXl4e1dXVPPbYY1xxxRW8/PLLHHbYYRx00EF85jOf4brrruPkk09m5cqVfOxjH2Px4vqTFZSUlDB69OjaxyNHjuSvf/0r+fn5/OlPf+Lb3/42jz76KDNmzOChhx7i5ptvZu3ataxdu5aJEyfy7W9/m6lTp/LrX/+aLVu2MGnSJD760Y8C8Pe//50FCxbQp08fqqqqeOyxx+jRowcbN25k8uTJTJ8+nZKSEh599FFef/11KisrmTBhAscdFzPvXH311dxxxx2MGDGCl19+mS9/+cv85S9/2W1fLFiwgLlz5/Lhhx8yfvx4zjnnnNrtv/nmmwwbNozFixczc+ZMXnzxRQoKCvjyl7/M/fffz9lnn81VV13FnDlzGDZsGB988MFu67/lllt49tlnGTRoUL1ATvne977H+PHjefzxx/nLX/7CpZdeyvz58wFYsmQJs2fPZvv27Rx11FFcc801u41HmDFjRm0gp/va177GpZdeWu+51atXM2RI3UxGgwcPZvXq1Y2GQmPS35+fn0/Pnj3ZtGkT/fr1Y+LEidx6663ccMMNe1nL/svJoBDZVzt37uTYY48FokZx5ZVXNrrczJkzmTNnDkuWLOG2226jqKhot2X+9Kc/1fsm17t3771u/8ILLyQvLw+IA80tt9zCFVdcwYMPPsiMGTNq17toUd3lWrZt20ZZWVm9msratWvp379uAtCtW7dy2WWXsXTpUsyMyspKAC666CKmTZvGzTffzEMPPcQFF1wAwP/+7/8ya9as2lpAeXl5bfPFmWeeSZ8+cal0d+fb3/42c+bMoVOnTqxevZp169bx4osvcv7551NUVERRURHnnXceELWsl156iQsvvLC2bLt27aIx559/Pl26dKFLly6cfvrpvPLKK/Tq1YtJkybVnvv/5z//mXnz5nH88ccD8fsbMGAAc+fO5dRTT61dLlXedCeddBKXX345F110EZ/85Cd3e/2FF16orQVOnTqVTZs2sW3bNgDOOeccCgsLKSwsZMCAAaxbt47BgwfXe//MmTMb/VytbcCAAaxZs9slyLMiJ4NCNQrZV6k+inS33357bdPH00/HBAIzZszgtttuo6SkhGnTpjF9+nQOPvjgjLaRflpiw/PYu3btWnt/ypQpLFu2jA0bNvD444/z3e9+F4Camhrmzp3baDilf470dd94442cfvrpPPbYY6xYsYLTTjsNgEGDBtG3b18WLFjAzJkza/tk3J1HH32Uo446qt56X3755XplvP/++9mwYQPz5s2joKCAoUOHNnlufk1NDb169dptHzem4embqcfp23d3LrvsMn784x/XW/bJJ5/c6/rvuOMOXn75ZZ566imOO+445s2bt9f3pBQWFtbez8vLa7S/ZF9qFIMGDarXz1VaWsqgQYMyLk/q/YMHD6aqqoqtW7fSt29fIP7GurTStNk52UehoJCW8I//+I/Mnz+f+fPnM3DgwHqvTZw4kUsuuYSf//znu73vzDPP5Pbbb699nGp6Ouigg1i8eDE1NTU89thje9yumfEP//APfO1rX+Poo4+u/cefNm0av/zlL2uXa+yge/TRR9frX9m6dWvtgafhGTAzZszgJz/5CVu3bq1tQvvYxz7GL3/5y9p27tdee63RMm7dupUBAwZQUFDA7Nmzee+9mKH6pJNO4sknn6S8vJyysjJ+//vfA9CjRw+GDRvGww8/DMSB/vXXX2903U888QTl5eVs2rSJ5557rrbWkO6MM87gkUceYf369UD0K7z33ntMnjyZOXPm8O6779Y+39Dy5cs54YQTuOWWW+jfv/9uJySccsop3H///UD0XfTr148ePXo0WtbGzJw5s/bvJv3WMCQgOtnvvfde3J25c+fSs2fPjJudUu+/5557gOj3mjp1am2wvv322/WaIbMpJ4MCNNdTLuvVCzZsaLlbr17ZKec3vvENfvOb37B9+/Z6z3/3u99l8+bNtR2ms2fPBuDWW2/l3HPP5cQTT9zrwWDGjBncd999tc1OAL/4xS8oKSlh7NixjBo1qtEzs0aOHMnWrVtry3TDDTfwrW99i/Hjx+/27feCCy7gwQcf5KKLLqp97sYbb6SyspKxY8dyzDHHcOONNzZavs9+9rOUlJQwZswY7r33XkaOHAnA8ccfz/Tp0xk7dixnn302Y8aMoWfPnkDUQu666y7GjRvHMcccU68DPN3YsWM5/fTTmTx5MjfeeONuIQ0watQofvCDHzBt2jTGjh3LmWeeWdvsduedd/LJT36ScePG1dt/Kddffz1jxoxh9OjRnHjiibUd/yk33XQT8+bNY+zYsXzzm9+sPRBnw8c//nEOP/xwhg8fzlVXXcWvfvWr2tdSTaEQv8fBgwezY8cOBg8ezE033QTAlVdeyaZNmxg+fDg/+9nPuPXWW2vfM3v27Nr+nWwzz7Gv50OGTPTvfa+EoiI46CB45x1duKi9W7x4MUcffXRbF6PD+I//+A+6d+/OF77whTbZfqrfZMeOHZx66qnceeedjY4NaMxNN91Et27d+PrXv57lUnZsu3bt4iMf+QgvvPAC+fm79yA09j9nZvPcfWJztpeTNYqaGtUo5MB1zTXX1GtLb21XX301xx57LBMmTOBTn/pUxiEhLWflypXceuutjYZENuRkZzYoKOTAVVRUxCWXXNJm23/ggQea/d5Uk4rsnxEjRjBixIhW215O1ih0PYrck2tNnCK5Khv/azkbFJI7ioqK2LRpk8JCJMtS16No6hTr5sjJpifVKHLL4MGDKS0tZcOGDW1dFJEOL3WFu5aUk0EhuaWgoKBFr7YlIq0rJ5uedNaTiEjrycmgAAWFiEhrycmgUB+FiEjrydmgEBGR1qGgEBGRJikoRESkSTkZFKA+ChGR1pKTQaHObBGR1pOTQaFxFCIirScng0I1ChGR1pOTQSEiIq0nJ4NCZz2JiLSenA0KNT2JiLQOBYWIiDRJQSEiIk3KyaAABYWISGvJalCY2Vlm9paZLTOzbzax3KfMzM1sYibrralpuTKKiEjTshYUZpYH3A6cDYwCLjazUY0s1x34KvBytsoiIiLNl80axSRgmbu/4+4VwIPA+Y0s933gX4HyTFfsDp1yttFMRCS3ZPNwOwhYlfa4NHmulplNAIa4+1NNrcjMrjazEjMrKSvbgjv079/yBRYRkd212fdyM+sE/Az4570t6+53uvtEd5/YrVsv8vMhLy/7ZRQRkewGxWpgSNrjwclzKd2B0cBzZrYCmAzMyrRDW0REWkc2g+JVYISZDTOzzsCngVmpF919q7v3c/eh7j4UmAtMd/eSLJZJRET2UdaCwt2rgGuBZ4HFwEPuvtDMbjGz6dnaroiItKz8bK7c3Z8Gnm7w3L/sYdnTslkWERFpHp1kKiIiTVJQiIhIkxQUIiLSpJwLCl20SESkdeVcUIBmjhURaU05GRQiItJ6OkRQVFfD3LnwxBNqmhIRaWkdIigqK6GsDLZsidAQEZGWk5NB0VgfRXW1ahMiItmQk0HRmOpq1SZERLIhJ4NiTzUKXSJVRKTl5WRQNEZBISKSHTkZFI3VKCoqWr8cIiIHgpwMisZUVbV1CUREOqYOERQ1NXGKrIiItLycDoodO6Im8fzzCgoRkWzJ6aBInQ5bVhaBoTmgRERaXk4GRcNAcI/mp7y8timPiEhHlpNB0VBNTdw6dYhPIyLSvuTcodW9fo3CPW7V1Wp6EhHJhpwLioZS8ztpnicRkezIyaBIrzmkRmNrnicRkezI+aBQTUJEJLtyLijy82HMmLrHqkmIiGRXzgVFYSH07NnWpRAROXDkXFA0pBqFiEh25XxQqI9CRCS7cj4oamoiLDTYTkQkO3L+8JoabKegEBHJjpw+vKbmeHKHPn3aujQiIh1TTgdFUVFdWBQWtnVpREQ6ppwOioMPhv79NSGgiEg2dYjDq4JCRCR7OsThVUEhIpI9WT28mtlZZvaWmS0zs2828vqXzOwNM5tvZi+Y2ajmbEcXLRIRyZ6sBYWZ5QG3A2cDo4CLGwmCB9x9jLsfC/wE+Nm+bifVma1rUYiIZEc2axSTgGXu/o67VwAPAuenL+Du29IedgX2eZx16up2IiKSHflZXPcgYFXa41LghIYLmdk/Al8DOgNTG1uRmV0NXA0wYMCh9V5TUIiIZFebdwG7++3ufgTwDeC7e1jmTnef6O4Te/bsX+81BYWISHZlXKMws0HAYenvcfc5TbxlNTAk7fHg5Lk9eRD4f5mWJ0VBISKSXRkFhZn9KzADWASkJvZ2oKmgeBUYYWbDiID4NPCZBusd4e5Lk4fnAEvZR6kpPEREJDsyrVF8AjjK3XdlumJ3rzKza4FngTzg1+6+0MxuAUrcfRZwrZl9FKgENgOX7VvxYeRI+OAD2LkTdu2Ct9+GUc06yVZERBqTaVC8AxQAGQcFgLs/DTzd4Ll/Sbv/1X1ZX2MOPjhuixbF49deU1CIiLSkTINiBzDfzP5MWli4+1eyUqpmqq7WeAoRkZaWaVDMSm7tWnU15GfzhF8RkQNQRodVd7/HzDoDRyZPveXuldkrVvOkRmhXV2tKDxGRlpLROAozO404I+l24FfA22Z2ahbL1SxVVbBjB7z1VluXRESk48i0oeanwDR3fwvAzI4Efgccl62CNUd1ddwqKtq6JCIiHUemI7MLUiEB4O5vE2dBtRupyQHVTyEi0rIyPaSWmNl/A/cljz8LlGSnSM2Tqk0oKEREWlamNYpriFHZX0lui5Ln2o2+fetqFCIi0nIyPetpF3GtiH2+XkRrGTQIli6NDm0REWk5TQaFmT3k7heZ2Rs0cq0Idx+btZI1Q3W1LokqItLS9lajSE2xcW62C9IS3NX0JCLS0pr8/u3ua5O7G4FV7v4eUAiMA9ZkuWz7bNQoDbQTEWlpmTbUzAGKkmtS/C9wCXB3tgrVXAMHQkG7OmlXRCT3ZRoU5u47gE8Cv3L3C4FjslcsERFpLzIOCjObQoyfeCp5To08IiIHgEyD4p+AbwGPJRcfOhyYnb1iiYhIe5HpOIrngefTHr9DDLwTEZEObm/jKP6vu/+TmT1J4+MopmetZCIi0i7srUbx2+Tnv2e7ICIi0j41GRTuPi+5WwLsdPcaADPLI8ZTiIhIB5dpZ/afgeK0x12AP7V8cUREpL3JNCiK3L0s9SC5X9zE8iIi0kFkGhQfmtmE1AMzOw7YmZ0iiYhIe5LpJX7+CXjYzNYABhwMzMhaqUREpN3IdBzFq2Y2Ejgqeeotd6/MXrFERKS9yKjpycyKgW8AX3X3N4GhZpYTU4+LiMj+ybSP4jdABTAlebwa+EFWSiQiIu1KpkFxhLv/BKgESGaStayVSkRE2o1Mg6LCzLqQTONhZkcAu7JWqnZq1664ip6IyIEk06D4HvAHYIiZ3U8MwLsha6VqYx9+CBUV9Z9bvx7+8AeYN6/x94iIdFR7PevJzAxYQly0aDLR5PRVd9+Y5bK1iW3b4KWXoKwMLrgANm6EN9+E8nJYswaqqmDMGCjUBCYicoDYa1C4u5vZ0+4+hrqLFnVYu3bB6tVgBps3w+uvw7vvxmsffBAB8dZbUFMDQ4ZA375tW14RkWzLdMDd383seHd/NaulaWPu8M47UZs45BBYtCgeb9kClZXQqRNs3x4BsmxZPKegEJGOLtOgOAH4nJmtAD4kmp/c3cdmq2BtoaYmmp527IhQWL48mp769o1bQUHULlasiNpF585tXWIRkezLNCg+ltVStBMrVkSndVFRBEZZGfTvD+PHx+tVVREU27ZFSLz/PoweDXn7cPXwHTvg7behe3c44oisfAwRkRa1tyvcFQFfAoYDbwB3uXtVpis3s7OAnwN5wH+7+60NXv8a8AWgCtgAfN7d39unT9BM7rBkCWzdCiecEM1I778fNYiioui83r4dTjyx7j01NbGcWdQ41qyJmsWGDbByJUydGsssXQpdusDw4fW3WV0Nr70WneOpMDr8cOjZszU+sYhI8+zt9Nh7gIlESJwN/DTTFScXN7o9ed8o4GIzG9VgsdeAiUkT1iPATzJd//6qrIRVq2DhQli7Ng74y5ZFLaK6OgJk/PgIhZT8/Gh+OuKIOH12+/YIi5KSqI3s2hWd36+8Ai+/vPs2ly2L5TZsiH6PBQvg8cdb6xOLiDTP3pqeRiVnO2FmdwGv7MO6JwHL3P2d5P0PAucDi1ILuPvstOXnAp/bh/U3W00NbNoUB+uKCpg9O5qQysrg2GOjWWnzZjjooPrv69QJTj45VfYImi1bImTy8yMA1q2L28CBUFoKgwZFKL33Xl2fx3HHRaCUlcHgwa3xiUVEmm9vQVE7Q6y7V5nt06wdg4BVaY9LiU7xPbkSeKaxF8zsauBqgAEDDt2XMjTq/ffhr3+NA/yOHdEM9MYbUZM4+OBYpk+fptcxYEAExbvvQnFxBM4bb0SNoWvXaFZasiSaoVbJWpcNAAAVbElEQVStimDZtClqKT16wJQpESiVmoNXRNq5vTU9jTOzbcltOzA2dd/MtrVUIczsc0QT17819rq73+nuE919Ys+e/fd7e2Vl0bS0c2f0RezYEaOxe/XKfB0VFVEzWb8+ag8QIZEaX5EaoPfGGxFI778fHdg9esSy3bpFLQRi2fLy/f5YIiJZ0WSNwt334Xye3awGhqQ9Hpw8V4+ZfRT4DvARd2+R+aNqavb8WllZfMvfuTNuxcURElVV0eyUqfz8eN+QIXF/+/YIgpNOin6Nt9+Ovo/i4rgddVTUQtKlxmc8/HCU+cILY1kRkfYk09Njm+NVYISZDSMC4tPAZ9IXMLPxwH8CZ7n7+pbYqDv8/e9xNlFR0e6v19REDWLbtjh49+8fHdpduuzeJ9GU4uI4WypVCzn66AiNlIEDoV+/pteZnx/NT1u3RnPVs8/C2Wc3Xm4RkbaStaBI+jSuBZ4lTo/9tbsvNLNbgBJ3n0U0NXUjLrMKsNLdp+/Pdquq4lZZufsBd/Xq6ET+4IPoI+jdO56fMmX39WQi9X6oHxIAxxyz9/ePHBnr2L49+jry82Hx4rpxGyIi7UE2axS4+9PA0w2e+5e0+x9t6W1WVu65g7isLJqDysujj6Ct5eXV9W9UVkafxsqVETIa9S0i7UWm04znlKq0IYEVFdHhvHVrHIh37YpbQUHbla8xo0dHmdasiVNzIYJNndwi0tY6XFC4x2muzz8fj9eujetIzJ0b39a3b29/IZFyyCERbNXV0UT2+9/Do4/WBYeISFvocEGRqk1sTK6W4R4H33Xr4mdRUfQNtEfDhsWgvsWL4wJJ770XITFrVgzmExFpCx0uKA4+OL6RV1fH44qKaGqqqIiznaZM2f001fYi1RG/cGEE29ChcQrv5s3wt7/VfSYRkdbU4YJi0KC6WsWHH8aZTukD2vZlptfW1rlz1CiKimDatJhU8KSTIixWr9YobhFpG1k966mtVFTEqaavvBJhUV4eB+B9GXndFgoK6uaSSiksjMBQP4WItJUOV6MoLIwmmoKC6Lxety4GvZWXx6C6XKRpyEWkLXW4oOjRI76BV1VF30R5ebT19+hRN2Yh11RVRfi91ypX6hARqa/DBQVEs1NlZTRBffhhTI8xeXL8zEXl5fE55s9v65KIyIGoQwZFWVndabHFxbk/ynnQoOjg1oSBItIWOmRQVFREjaI9jsBujvz8+vNKiYi0pg4ZFBBhkZcX38Y7AvcIv9mz4Y9/jMciIq2hQ54em2p2mjq1/jWvc9m2bdFXsWpVnL1VXV134SMRkWzqkIeaqqoIio4SEhCz3a5fHxdHqq6GBx6I5qiamui/yM+PnwcdFFOBtOeBhSKSWzpkUBx1VFx6tCM55pi4rV4NCxbE5Ibr10doFBVFOHbrBm++GUExbVrde7dti1qWxmOISHN0yKDo3bvjdv4OGhSzzJaVxSmzeXkxqNA9mqY2b45R6MuXx8SIGzfGhII1NXDRRREmlZVRA+lINS4RyZ4OGRQdXadOMYCwR494nD7J4fz5UYN4/vmYIyp1PfCuXeGpp6Bv36htjR0bNxGRvVFQdDAjR0bz05Yt0dQ0fnw0T73ySlwUqbQ0ah59+8IRR8Tpw2++GbWOXr1gwoSOcUqxiLQcBUUHU1QEEyfu/vzkydEsVVgIixbFPFgPPRRNWUuXRs2je/cIkOHDW7/cItJ+KSgOEEVF0bcBcPjhMW9UWVnUJI48MmoRb78dobF6dYRKUVHUOkaMaNuyi0jbUlAcgAYOjNNo3303JkxMjcdYsiQ6wXftin4Ns6hpjBgRneEbNkT/SP/+bVp8EWllCooDVF7e7k1Mxx0XtYpx4yIYVqyI5RYujBpIaWkExbhxcPzxbVJsEWkDCgqp1b17hEVK6oyptWvjTKoePeJ026VLFRQiBxIFhexRv35RixgwIILBLDrCU81QaoISOTAoKGSPjjgibul27oxO8CefjH6Ozp3hhBNiIF+nDjvFpMiBTUEh+2TcuBh3sW5dNEN17hzjM7p1i9vpp+f+9T9EpD4FheyT/Hw49thofqqoiNBYsyYe9+oVM9uecoqmBxHpSBQU0iydOtUN7quujtB46aXow9i4MQbuHXZYBEefPtEpvmFDvNajBxx6aNQ8qqsjZDQaXKT9UlDIfsvLi0AYNy5OpV2+PK6bsWhR1CwKC+tqIBUVUSsZPjx+btsWg/tGjIiaiIi0PwoKaTH9+sXBvrwcli2LWkRVVQRBfn5MTNinT4z8XrIkZrEtKKi7xvmHH8YU8YcequtpiLQnCgppUZ06QXFx0zPT9u8fNYxu3eomJXz//QiUd9+N9w8eHCEzZEhMdKgzqkTajoJCWl3DCygdcwyMGhXNVevWRY1j06ZoplqxAhYvjqaqMWMUGCJtQUEhbc4sbocdFrcdOyIQyssjJN5+OwKkU6cICxFpXQoKaXeKi+NnURFMmRId3n//ewTGxo3RJFVQACefXDehoYhkT1b/zczsLODnQB7w3+5+a4PXTwX+LzAW+LS7P5LN8khu6tEjQuOdd6Ipqro6wmTTpugQ7949+jEajiIXkZaRtaAwszzgduBMoBR41cxmufuitMVWApcDX89WOaRjmDQpzozauTPGX7zySsxoW1kZz69fDx98EFfo0xlTIi0rmzWKScAyd38HwMweBM4HaoPC3Vckr9VksRzSAaQ6sbt1i58nn1z32vLlcTruzp0xs+3JJ8cptiLSMrIZFIOAVWmPS4ETmrMiM7sauBpgwAAdAaS+I46IS7ouWBC1jPLymLCwoiLOsOrdO+6PGhVNWO7Rea7R4CKZyYmuQHe/E7gT4MgjJ3obF0faoaKiaJ5atSpqFRs3RrNUt24RCjU10cexc2cERUEBnHNOhIhOuRVpWjaDYjUwJO3x4OQ5kawZMiQG65lFUJSWRmf4O+/EXFN5efFaWRk88kgETHFxTHSoa4OLNC6bQfEqMMLMhhEB8WngM1ncnghQN3NtQQEMGxb3+/ate72qCl5/PWoZmzdHLSQvL86m6t49liksjPdoFlyRLAaFu1eZ2bXAs8Tpsb9294VmdgtQ4u6zzOx44DGgN3Cemd3s7sdkq0wiEGMv0i/5+uqrERalpREuNTWxTFFRBMjQodE53r9/TH5YXR0BoiYrOVCYe241+R955ET/3e9K2roY0oFUVMSpteXlUcMoLo6f7tFEVVgYt86d42dFRdzv2TOW7dSpbqr04uKY2LC4uK6ZS6Q9MLN57j6xOe/Nic5skWzq3BkOPjjuDx1a/7Xy8hgZvnZtTC1SXh4H/82bY06q1JlT1dV14zcWL47+kd6969bbo0fcDj44ltNYD8klCgqRJhQVxW3AgN1fq6qK0HCPpqodO2I8x/bt8dqmTTEbbqpmkZ8fNZKCApg8OUKpsLDVP5LIPlNQiDRTw3mmGk6vXlUVIbJ1azwuLa1r5vrjH6O/o3v3qI307w8DB8YsubrmuLQ3CgqRLEkFSb9+9X+uXAlbtkST1s6dccGm99+PKwIuWxbjO9Q0Je2JgkKklR16aP0pRlJXAlywIEJk8WIYPbrtyifSkIJCpI3l58dt3Lg4Vff112P+qm7dIkA6dYpBhIMGRYe4SGtTUIi0E4WF0W/x/vsREF26xFlW7tEs1a0bnHqqRpBL61NQiLQj6QMBU1LjOf72t7iAU3Fx9HfojClpLQoKkXbOLM6OGjEiZsf9n/+Brl3hxBPjOXV8S7YpKERyxLBhMenhxo3RjzFnTtQyBgyIkeGp8RwjR0ZneadOmmZEWoaCQiSH5OfH6O5OnWJk+ObNMS4jNY1IVVUM8isoqBsB3rNnhMeRR6r2Ic2joBDJQQMGND5avLo6LgtbUQHr1sXP9esjVF57LWbE7dIlzqI6/PDWL7fkJgWFSAeSlweHHBL3DzssfrrDG2/AmjVxRlVlZXSGr1tXd9qtmqikKQoKkQ7OrP7UItu3x3iNl16KM6e6d4c+feLU2549266c0n4pKEQOMN27w9Sp0aexZk1ci2P9+rjfq1ecUVVUFP0hvXrFmVWpgX+piQ1VAzmwKChEDlCpEd+DB0en+Jo1MQfVqlV1QWAWNQ+z6P8oKIiwKC6OwOjaNR6nLvbUp08MDOzWre4iUF26xLo6d66bll1yi4JCROjdO27p3GPCwsrKmMCwa9d4vGZNdJLn5UVNxB127YpbYWGESl5ehESnTvHYLIKia9eYhiQVHgcdFLfq6rprevTvrxpLe6OgEJFGmUXNAOpCpF+/uk7yhqqq4mBfWRlB0qVLhAfEVCRbt0aNZevWWK6sLJq4IMLGPYKmd++onfTvH2dpFRbG/Z49FSBtRUEhIi0iNblh+tQiqZoD1F3tL6W6OjrWU2FhFtfsKC+PKwquWhXLpJqsiovjZ2FhBFa/ftGH0rt31FQkexQUItIm8vLiQJ/uiCPqP3aPKwdu2hQBUlERTV8rV8ZreXnROX/QQVHjyM+PwOnVK64gmAoh2T8KChFpt8yittBYjaGqKpqvli2LObAggiTV19GjR4THkUdGh32qz6RnT41Q31cKChHJSanTdydO3P21jRtjQOHy5TEqPf3a5t27x/v694/5s7p0qZsXK3Vf6lNQiEiHk+rDOPro6FyHCIr166NDfenSuL36at1ZWamg6NMn+kXM6qY5GTAgaiFdux6YQaKgEJEOq1On+p3r6ZehraysC5GamhhLsnFj3MrLo1lryZJ4LTXJYmoUe+fOcUbY6NEHRngoKETkgFRQUH8AYLduMY17Sk1N1CpS/R7r10en+po1ESI1NTHRYu/e0Zk+cmSERmrMSOossI6gg3wMEZGWlaolpGokQ4fGDerOxlq3rm4alIUL697XqVOEUP/+UQvp1y9e69kzxoakBiLmynQoCgoRkX2UOhvr8MPjVlUVNY+amvhZURG1jxUrYvmCggiWgoKobaQUFUWN5IgjohnLrE0+zl4pKERE9lNjzUyNDTDcsSNCpaYGdu6Mad/feisGGpaU1M2fBREq7lHjGDiwrhbSpUtMJV9c3DqfDRQUIiKtItUZnm7w4AiQNWsiOHbtig72mpp4fdu2eLxsWTwuL48aSVFR9IscdVSsM3VGVmFh3ZUNW7J2oqAQEWlDeXn1O9GbUlMT054sXx6n9y5fXr9PJDUhY2FhhElxcQw83N8BhgoKEZEckRpZPmFCPK6qqpu9t6IiahxVVdHElZqAcenS/d+ugkJEJEel+kX2dp2PDRv2bzs5cGKWiIi0JQWFiIg0SUEhIiJNympQmNlZZvaWmS0zs2828nqhmc1MXn/ZzIZmszwiIrLvshYUZpYH3A6cDYwCLjazUQ0WuxLY7O7Dgf8A/jVb5RERkebJ5llPk4Bl7v4OgJk9CJwPLEpb5nzgpuT+I8BtZmbu7ntaqfv+9+CLiBxIqqr27/3ZDIpBwKq0x6XACXtaxt2rzGwr0BfYmL6QmV0NXJ083HX22fZmVkqce/rRYF8dwLQv6mhf1NG+qHNUc9+YE+Mo3P1O4E4AMytx90auaXXg0b6oo31RR/uijvZFHTMrae57s9mZvRpIH5g+OHmu0WXMLB/oCWzKYplERGQfZTMoXgVGmNkwM+sMfBqY1WCZWcBlyf0LgL801T8hIiKtL2tNT0mfw7XAs0Ae8Gt3X2hmtwAl7j4LuAv4rZktAz4gwmRv7sxWmXOQ9kUd7Ys62hd1tC/qNHtfmL7Ai4hIUzQyW0REmqSgEBGRJrXboND0H3Uy2BdfM7NFZrbAzP5sZoe1RTlbw972RdpynzIzN7MOe2pkJvvCzC5K/jYWmtkDrV3G1pLB/8ihZjbbzF5L/k8+3hblzDYz+7WZrTdrfKyZhV8k+2mBmU3IaMXu3u5uROf3cuBwoDPwOjCqwTJfBu5I7n8amNnW5W7DfXE6UJzcv+ZA3hfJct2BOcBcYGJbl7sN/y5GAK8BvZPHA9q63G24L+4ErknujwJWtHW5s7QvTgUmAG/u4fWPA88ABkwGXs5kve21RlE7/Ye7VwCp6T/SnQ/ck9x/BDjDrCWvEttu7HVfuPtsd9+RPJxLjFnpiDL5uwD4PjFvWHlrFq6VZbIvrgJud/fNAO6+vpXL2Foy2RcO9Eju9wTWtGL5Wo27zyHOIN2T84F7PcwFepnZIXtbb3sNisam/xi0p2XcvQpITf/R0WSyL9JdSXxj6Ij2ui+SqvQQd3+qNQvWBjL5uzgSONLMXjSzuWZ2VquVrnVlsi9uAj5nZqXA08D/aZ2itTv7ejwBcmQKD8mMmX0OmAh8pK3L0hbMrBPwM+DyNi5Ke5FPND+dRtQy55jZGHff0qalahsXA3e7+0/NbAoxfmu0u9e0dcFyQXutUWj6jzqZ7AvM7KPAd4Dp7r6rlcrW2va2L7oDo4HnzGwF0QY7q4N2aGfyd1EKzHL3Snd/F3ibCI6OJpN9cSXwEIC7/w0oIiYMPNBkdDxpqL0Ghab/qLPXfWFm44H/JEKio7ZDw172hbtvdfd+7j7U3YcS/TXT3b3Zk6G1Y5n8jzxO1CYws35EU9Q7rVnIVpLJvlgJnAFgZkcTQXEgXrBgFnBpcvbTZGCru6/d25vaZdOTZ2/6j5yT4b74N6Ab8HDSn7/S3ae3WaGzJMN9cUDIcF88C0wzs0VANXC9u3e4WneG++Kfgf8ys+uIju3LO+IXSzP7HfHloF/SH/M9oADA3e8g+mc+DiwDdgBXZLTeDrivRESkBbXXpicREWknFBQiItIkBYWIiDRJQSEiIk1SUIiISJMUFCINmFm1mc03szfN7Ekz69XC67/czG5L7t9kZl9vyfWLtDQFhcjudrr7se4+mhij849tXSCRtqSgEGna30ibNM3MrjezV5O5/G9Oe/7S5LnXzey3yXPnJddKec3M/mRmB7VB+UX2W7scmS3SHphZHjHtw13J42nEXEmTiPn8Z5nZqcQcY98FTnT3jWbWJ1nFC8Bkd3cz+wJwAzFCWCSnKChEdtfFzOYTNYnFwB+T56clt9eSx92I4BgHPOzuGwHcPXU9gMHAzGS+/87Au61TfJGWpaYnkd3tdPdjgcOImkOqj8KAHyf9F8e6+3B3v6uJ9fwSuM3dxwBfJCaiE8k5CgqRPUiuGvgV4J+TqeyfBT5vZt0AzGyQmQ0A/gJcaGZ9k+dTTU89qZvC+TJEcpSankSa4O6vmdkC4GJ3/20yRfXfkll6y4DPJTOV/hB43syqiaapy4mrqj1sZpuJMBnWFp9BZH9p9lgREWmSmp5ERKRJCgoREWmSgkJERJqkoBARkSYpKEREpEkKChERaZKCQkREmvT/AeDa85pJ7KRwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, _ = precision_recall_curve(y_test, pred)\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n",
    "average_precision = average_precision_score(y_test, pred)\n",
    "\n",
    "plt.figure()\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(\n",
    "    recall, precision, alpha=0.2, color='b',\n",
    "    label='P-R curve (average precision = %0.2f)' % average_precision\n",
    ")\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 0.5])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.legend(loc=\"upper right\")\n",
    "plt.title('Precision-Recall curve - EoL Model')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate Model : Extract Feature Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_weights = pd.DataFrame({\n",
    "    'feature_name': feature_names,\n",
    "    'feature_weight': list(clf.get_params()['lr'].coef_[0])\n",
    "}).sort_values(by='feature_weight')\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_name</th>\n",
       "      <th>feature_weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>131710</th>\n",
       "      <td>4001171 - condition - Liver cell carcinoma - 730 days</td>\n",
       "      <td>0.441087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120087</th>\n",
       "      <td>2211508 - procedure - Radiologic examination, abdomen; complete, including decubitus and/or erect views - 730 days</td>\n",
       "      <td>0.444479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63691</th>\n",
       "      <td>4152384 - condition - Non-rheumatic aortic sclerosis - 180 days</td>\n",
       "      <td>0.457767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77428</th>\n",
       "      <td>196925 - condition - Secondary malignant neoplasm of retroperitoneum and peritoneum - 365 days</td>\n",
       "      <td>0.464019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136610</th>\n",
       "      <td>4155034 - condition - Laceration of forearm - 730 days</td>\n",
       "      <td>0.464407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156125</th>\n",
       "      <td>2111054 - procedure - Treatment of extensive or progressive retinopathy (eg, diabetic retinopathy), photocoagulation - 10000 days</td>\n",
       "      <td>0.473012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136643</th>\n",
       "      <td>4157389 - condition - Protein deficiency disease - 730 days</td>\n",
       "      <td>0.493533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147484</th>\n",
       "      <td>1719007 - drug - Osmotic 24 HR Nifedipine 30 MG Extended Release Oral Tablet - 10000 days</td>\n",
       "      <td>0.509225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139214</th>\n",
       "      <td>43011909 - drug - 12 HR Aspirin 25 MG / Dipyridamole 200 MG Extended Release Oral Capsule [Aggrenox] - 730 days</td>\n",
       "      <td>0.532505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175240</th>\n",
       "      <td>42873649 - drug - 0.5 ML Influenza A virus vaccine, A-California-7-2009 (H1N1)-like virus 0.03 MG/ML / Influenza A virus vaccine, A-Victoria-361-2011 (H3N2)-like virus 0.03 MG/ML / Influenza B virus vaccine, B-Wisconsin-1-2010-like virus 0.03 MG/ML Prefilled Syringe [Fluar - 10000 days</td>\n",
       "      <td>0.549499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120766</th>\n",
       "      <td>2313827 - procedure - Rhythm ECG, 1-3 leads; interpretation and report only - 730 days</td>\n",
       "      <td>0.555944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178096</th>\n",
       "      <td>438383 - condition - Essential thrombocythemia - 10000 days</td>\n",
       "      <td>0.559293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181258</th>\n",
       "      <td>78097 - condition - Secondary malignant neoplasm of bone - 10000 days</td>\n",
       "      <td>0.565785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156391</th>\n",
       "      <td>2211374 - procedure - Radiologic examination, ribs, bilateral; 3 views - 10000 days</td>\n",
       "      <td>0.567901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144897</th>\n",
       "      <td>78706 - condition - Benign neoplasm of urinary system - 730 days</td>\n",
       "      <td>0.609550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173044</th>\n",
       "      <td>4155034 - condition - Laceration of forearm - 10000 days</td>\n",
       "      <td>0.645616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100326</th>\n",
       "      <td>4168700 - condition - Localized enlarged lymph nodes - 365 days</td>\n",
       "      <td>0.663873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167861</th>\n",
       "      <td>378419 - condition - Alzheimer's disease - 10000 days</td>\n",
       "      <td>0.667106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169200</th>\n",
       "      <td>40171499 - drug - 24 HR Verapamil hydrochloride 120 MG Extended Release Oral Capsule - 10000 days</td>\n",
       "      <td>0.709910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177860</th>\n",
       "      <td>437366 - condition - Contracture of tendon sheath - 10000 days</td>\n",
       "      <td>0.734703</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                          feature_name  \\\n",
       "131710  4001171 - condition - Liver cell carcinoma - 730 days                                                                                                                                                                                                                                            \n",
       "120087  2211508 - procedure - Radiologic examination, abdomen; complete, including decubitus and/or erect views - 730 days                                                                                                                                                                               \n",
       "63691   4152384 - condition - Non-rheumatic aortic sclerosis - 180 days                                                                                                                                                                                                                                  \n",
       "77428   196925 - condition - Secondary malignant neoplasm of retroperitoneum and peritoneum - 365 days                                                                                                                                                                                                   \n",
       "136610  4155034 - condition - Laceration of forearm - 730 days                                                                                                                                                                                                                                           \n",
       "156125  2111054 - procedure - Treatment of extensive or progressive retinopathy (eg, diabetic retinopathy), photocoagulation - 10000 days                                                                                                                                                                \n",
       "136643  4157389 - condition - Protein deficiency disease - 730 days                                                                                                                                                                                                                                      \n",
       "147484  1719007 - drug - Osmotic 24 HR Nifedipine 30 MG Extended Release Oral Tablet - 10000 days                                                                                                                                                                                                        \n",
       "139214  43011909 - drug - 12 HR Aspirin 25 MG / Dipyridamole 200 MG Extended Release Oral Capsule [Aggrenox] - 730 days                                                                                                                                                                                  \n",
       "175240  42873649 - drug - 0.5 ML Influenza A virus vaccine, A-California-7-2009 (H1N1)-like virus 0.03 MG/ML / Influenza A virus vaccine, A-Victoria-361-2011 (H3N2)-like virus 0.03 MG/ML / Influenza B virus vaccine, B-Wisconsin-1-2010-like virus 0.03 MG/ML Prefilled Syringe [Fluar - 10000 days   \n",
       "120766  2313827 - procedure - Rhythm ECG, 1-3 leads; interpretation and report only - 730 days                                                                                                                                                                                                           \n",
       "178096  438383 - condition - Essential thrombocythemia - 10000 days                                                                                                                                                                                                                                      \n",
       "181258  78097 - condition - Secondary malignant neoplasm of bone - 10000 days                                                                                                                                                                                                                            \n",
       "156391  2211374 - procedure - Radiologic examination, ribs, bilateral; 3 views - 10000 days                                                                                                                                                                                                              \n",
       "144897  78706 - condition - Benign neoplasm of urinary system - 730 days                                                                                                                                                                                                                                 \n",
       "173044  4155034 - condition - Laceration of forearm - 10000 days                                                                                                                                                                                                                                         \n",
       "100326  4168700 - condition - Localized enlarged lymph nodes - 365 days                                                                                                                                                                                                                                  \n",
       "167861  378419 - condition - Alzheimer's disease - 10000 days                                                                                                                                                                                                                                            \n",
       "169200  40171499 - drug - 24 HR Verapamil hydrochloride 120 MG Extended Release Oral Capsule - 10000 days                                                                                                                                                                                                \n",
       "177860  437366 - condition - Contracture of tendon sheath - 10000 days                                                                                                                                                                                                                                   \n",
       "\n",
       "        feature_weight  \n",
       "131710  0.441087        \n",
       "120087  0.444479        \n",
       "63691   0.457767        \n",
       "77428   0.464019        \n",
       "136610  0.464407        \n",
       "156125  0.473012        \n",
       "136643  0.493533        \n",
       "147484  0.509225        \n",
       "139214  0.532505        \n",
       "175240  0.549499        \n",
       "120766  0.555944        \n",
       "178096  0.559293        \n",
       "181258  0.565785        \n",
       "156391  0.567901        \n",
       "144897  0.609550        \n",
       "173044  0.645616        \n",
       "100326  0.663873        \n",
       "167861  0.667106        \n",
       "169200  0.709910        \n",
       "177860  0.734703        "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_weights.tail(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_name</th>\n",
       "      <th>feature_weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>84104</th>\n",
       "      <td>2213282 - procedure - Level III - Surgical pathology, gross and microscopic examination Abortion, induced Abscess Aneurysm - arterial/ventricular Anus, tag Appendix, other than incidental Artery, atheromatous plaque Bartholin's gland cyst Bone fragment(s), other than pathologi - 365 days</td>\n",
       "      <td>-0.869867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177139</th>\n",
       "      <td>43528020 - procedure - Nerve conduction studies; 9-10 studies - 10000 days</td>\n",
       "      <td>-0.651872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118438</th>\n",
       "      <td>2109194 - procedure - Colonoscopy, flexible; diagnostic, including collection of specimen(s) by brushing or washing, when performed (separate procedure) - 730 days</td>\n",
       "      <td>-0.552424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48359</th>\n",
       "      <td>2514405 - procedure - Initial hospital care, per day, for the evaluation and management of a patient, which requires these 3 key components: A comprehensive history; A comprehensive examination; and Medical decision making of moderate complexity. Counseling and/or coordination - 180 days</td>\n",
       "      <td>-0.550425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177317</th>\n",
       "      <td>43533326 - procedure - Carrying, moving and handling objects, projected goal status, at therapy episode outset, at reporting intervals, and at discharge or to end reporting - 10000 days</td>\n",
       "      <td>-0.505235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173429</th>\n",
       "      <td>4184592 - procedure - Aortography - 10000 days</td>\n",
       "      <td>-0.501241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157738</th>\n",
       "      <td>2514528 - procedure - Periodic comprehensive preventive medicine reevaluation and management of an individual including an age and gender appropriate history, examination, counseling/anticipatory guidance/risk factor reduction interventions, and the ordering of laboratory/diag - 10000 days</td>\n",
       "      <td>-0.478271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149786</th>\n",
       "      <td>19135223 - drug - Hydrochlorothiazide 25 MG / Olmesartan medoxomil 40 MG Oral Tablet [Benicar HCT] - 10000 days</td>\n",
       "      <td>-0.463479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84618</th>\n",
       "      <td>2314215 - procedure - Therapeutic, prophylactic, or diagnostic injection (specify substance or drug); intravenous push, single or initial substance/drug - 365 days</td>\n",
       "      <td>-0.461951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175126</th>\n",
       "      <td>42800464 - drug - 0.5 ML Bordetella pertussis filamentous hemagglutinin vaccine, inactivated 0.01 MG/ML / Bordetella pertussis fimbriae 2/3 vaccine, inactivated 0.01 MG/ML / Bordetella pertussis pertactin vaccine, inactivated 0.006 MG/ML / Bordetella pertussis toxoid vacci - 10000 days</td>\n",
       "      <td>-0.450040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129984</th>\n",
       "      <td>317893 - condition - Paroxysmal supraventricular tachycardia - 730 days</td>\n",
       "      <td>-0.449737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141673</th>\n",
       "      <td>438430 - condition - Squamous blepharitis - 730 days</td>\n",
       "      <td>-0.442513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181637</th>\n",
       "      <td>81878 - condition - Benign paroxysmal positional vertigo - 10000 days</td>\n",
       "      <td>-0.438322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144888</th>\n",
       "      <td>78619 - condition - Contusion of knee - 730 days</td>\n",
       "      <td>-0.422702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117560</th>\n",
       "      <td>2106988 - procedure - Insertion of new or replacement of permanent pacemaker with transvenous electrode(s); atrial and ventricular - 730 days</td>\n",
       "      <td>-0.412711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133591</th>\n",
       "      <td>40225811 - drug - 5 ML Lidocaine Hydrochloride 20 MG/ML Prefilled Syringe - 730 days</td>\n",
       "      <td>-0.390625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157975</th>\n",
       "      <td>260427 - condition - Common cold - 10000 days</td>\n",
       "      <td>-0.389249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58195</th>\n",
       "      <td>374375 - condition - Impacted cerumen - 180 days</td>\n",
       "      <td>-0.383727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120868</th>\n",
       "      <td>2313972 - procedure - Physician or other qualified health care professional services for outpatient cardiac rehabilitation; with continuous ECG monitoring (per session) - 730 days</td>\n",
       "      <td>-0.378724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141739</th>\n",
       "      <td>438725 - condition - Disorder of magnesium metabolism - 730 days</td>\n",
       "      <td>-0.377760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                              feature_name  \\\n",
       "84104   2213282 - procedure - Level III - Surgical pathology, gross and microscopic examination Abortion, induced Abscess Aneurysm - arterial/ventricular Anus, tag Appendix, other than incidental Artery, atheromatous plaque Bartholin's gland cyst Bone fragment(s), other than pathologi - 365 days     \n",
       "177139  43528020 - procedure - Nerve conduction studies; 9-10 studies - 10000 days                                                                                                                                                                                                                           \n",
       "118438  2109194 - procedure - Colonoscopy, flexible; diagnostic, including collection of specimen(s) by brushing or washing, when performed (separate procedure) - 730 days                                                                                                                                  \n",
       "48359   2514405 - procedure - Initial hospital care, per day, for the evaluation and management of a patient, which requires these 3 key components: A comprehensive history; A comprehensive examination; and Medical decision making of moderate complexity. Counseling and/or coordination - 180 days     \n",
       "177317  43533326 - procedure - Carrying, moving and handling objects, projected goal status, at therapy episode outset, at reporting intervals, and at discharge or to end reporting - 10000 days                                                                                                            \n",
       "173429  4184592 - procedure - Aortography - 10000 days                                                                                                                                                                                                                                                       \n",
       "157738  2514528 - procedure - Periodic comprehensive preventive medicine reevaluation and management of an individual including an age and gender appropriate history, examination, counseling/anticipatory guidance/risk factor reduction interventions, and the ordering of laboratory/diag - 10000 days   \n",
       "149786  19135223 - drug - Hydrochlorothiazide 25 MG / Olmesartan medoxomil 40 MG Oral Tablet [Benicar HCT] - 10000 days                                                                                                                                                                                      \n",
       "84618   2314215 - procedure - Therapeutic, prophylactic, or diagnostic injection (specify substance or drug); intravenous push, single or initial substance/drug - 365 days                                                                                                                                  \n",
       "175126  42800464 - drug - 0.5 ML Bordetella pertussis filamentous hemagglutinin vaccine, inactivated 0.01 MG/ML / Bordetella pertussis fimbriae 2/3 vaccine, inactivated 0.01 MG/ML / Bordetella pertussis pertactin vaccine, inactivated 0.006 MG/ML / Bordetella pertussis toxoid vacci - 10000 days       \n",
       "129984  317893 - condition - Paroxysmal supraventricular tachycardia - 730 days                                                                                                                                                                                                                              \n",
       "141673  438430 - condition - Squamous blepharitis - 730 days                                                                                                                                                                                                                                                 \n",
       "181637  81878 - condition - Benign paroxysmal positional vertigo - 10000 days                                                                                                                                                                                                                                \n",
       "144888  78619 - condition - Contusion of knee - 730 days                                                                                                                                                                                                                                                     \n",
       "117560  2106988 - procedure - Insertion of new or replacement of permanent pacemaker with transvenous electrode(s); atrial and ventricular - 730 days                                                                                                                                                        \n",
       "133591  40225811 - drug - 5 ML Lidocaine Hydrochloride 20 MG/ML Prefilled Syringe - 730 days                                                                                                                                                                                                                 \n",
       "157975  260427 - condition - Common cold - 10000 days                                                                                                                                                                                                                                                        \n",
       "58195   374375 - condition - Impacted cerumen - 180 days                                                                                                                                                                                                                                                     \n",
       "120868  2313972 - procedure - Physician or other qualified health care professional services for outpatient cardiac rehabilitation; with continuous ECG monitoring (per session) - 730 days                                                                                                                  \n",
       "141739  438725 - condition - Disorder of magnesium metabolism - 730 days                                                                                                                                                                                                                                     \n",
       "\n",
       "        feature_weight  \n",
       "84104  -0.869867        \n",
       "177139 -0.651872        \n",
       "118438 -0.552424        \n",
       "48359  -0.550425        \n",
       "177317 -0.505235        \n",
       "173429 -0.501241        \n",
       "157738 -0.478271        \n",
       "149786 -0.463479        \n",
       "84618  -0.461951        \n",
       "175126 -0.450040        \n",
       "129984 -0.449737        \n",
       "141673 -0.442513        \n",
       "181637 -0.438322        \n",
       "144888 -0.422702        \n",
       "117560 -0.412711        \n",
       "133591 -0.390625        \n",
       "157975 -0.389249        \n",
       "58195  -0.383727        \n",
       "120868 -0.378724        \n",
       "141739 -0.377760        "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_weights.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nonzero procedure feature weights : 488\n",
      "Number of nonzero condition feature weights : 744\n",
      "Number of nonzero drug feature weights : 321\n"
     ]
    }
   ],
   "source": [
    "for category in ['procedure', 'condition', 'drug', 'specialty']:\n",
    "    print('Number of nonzero {} feature weights : {}'.format(\n",
    "        category,\n",
    "        sum(feature_weights.loc[\n",
    "            [i for i,j in enumerate(feature_names) if '- ' + category + ' -' in j]\n",
    "        ]['feature_weight'] != 0)\n",
    "    ))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}