ddsm_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The results for selected model training runs are included below. The full logs are in \"model notes.xlsx\".\n",
    "\n",
    "Model 1.0.0.35b.96 had the best performance in scoring high accuracy and recall and generalizing well to the MIAS data. We would guess that this is because it was trained on Dataset 9 which did not resize the raw images depending on the size of the ROI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ddsm_results = pd.read_csv(\"ddsm_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.0288</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1759</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0.0.29g.6</td>\n",
       "      <td>B</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.8299</td>\n",
       "      <td>0.0477</td>\n",
       "      <td>0.8224</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0.0.46b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8242</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.8811</td>\n",
       "      <td>0.0157</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>vgg_16.3.06b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8138</td>\n",
       "      <td>0.9884</td>\n",
       "      <td>0.4357</td>\n",
       "      <td>0.9797</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0.0.46b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.5434</td>\n",
       "      <td>0.7549</td>\n",
       "      <td>0.0911</td>\n",
       "      <td>0.8136</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>inception_v4.05b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.1828</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1066</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0.0.46l.6</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.1800</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1690</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>B</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.1800</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1690</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.1139</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0325</td>\n",
       "      <td>0.9741</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "0          1.0.0.29b.8              B        8          2         Scratch   \n",
       "14       1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "17        1.0.0.35b.98              B        9          6         Scratch   \n",
       "3         1.0.0.29l.14              L        6          2         Scratch   \n",
       "1          1.0.0.29l.8              L        8          2         Scratch   \n",
       "4       vgg_16.3.04b.9              B        9          2         Scratch   \n",
       "6     vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "10       1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "15       1.0.0.28.2b.9              B        9          2         Scratch   \n",
       "16        1.0.0.35b.94              B        9          4         Scratch   \n",
       "5       vgg_16.3.04l.6              L        6          2         Scratch   \n",
       "2          1.0.0.29g.6              B        6          2         Scratch   \n",
       "7          1.0.0.46b.9              B        9          2         Scratch   \n",
       "18      vgg_16.3.06b.9              B        9          6         Scratch   \n",
       "8          1.0.0.46b.8              B        8          2         Scratch   \n",
       "13  inception_v4.05b.9              B        9          2         Scratch   \n",
       "12         1.0.0.46l.6              L        6          2         Scratch   \n",
       "11         1.0.0.46b.6              B        6          2         Scratch   \n",
       "9          1.0.0.46l.8              L        8          2         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "3     All      40    0.8376         0.9142       0.9353    0.1665   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "5     All      20    0.8376         0.8333       0.0288    0.4783   \n",
       "2     All      35    0.8376         0.8299       0.0477    0.8224   \n",
       "7     All      35    0.8333         0.8242       0.0000    0.8811   \n",
       "18    All      30    0.8333         0.8138       0.9884    0.4357   \n",
       "8     All      30    0.8333         0.5434       0.7549    0.0911   \n",
       "13    All      20    0.8333         0.1828       1.0000    0.1066   \n",
       "12    All      20    0.8376         0.1800       1.0000    0.1690   \n",
       "11    All      20    0.8376         0.1800       1.0000    0.1690   \n",
       "9     All      20    0.8333         0.1139       1.0000    0.0325   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "0        0.9831                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "17       0.8173                     Tuning hyperparameters  \n",
       "3        0.9663                                        NaN  \n",
       "1        0.9781                           Unstable results  \n",
       "4        0.6517                                        NaN  \n",
       "6        0.6553                                        NaN  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "15       0.5342                                        NaN  \n",
       "16       0.8870                                        NaN  \n",
       "5        0.1759                                        NaN  \n",
       "2        0.0144                                        NaN  \n",
       "7        0.0157                                        NaN  \n",
       "18       0.9797                                        NaN  \n",
       "8        0.8136                                        NaN  \n",
       "13       1.0000                                        NaN  \n",
       "12       1.0000                                        NaN  \n",
       "11       1.0000                                        NaN  \n",
       "9        0.9741                                        NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Results sorted by best accuracy\n",
    "ddsm_results.sort_values(\"Test Accuracy\", inplace=True, ascending=False)\n",
    "ddsm_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I am skeptical of the results for models 1.0.0.29 as we were not able to replicate the results in future training runs. Models 1.0.0.29 and 1.0.0.46 also did not score well in both accuracy and recall on the MIAS data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "0        1.0.0.29b.8              B        8          2         Scratch   \n",
       "14     1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "17      1.0.0.35b.98              B        9          6         Scratch   \n",
       "3       1.0.0.29l.14              L        6          2         Scratch   \n",
       "1        1.0.0.29l.8              L        8          2         Scratch   \n",
       "4     vgg_16.3.04b.9              B        9          2         Scratch   \n",
       "6   vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "10     1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "15     1.0.0.28.2b.9              B        9          2         Scratch   \n",
       "16      1.0.0.35b.94              B        9          4         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "3     All      40    0.8376         0.9142       0.9353    0.1665   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "0        0.9831                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "17       0.8173                     Tuning hyperparameters  \n",
       "3        0.9663                                        NaN  \n",
       "1        0.9781                           Unstable results  \n",
       "4        0.6517                                        NaN  \n",
       "6        0.6553                                        NaN  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "15       0.5342                                        NaN  \n",
       "16       0.8870                                        NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove any rows where the accuracy is below the baseline\n",
    "ddsm_results_filtered = ddsm_results[ddsm_results[\"Test Accuracy\"] > ddsm_results[\"Baseline\"]]\n",
    "ddsm_results_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "0        1.0.0.29b.8              B        8          2         Scratch   \n",
       "14     1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "10     1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "3       1.0.0.29l.14              L        6          2         Scratch   \n",
       "1        1.0.0.29l.8              L        8          2         Scratch   \n",
       "17      1.0.0.35b.98              B        9          6         Scratch   \n",
       "16      1.0.0.35b.94              B        9          4         Scratch   \n",
       "4     vgg_16.3.04b.9              B        9          2         Scratch   \n",
       "6   vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "15     1.0.0.28.2b.9              B        9          2         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "3     All      40    0.8376         0.9142       0.9353    0.1665   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "0        0.9831                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "3        0.9663                                        NaN  \n",
       "1        0.9781                           Unstable results  \n",
       "17       0.8173                     Tuning hyperparameters  \n",
       "16       0.8870                                        NaN  \n",
       "4        0.6517                                        NaN  \n",
       "6        0.6553                                        NaN  \n",
       "15       0.5342                                        NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Results sorted by best recall\n",
    "ddsm_results_filtered.sort_values(\"Test Recall\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "4     vgg_16.3.04b.9              B        9          2         Scratch   \n",
       "15     1.0.0.28.2b.9              B        9          2         Scratch   \n",
       "17      1.0.0.35b.98              B        9          6         Scratch   \n",
       "16      1.0.0.35b.94              B        9          4         Scratch   \n",
       "6   vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "14     1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "10     1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "0        1.0.0.29b.8              B        8          2         Scratch   \n",
       "3       1.0.0.29l.14              L        6          2         Scratch   \n",
       "1        1.0.0.29l.8              L        8          2         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "3     All      40    0.8376         0.9142       0.9353    0.1665   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "4        0.6517                                        NaN  \n",
       "15       0.5342                                        NaN  \n",
       "17       0.8173                     Tuning hyperparameters  \n",
       "16       0.8870                                        NaN  \n",
       "6        0.6553                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "0        0.9831                                        NaN  \n",
       "3        0.9663                                        NaN  \n",
       "1        0.9781                           Unstable results  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sort by best accuracy on MIAS data\n",
    "ddsm_results_filtered.sort_values(\"MIAS Acc\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "0        1.0.0.29b.8              B        8          2         Scratch   \n",
       "14     1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "1        1.0.0.29l.8              L        8          2         Scratch   \n",
       "10     1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "3       1.0.0.29l.14              L        6          2         Scratch   \n",
       "16      1.0.0.35b.94              B        9          4         Scratch   \n",
       "17      1.0.0.35b.98              B        9          6         Scratch   \n",
       "6   vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "4     vgg_16.3.04b.9              B        9          2         Scratch   \n",
       "15     1.0.0.28.2b.9              B        9          2         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "3     All      40    0.8376         0.9142       0.9353    0.1665   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "0        0.9831                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "1        0.9781                           Unstable results  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "3        0.9663                                        NaN  \n",
       "16       0.8870                                        NaN  \n",
       "17       0.8173                     Tuning hyperparameters  \n",
       "6        0.6553                                        NaN  \n",
       "4        0.6517                                        NaN  \n",
       "15       0.5342                                        NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sorted by highest MIAS recall\n",
    "ddsm_results_filtered.sort_values(\"MIAS Recall\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results by Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset 9\n",
    "\n",
    "We focused on this dataset as the others resized the images based on the size of the ROI. This dataset did not, making models trained on it generalize the best to the MIAS data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0.0.35b.98</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9571</td>\n",
       "      <td>0.9012</td>\n",
       "      <td>0.8802</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>Tuning hyperparameters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vgg_16.3.04b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8881</td>\n",
       "      <td>0.3589</td>\n",
       "      <td>0.9314</td>\n",
       "      <td>0.6517</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0.0.28.2b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8675</td>\n",
       "      <td>0.2756</td>\n",
       "      <td>0.9165</td>\n",
       "      <td>0.5342</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0.0.35b.94</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8502</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.7328</td>\n",
       "      <td>0.8870</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0.0.46b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8242</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.8811</td>\n",
       "      <td>0.0157</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>vgg_16.3.06b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8138</td>\n",
       "      <td>0.9884</td>\n",
       "      <td>0.4357</td>\n",
       "      <td>0.9797</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>inception_v4.05b.9</td>\n",
       "      <td>B</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.1828</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1066</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model Classification  Dataset  XE Weight Initialization  \\\n",
       "17        1.0.0.35b.98              B        9          6        Scratch   \n",
       "4       vgg_16.3.04b.9              B        9          2        Scratch   \n",
       "15       1.0.0.28.2b.9              B        9          2        Scratch   \n",
       "16        1.0.0.35b.94              B        9          4        Scratch   \n",
       "7          1.0.0.46b.9              B        9          2        Scratch   \n",
       "18      vgg_16.3.06b.9              B        9          6        Scratch   \n",
       "13  inception_v4.05b.9              B        9          2        Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "17    All      35    0.8333         0.9571       0.9012    0.8802   \n",
       "4     All      30    0.8333         0.8881       0.3589    0.9314   \n",
       "15    All      20    0.8333         0.8675       0.2756    0.9165   \n",
       "16    All      30    0.8333         0.8502       0.7438    0.7328   \n",
       "7     All      35    0.8333         0.8242       0.0000    0.8811   \n",
       "18    All      30    0.8333         0.8138       0.9884    0.4357   \n",
       "13    All      20    0.8333         0.1828       1.0000    0.1066   \n",
       "\n",
       "    MIAS Recall                   Notes  \n",
       "17       0.8173  Tuning hyperparameters  \n",
       "4        0.6517                     NaN  \n",
       "15       0.5342                     NaN  \n",
       "16       0.8870                     NaN  \n",
       "7        0.0157                     NaN  \n",
       "18       0.9797                     NaN  \n",
       "13       1.0000                     NaN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddsm_results.loc[ddsm_results.Dataset == 9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model 1.0.0.35b.96 is the model which we feel has the best performance on the test dataset while generalizing well to the MIAS dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset 8\n",
    "\n",
    "While some models performed very well on Dataset 8, those models did not perform well on the MIAS dataset. They all traded off accuracy and recall."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0.0.29b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9930</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2189</td>\n",
       "      <td>0.9831</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0.0.46b.8.4</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>All</td>\n",
       "      <td>10</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.9896</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.2746</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0.0.29l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.1462</td>\n",
       "      <td>0.9781</td>\n",
       "      <td>Unstable results</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vgg_16.3.04b.8.1</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8747</td>\n",
       "      <td>0.2951</td>\n",
       "      <td>0.3986</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0.0.46b.8.2</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>FC</td>\n",
       "      <td>5</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.8711</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.2535</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>Retrained only fc layers from 1.0.0.46l.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0.0.46b.8</td>\n",
       "      <td>B</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>30</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.5434</td>\n",
       "      <td>0.7549</td>\n",
       "      <td>0.0911</td>\n",
       "      <td>0.8136</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0.0.46l.8</td>\n",
       "      <td>L</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.1139</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0325</td>\n",
       "      <td>0.9741</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model Classification  Dataset  XE Weight  Initialization  \\\n",
       "0        1.0.0.29b.8              B        8          2         Scratch   \n",
       "14     1.0.0.46b.8.4              B        8          2     1.0.0.46b.6   \n",
       "1        1.0.0.29l.8              L        8          2         Scratch   \n",
       "6   vgg_16.3.04b.8.1              B        8          2  vgg_16.3.04l.6   \n",
       "10     1.0.0.46b.8.2              B        8          2     1.0.0.46l.8   \n",
       "8        1.0.0.46b.8              B        8          2         Scratch   \n",
       "9        1.0.0.46l.8              L        8          2         Scratch   \n",
       "\n",
       "   Layers  Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  \\\n",
       "0     All      30    0.8333         0.9930       1.0000    0.2189   \n",
       "14    All      10    0.8333         0.9896       0.9776    0.2746   \n",
       "1     All      35    0.8333         0.8890       0.9092    0.1462   \n",
       "6      FC       5    0.8333         0.8747       0.2951    0.3986   \n",
       "10     FC       5    0.8333         0.8711       0.9434    0.2535   \n",
       "8     All      30    0.8333         0.5434       0.7549    0.0911   \n",
       "9     All      20    0.8333         0.1139       1.0000    0.0325   \n",
       "\n",
       "    MIAS Recall                                      Notes  \n",
       "0        0.9831                                        NaN  \n",
       "14       0.9811                                        NaN  \n",
       "1        0.9781                           Unstable results  \n",
       "6        0.6553                                        NaN  \n",
       "10       0.9749  Retrained only fc layers from 1.0.0.46l.8  \n",
       "8        0.8136                                        NaN  \n",
       "9        0.9741                                        NaN  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddsm_results.loc[ddsm_results.Dataset == 8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset 6\n",
    "\n",
    "We stopped training on Dataset 6 after Dataset 9 was created. These results are just included as a reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Classification</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>XE Weight</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>Layers</th>\n",
       "      <th>Epochs</th>\n",
       "      <th>Baseline</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test Recall</th>\n",
       "      <th>MIAS Acc</th>\n",
       "      <th>MIAS Recall</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0.0.29l.14</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>40</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.9142</td>\n",
       "      <td>0.9353</td>\n",
       "      <td>0.1665</td>\n",
       "      <td>0.9663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>vgg_16.3.04l.6</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.8333</td>\n",
       "      <td>0.0288</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1759</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0.0.29g.6</td>\n",
       "      <td>B</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>35</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.8299</td>\n",
       "      <td>0.0477</td>\n",
       "      <td>0.8224</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0.0.46l.6</td>\n",
       "      <td>L</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.1800</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1690</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0.0.46b.6</td>\n",
       "      <td>B</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Scratch</td>\n",
       "      <td>All</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8376</td>\n",
       "      <td>0.1800</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.1690</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Model Classification  Dataset  XE Weight Initialization Layers  \\\n",
       "3     1.0.0.29l.14              L        6          2        Scratch    All   \n",
       "5   vgg_16.3.04l.6              L        6          2        Scratch    All   \n",
       "2      1.0.0.29g.6              B        6          2        Scratch    All   \n",
       "12     1.0.0.46l.6              L        6          2        Scratch    All   \n",
       "11     1.0.0.46b.6              B        6          2        Scratch    All   \n",
       "\n",
       "    Epochs  Baseline  Test Accuracy  Test Recall  MIAS Acc  MIAS Recall Notes  \n",
       "3       40    0.8376         0.9142       0.9353    0.1665       0.9663   NaN  \n",
       "5       20    0.8376         0.8333       0.0288    0.4783       0.1759   NaN  \n",
       "2       35    0.8376         0.8299       0.0477    0.8224       0.0144   NaN  \n",
       "12      20    0.8376         0.1800       1.0000    0.1690       1.0000   NaN  \n",
       "11      20    0.8376         0.1800       1.0000    0.1690       1.0000   NaN  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddsm_results.loc[ddsm_results.Dataset == 6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Effect of Contrast and Threshold on Accuracy and Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results of trained model 2.0.0.35b evaluated with different contrast and threshold levels\n",
    "aug_results = pd.read_csv(\"eval_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# contrast\n",
    "contrast_1 = aug_results[aug_results.Contrast == 1]\n",
    "contrast_15  = aug_results[aug_results.Contrast == 1.5]\n",
    "contrast_2  = aug_results[aug_results.Contrast == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXJwkkbAqSgEIQUHFhDW2EWjdwq9Iqbr2Ke2vlWmu9emurvbVei13Utlj91S62emvVW1xubWlrV8W12hIroOACCmJAMYAIsif5/P74nplMhkkyCTmZSfJ+Ph7nMWf5zplPTpLvZ873e873mLsjIiICUJDrAEREJH8oKYiISJKSgoiIJCkpiIhIkpKCiIgkKSmIiEiSkoLEysxuMLP7OuBzRpiZm1lRG947xcyqm9n+CzP75u5FKNI5tPofSCSVmX2Ystgb2A7URcv/3vERdV5mZsAbwDZ3H53reKR70pmC7BZ375uYgJXAySnr7m/NvtryLb+LOQoYBOxnZod25Afr2EuCkoJ0hJ5m9ksz22Rmi82sMrHBzFaY2TVmtgjYbGZFZjbEzP7PzGrMbLmZXZFSfpKZVZnZRjNbY2az0z7rXDNbaWZrzexrKe8rNrMfmNnqaPqBmRVnCtbMJprZv6J4HwBKmihXbGYbzGxsyroyM9tqZoPMrNTMfh+VWW9mT5tZc/9zFwK/BR6N5lM/ay8z+58o9vfN7Dcp26ab2YLomLxhZiemHNvjUsolm/JSmtsuNrOVwOPR+ofM7F0z+8DMnjKzMSnv72Vm3zezt6Ltz0Tr/mBmX0yLd5GZndrMzyp5SklBOsIpwBygPzAX+GHa9hnAJ6Pt9cDvgIXAUOBY4Eoz+0RU9jbgNnffA9gfeDBtX0cAB0Xvu97MDonWfw34GFABTAAmAdelB2pmPYHfAPcCewEPAWdk+qHcfTvw6yj+hH8DnnT394AvAdVAGTAY+C8g47gyZtYbOBO4P5rOjmJJuJfQPDeGcDZxa/S+ScAvgS8Tjt9RwIpMn9GEo4FDgMTx/SMwKvqMf0WxJHwP+CjwccKx+Qrh93UPcF7KzzKB8Lt7tBVxSL5wd02a2mUiVEbHpa27AfhbyvJoYGvaez6bsjwZWJm2j68C/xPNPwV8AyhNKzOCUOGWp6z7J3B2NP8GMC1l2yeAFdH8FKA6mj8KWA1YStm/A99s4mc+DngzZflZ4IJofhbhm/8BWRy784AaQj9fMbABOC3atg+h8h2Q4X0/BW7N5vcR/S7uSzte+zUTU/+ozJ6EL5BbgQkZyhUD64FR0fL3gB/l+u9RU9smnSlIR3g3ZX4LUJLWhv12yvxwYEjU5LLBzDYQvmEPjrZfDBwIvGpm883sUy18Vt9ofgjwVsq2t6J16YYAqzyq3VLKNuVxoJeZTTaz4YQzkUeibd8FlgF/MbM3zezaZvZzIfCgu9d6wxlIoglpGLDe3d/P8L5hhITXVsljb2aFZnZT1AS1kYYzjtJoKsn0WVG8DwLnRc1jMwhnNtIJqXNJ8kFqBfw2sNzdR2Us6L4UmBFVPqcDD5vZwCw+YzUh4SyOlveN1qV7BxhqZpaSGPaliYrX3evN7EFCRbgG+L27b4q2bSI0IX0papufZ2bz3f2x1H2YWTlwDDDJzBJNVb0JybOUcEz2MrP+7r4hLYS3Cc1omWyO9pOwd6YfIWX+HGA64exnBeEM4X3AgLXAtuizFmbYzz2ERPAMsMXdn2siJslzOlOQfPNPYGPU+dwr+vY6NnE1jpmdZ2Zl7l5PaGKBhktgm/Mr4LqoI7gUuB7IdP/Ec0AtcEXU6X06of+hOf8LnAWcG80TxfopMzsgutR0YxRnpljPB14n9IVURNOBhP6IGe7+DqGt/0dmNsDMepjZUdF77wI+Y2bHmlmBmQ01s4OjbQsIfRM9LHTun9nCz9GPcEnxOkIy+XZiQ3S87wZmRxcCFJrZYYnO+igJ1APfR2cJnZqSguQVd68DTiZUjMsJ31B/TvjWCnAisNjC/RG3EfoMtmWx628CVcAi4CVCJ+ouN6S5+w7CGchFhG/JZxGacpqL+R+Eb+VDCJV3wijgb8CHhGTzI3d/IsMuLoy2vZs6AT+hoQnpfGAn8CrwHnBl9Nn/BD5D6Hj+AHiScEYE8HXCN/v3Cf0wyYTVhF8SmspWAUuA59O2X004dvMJfQg307gO+SUwjszJVjoJa9x0KiLSNmZ2ATDT3Y/IdSzSdjpTEJHdFl1SexlwZ65jkd2jpCAiuyW6h6SG0NHeUhOV5Dk1H4mISFJsZwpmdreZvWdmLzex3czsdjNbFt0S/5G4YhERkezEeZ/CLwjDGfyyie0nEa7OGEW4i/XH0WuzSktLfcSIEe0ToYhIN/HCCy+sdfeylsrFlhTc/SkzG9FMkenAL6MbhJ43s/5mtk90TXaTRowYQVVVVTtGKiLS9ZlZc3fmJ+Wyo3kojYc3qI7W7cLMZloYGbOqpqamQ4ITEemOcpkULMO6jL3e7n6nu1e6e2VZWYtnPyIi0ka5TArVhMG8EsrJPBaNiIh0kFwmhbnABdFVSB8DPmipP0FEROIVW0ezmf2KME59qYWHov830APA3X9CeADHNMLQwlsI47eIiEgOxXn10YwWtjvwhbg+X0REWk/DXIiISJIesiOSY/X1sH07bNvWMKUvNzUlytXXQ1ERFBaGKTGf63UFBWAZrjN0h7q6EHdiSl/OtK69ymRalxjxx2zXqbXr43rPsGFQWhrP32GCkoJ0e7W12VW82VbQrS2zY0euj0C8EgkjtQKWtvnxj+HSS+P9DCUFyUvuobLcvBk+/DDza1Pbtm5tXSVel81z25phBr16QUkJFBeH1/SptLRhvqkyqVNLZVK3FxeHGOrrQ4KrqwtTYj4f1tXVhbOG9ClxNtHUclvLtOV9ZuHvLn1K/D1muz7O90yYsHt/q9lQUpDd4h4q4Wwr7Na81tZmH0dhIfTtG6ZevRoq6ZIS6N0b9tpr9yrh5soUFWVuIuloiW/kIrtDSaGbyFR5Zzvf1LZE5d+a0deLi6FPn1B5p74OGZJ5fbavPXvmR8Us0tkpKeQZ99CksbsVdqb51lTevXrtWvn27RuaQdpScSfmi/QXJ5LX9C/aSjt3wpYtoZLdsqX5+baU27y5dR1xJSWZK9+BAzOvz2a+Tx81Q4h0V90mKbz+OixY0LrKOlPF3Zp27oTevcPUp0/j+b32gvLyxttaqrjT16nyFpH21G2Swm9/C1/5yq7rEx2R6ZX2HnvA3ntnrsxbM9+rl9q6RaTz6DZJ4cILYdq0xpV2r176pi0ikqrbJIVBg8IkIiJN09hHIiKSpKQgIiJJSgoiIpKkpCAiIklKCiIikhRrUjCzE83sNTNbZmbXZtg+3MweM7NFZvaEmZXHGY+IiDQvtqRgZoXAHcBJwGhghpmNTiv2PeCX7j4emAV8J654RESkZXGeKUwClrn7m+6+A5gDTE8rMxp4LJqfl2G7iIh0oDiTwlDg7ZTl6mhdqoXAGdH8aUA/MxuYviMzm2lmVWZWVVNTE0uwIiISb1LINOJP+uDNVwNHm9mLwNHAKmCXIefc/U53r3T3yrKysvaPVEREgHiHuagGhqUslwOrUwu4+2rgdAAz6wuc4e4fxBiTiIg0I84zhfnAKDMbaWY9gbOBuakFzKzUzBIxfBW4O8Z4RESkBbElBXevBS4H/gy8Ajzo7ovNbJaZnRIVmwK8ZmavA4OBb8UVj4iItMy8Nc9ozAOVlZVeVVWV6zBERDoVM3vB3StbKqc7mkVEJElJQUREkpQUREQkSUlBRESSlBRERCRJSUFERJKUFEREJElJQUREkpQUREQkSUlBRESSlBRERCRJSUFERJKUFEREJElJQUREkpQUREQkSUlBRESSlBRERCRJSUFERJJiTQpmdqKZvWZmy8zs2gzb9zWzeWb2opktMrNpccYjIiLNiy0pmFkhcAdwEjAamGFmo9OKXQc86O4TgbOBH8UVj4iItCzOM4VJwDJ3f9PddwBzgOlpZRzYI5rfE1gdYzwiItKCOJPCUODtlOXqaF2qG4DzzKwaeBT4YqYdmdlMM6sys6qampo4YhUREeJNCpZhnactzwB+4e7lwDTgXjPbJSZ3v9PdK929sqysLIZQRUQE4k0K1cCwlOVydm0euhh4EMDdnwNKgNIYYxIRkWbEmRTmA6PMbKSZ9SR0JM9NK7MSOBbAzA4hJAW1D4mI5EhsScHda4HLgT8DrxCuMlpsZrPM7JSo2JeAS8xsIfAr4CJ3T29iEhGRDlIU587d/VFCB3LquutT5pcAh8cZg4iIZE93NIuISJKSgoiIJCkpiIhIkpKCiIgkKSmIiEiSkoKIiCQpKYiISJKSgoiIJCkpiIhIkpKCiIgkKSmIiEiSkoKIiCQpKYiISJKSgoiIJCkpiIhIkpKCiIgkKSmIiEiSkoKIiCTFmhTM7EQze83MlpnZtRm232pmC6LpdTPbEGc8IiLSvNie0WxmhcAdwPFANTDfzOZGz2UGwN2vSin/RWBiXPGIiEjL4jxTmAQsc/c33X0HMAeY3kz5GcCvYoxHRERaEGdSGAq8nbJcHa3bhZkNB0YCjzexfaaZVZlZVU1NTbsHKiIiQZxJwTKs8ybKng087O51mTa6+53uXunulWVlZe0WoIiINBZnUqgGhqUslwOrmyh7Nmo6EhHJudg6moH5wCgzGwmsIlT856QXMrODgAHAc239oJ07d1JdXc22bdvaugvZTSUlJZSXl9OjR49chyIiuyG2pODutWZ2OfBnoBC4290Xm9ksoMrd50ZFZwBz3L2ppqUWVVdX069fP0aMGIFZplYriZO7s27dOqqrqxk5cmSuwxGR3RDnmQLu/ijwaNq669OWb9jdz9m2bZsSQg6ZGQMHDkQXAYh0fl3mjmYlhNzS8RfpGlpMCmZ2uZkN6IhgREQkt7I5U9ibcDfyg9GwFfpKmMG7777L2Wefzf7778/o0aOZNm0ar7/+epv29YMf/IAtW7a0S1xPPPEEf//73zNue/XVVznssMMoLi7me9/7XpP7WL58OZMnT2bUqFGcddZZ7Nixo11iE5H802JScPfrgFHAXcBFwFIz+7aZ7R9zbJ2Gu3PaaacxZcoU3njjDZYsWcK3v/1t1qxZ06b9NZcU6uoy3srRpOaSwl577cXtt9/O1Vdf3ew+rrnmGq666iqWLl3KgAEDuOuuu1oVg4h0Hln1KURXBr0bTbWES0gfNrNbYoyt05g3bx49evTg0ksvTa6rqKjgyCOPxN358pe/zNixYxk3bhwPPPAAECrrKVOmcOaZZ3LwwQdz7rnn4u7cfvvtrF69mqlTpzJ16lQA+vbty/XXX8/kyZN57rnnmDVrFoceeihjx45l5syZJC7cuv322xk9ejTjx4/n7LPPZsWKFfzkJz/h1ltvpaKigqeffrpR3IMGDeLQQw9t9jJSd+fxxx/nzDPPBODCCy/kN7/5TbsePxHJHy1efWRmVwAXAmuBnwNfdvedZlYALAW+Em+IrXPllbBgQfvus6ICfvCDpre//PLLfPSjH8247de//jULFixg4cKFrF27lkMPPZSjjjoKgBdffJHFixczZMgQDj/8cJ599lmuuOIKZs+ezbx58ygtLQVg8+bNjB07llmzZgEwevRorr8+XMR1/vnn8/vf/56TTz6Zm266ieXLl1NcXMyGDRvo378/l156KX379m3xbKAp69ato3///hQVhT+V8vJyVq1a1aZ9iUj+y+ZMoRQ43d0/4e4PuftOAHevBz4Va3RdwDPPPMOMGTMoLCxk8ODBHH300cyfPx+ASZMmUV5eTkFBARUVFaxYsSLjPgoLCznjjDOSy/PmzWPy5MmMGzeOxx9/nMWLFwMwfvx4zj33XO67775kJb67Mt0+om4lka4rm5rjUWB9YsHM+gGj3f0f7v5KbJG1UXPf6OMyZswYHn744Yzbmrsnr7i4ODlfWFhIbW1txnIlJSUUFhYC4Z6Myy67jKqqKoYNG8YNN9yQvJP7D3/4A0899RRz587lxhtvTCaL3VFaWsqGDRuora2lqKiI6upqhgwZstv7FZH8lM2Zwo+BD1OWN0frJHLMMcewfft2fvaznyXXzZ8/nyeffJKjjjqKBx54gLq6OmpqanjqqaeYNGlSs/vr168fmzZtyrgtkQBKS0v58MMPk8movr6et99+m6lTp3LLLbewYcMGPvzww2b3lQ0zY+rUqcnPueeee5g+vbkR0EWkM8smKVjqEBRRs1Gsd0J3NmbGI488wl//+lf2339/xowZww033MCQIUM47bTTGD9+PBMmTOCYY47hlltuYe+99252fzNnzuSkk05KdjSn6t+/P5dccgnjxo3j1FNP5dBDDwXCVUnnnXce48aNY+LEiVx11VX079+fk08+mUceeSRjR/O7775LeXk5s2fP5pvf/Cbl5eVs3LgRgGnTprF6dRi/8Oabb2b27NkccMABrFu3josvvrg9DpuItMb770PbRwPKmrU05JCZ/Rp4goazg8uAqe5+aryhZVZZWelVVVWN1r3yyisccsghuQhHUuj3IBKD+nr4n/+Ba64J7ePnndem3ZjZC+5e2VK5bM4ULgU+ThjptBqYDMxsU1QiIpK9RYvgyCPhc5+D0aNhYvxPLG6xGcjd3yMMey0iIh1h0ya44Qa47TYYMAB+8Qu44ALogCv/srlPoQS4GBgDlCTWu/tnY4xLRKT7cYdf/xr+4z9g1SqYORO+8x3Ya68OCyGb5qN7CeMffQJ4kvAEtbZfziIiIrt64w2YNg3OPBNKS+G55+CnP+3QhADZJYUD3P3rwGZ3vwf4JDAu3rBERLqJ7dvhxhth7Fh49tnQmVxVBR/7WE7CyebS0p3R6wYzG0sY/2hEbBGJiHQXf/sbXHYZLF0KZ50F3/8+DB2a05CyOVO4M3qewnXAXGAJcHOsUXVCXXno7IsuuoiRI0dSUVFBRUUFC9p7cCmR7uadd2DGDDj++NCP8Oc/w5w5OU8I0EJSiAa92+ju77v7U+6+n7sPcvefZrPz6PkLr5nZMjO7toky/2ZmS8xssZn9bxt+hpzr6kNnA3z3u99lwYIFLFiwgIqKilbFICKR2lr4f/8PDj4YHnkkXGH00ktwwgm5jiyp2aQQ3b18eVt2bGaFwB3AScBoYIaZjU4rMwr4KnC4u48BrmzLZ+VaVx46W0TayT//CZMmwRVXwGGHwcsvw3//N5SUtPzeDpRNn8Jfzexq4AHCuEcAuPv6pt8CwCRgmbu/CWBmc4DphOanhEuAO9z9/Wif77Ui9sxyMHZ2Vx46O+FrX/sas2bN4thjj+Wmm25qNJifiDTj/ffhq1+FO++EffaBBx8MVxjl6WjD2fQpfBb4AvAU8EI0VTX7jmAo8HbKcnW0LtWBwIFm9qyZPW9mJ2bakZnNNLMqM6uqqanJ4qPzR2cfOhvgO9/5Dq+++irz589n/fr13HyzupREWuQOv/wlHHQQ/Oxn4d6DV16BT386bxMCZHdH88g27jvTT50+0FIR4VGfUwj3PzxtZmPdfUNaDHcCd0IY+6jZT83B2NldeehsgH322ScZ72c+85lmO6VFBFi8OFxV9NRT4dLSv/wltDh0Ai2eKZjZBZmmLPZdDQxLWS4HVmco81t33+nuy4HXCEmiU+nKQ2cDvPPOO0BIcL/5zW8YO3bsbu1PpMvavBmuvTYkgJdeCk1Gzz7baRICZNd8dGjKdCRwA3BKFu+bD4wys5Fm1pMwftLctDK/AaYCmFkpoTnpzawizyNdfejsc889l3HjxjFu3DjWrl3Ldddd1x6HTaRr+e1vw6B1N98M558Pr70Gl1wCBdlUs/mjxaGzd3mD2Z7Ave7eYmIws2nAD4BC4G53/5aZzQKq3H2uhec6fh84EagDvuXuc5rbp4bOzl/6PUi3tGJFuKLod78LdyX/+MdwxBG5jmoX2Q6d3ZbeyC1k2cTj7o8SHueZuu76lHkH/jOaREQ6jx07YPZsmDUrnA1897uhM7mTX+KdzSipv6Ohg7iAcM/Bg3EGJSKS1554InQkv/IKnHZaGOJ62LAW39YZZHOmkHqpSS3wlrtXxxSPiEj+WrMGrr4a7rsPRoyA3/8ePvnJXEfVrrJJCiuBd9x9G4CZ9TKzEe6+ItbIRETyRV1duJLov/4rXGH0ta+F+d69cx1Zu8umW/whoD5luS5aJyLS9f3rX2FYissuC4/DXLQIvvnNLpkQILukUOTuOxIL0XzP+EISEckDH3wAX/wiHHoorFwJ998Pjz0WBrPrwrJJCjVmlrz81MymA2vjC6lz6oxDZ99///2MHz+e8ePH8/GPf5yFCxdmLLd8+XImT57MqFGjOOuss9ixY0fGciJdgjv86leh8r/jjnCG8OqrcM45eT08RXvJJilcCvyXma00s5XANcC/xxtW59JZh84eOXIkTz75JIsWLeLrX/86M2fOzFjummuu4aqrrmLp0qUMGDCAu+66q1UxiHQar70WnnFwzjlQXg7z54ehrvv3z3VkHcfds5qAvkC/bMvHNX30ox/1dEuWLNllXUd67LHH/Mgjj8y4rb6+3q+++mofM2aMjx071ufMmePu7vPmzfOjjz7azzjjDD/ooIP8nHPO8fr6er/tttu8R48ePnbsWJ8yZYq7u/fp08e//vWv+6RJk/zpp5/2b3zjG15ZWeljxozxSy65xOvr693d/bbbbvNDDjnEx40b52eddZYvX77cBw8e7EOGDPEJEyb4U0891eTPsH79eh8yZEjG+AcOHOg7d+50d/e///3vfsIJJ2TcR65/DyJttmWL+3XXuffs6b7nnu533OFeW5vrqNoV4abhFuvYbO5T+DZwi0eD1EVPYfuSu+flWAdX/ulKFrzbvkNnV+xdwQ9O7NpDZ991112cdNJJu6xft24d/fv3T466Wl5ezqpVq1o4YiKdgHsYl+jee8Nw1hs2wHnnhZvQWhiKpivLpvnoJE8ZtdTDsw+mxRdS19IZhs6eN28ed911V8YhsT3DMCjWDdpVpQtbuhSuvx723x+OPDLcc/CpT8HTT4cE0Y0TAmR3n0KhmRW7+3YI9ykAefuElea+0celMw+dvWjRIj73uc/xxz/+kYEDB+6yvbS0lA0bNlBbW0tRURHV1dUMGTKkxf2K5JW1a+GBB0Kl/49/hGEpjj0WvvGNcEdy3765jjBvZHOmcB/wmJldbGYXA38F7ok3rM6lsw6dvXLlSk4//XTuvfdeDjzwwIxlzIypU6cmP+eee+5h+vTpzR8QkXywbRs8/DBMnx6eeHb55bB1a2geevvt8IyD889XQkjTYlJw91uAbwKHEMY9+hMwPOa4OpXOOnT2rFmzWLduHZdddhkVFRVUVjYMoJg6dPbNN9/M7NmzOeCAA1i3bh0XX3zx7h4ykXjU14cH21xySWgG+vSnwxVEV14JCxeG6eqrQWe7Tcpq6GwzqwDOAf4NWA78n7v/MObYMtLQ2flLvwfJmddfD01D990XhrLu0wdOPz2cCRxzDETNr93Zbg+dbWYHEh6MMwNYBzxASCK7fn0VEeloNTUN/QT//GfoJzjuOLjxRjj1VDULtVFzHc2vAk8DJ7v7MgAzu6pDohIRyWTbNpg7NySCP/0JamthwgT43vdgxgw1C7WD5pLCGYQzhXlm9idgDpC31yK6uy6VzKFsmiFF2qS+vuFy0Ycego0bQ+V/1VWheWjcuFxH2KU0mRTc/RHgETPrA5wKXAUMNrMfA4+4+186KMYWlZSUsG7dOgYOHKjEkAPuzrp16ygpKcl1KNKVvPpqSAT33w9vvRX6Cc44IySCqVPVTxCTFu9TcPfNwP3A/Wa2F/Bp4FqgxaRgZicCtxGe0fxzd78pbftFwHeBxC2yP3T3n7fmB4Bwl211dTU1NTWtfau0k5KSEsrLy3MdhnR2770Hc+aEZFBVFfoJjj8evvWt0E/Qp0+uI+zyWvWMZndfD/w0mpplZoXAHcDxQDUw38zmuvuStKIPuPvlrYkjXY8ePRg5cuTu7EJEcmXr1sb9BHV1UFEB3/9+6CfYZ59cR9ittCoptNIkYJm7vwlgZnOA6UB6UhCR7iZxP8G994YbzDZuhKFD4UtfCs1DY8fmOsJuK86kMBR4O2W5GpicodwZZnYU8Dpwlbu/nV7AzGYCMwH23XffGEIVkQ7xyisN/QQrV4bLRhP9BFOmqJ8gD8SZFDL1+KZfovI74Ffuvt3MLiUMn3HMLm9yvxO4E8LNa+0dqIjEpL4+JIK//jXcWPbCC6Gf4IQT4DvfCUNQqJ8gr8SZFKqBYSnL5cDq1ALuvi5l8WfArsN0ikjnsWNHqPiffhqeeSYMTb1+fdg2cSLMnh36Cbr5SKT5LM6kMB8YZWYjCVcXnU0YKiPJzPZx93eixVOAV2KMR0Ta2wcfwHPPhQTw9NPhzuJo0EYOPDCMQHrEEWGI6v33z22skpXYkoK715rZ5cCfCZek3u3ui81sFuEJQHOBK6LnP9cC64GL4opHRNrB6tUNZwHPPAOLFoUmosJC+MhH4POfDwng8MNh0KBcRyttkNWAePkk04B4IhID93ADWeIs4JlnYPnysK1PHzjssHAWcMQR8LGPqW8gz+32gHgi0s3s2AH/+lfDWcAzz8C6qNtv0KBQ+V9xRXidMAF69MhtvBILJQWR7mrjRnj++YazgH/8I9xIBjBqFJxySkN/wAEHgIaQ6RaUFES6i3feaTgDePrp8MCZ+vpwiejEifDv/x6SwOGH6+qgbkxJQaQrcg8PnkntFH7jjbCtd+/QB3DddeEsYPJk6Ncvt/FK3lBSEOkKdu6EF19s3Cm8dm3YVlYWzgAuuyy8Tpyo/gBpkpKCSGezeTMsXhwuB124MLxWVcGWLWH7/vvDJz8ZzgKOOCLcL6D+AMmSkoJIvnIPzxtetKhxAli2LGyDMHbQuHFw8cUNSUCjispuUFIQyQebNsFLLzUkgMS0aVPYbhbOAMaPh/POC6/jx8OIEaGjWKSdKCmIdKT6+tDhm175v/lmQ5k99wwV/gUXhPsBxo+HMWP0IHrpEEoKInHZsGHXyv+llxra/gvQNYcWAAARfElEQVQKQnt/ZSV89rOh8p8wAYYNUx+A5IySgsjuqq0N7fyJNv/EtHJlQ5m99goV/iWXNDT9jBkDvXrlLm6RDJQURFpj3bpdO34XL24YGbSwEA4+OHT4Jir/8eNhyBB9+5dOQUlBJJOdO+G113ZNAKtTHglSVha+/X/hCw2V/yGHQHFx7uIW2U1KCiJr1jRu9lm4EJYsCYkBwo1eo0fDscc2tPuPHw+DB+c2bpEYKClI97F9e3g0ZHoCeO+9hjJDhoQK/xOfaEgABx2kO4Cl21BSkK7HPTTzpF/58+qroVMYQhPP2LHhzt/Utv/S0tzGLpJjSgrSue3YESr7BQvCtHBhmNalPP572LDwjf+UUxoq/1GjoEh//iLpYv2vMLMTgdsIj+P8ubvf1ES5M4GHgEPdXY9Vk8zefz9U+InKf8GCcOVPou2/pCQM+XDaaQ3t/uPGwYABuY1bpBOJLSmYWSFwB3A8UA3MN7O57r4krVw/4ArgH3HFIp2Me3jsY2rlv2BB4+v+Bw+Gigo44YTwWlGhb/8i7SDO/6BJwDJ3fxPAzOYA04ElaeVuBG4Bro4xFslX27bByy83rvwXLQpPBYNw1+9BB8HHPx6Gfq6oCGcBegiMSCziTApDgbdTlquByakFzGwiMMzdf29mTSYFM5sJzATYd999YwhVOsR77zWu/BcuDP0BdXVhe9++DQO+JSr/sWPDQ2FEpEPEmRQy3b7pyY1mBcCtwEUt7cjd7wTuBKisrPQWikuu1dWFYR/Sm3/eeaehTHl5qPgT7f8VFbDffhrxUyTH4kwK1cCwlOVyIOV2UPoBY4EnLNz+vzcw18xOUWdzJ/Lhh2GQt9TKP3XQt6KicOPX8cc3VP4TJsDAgbmNW0QyijMpzAdGmdlIYBVwNnBOYqO7fwAkLwo3syeAq5UQ8tSWLfDWW2GI58RlnwsWwNKlDQ986d8/VPqXXNLQ+athH0Q6ldiSgrvXmtnlwJ8Jl6Te7e6LzWwWUOXuc+P6bGkl9zDM81tvZZ5WrGh43m/CyJGh0j/33IZv//vuq0HfRDq5WK/fc/dHgUfT1l3fRNkpccbSrbmH8X2aqvDfeqvhCV8JJSUwfHiYPvKRhvkRI0Ln75575uInEZGY6aLurqC2FlatarrSX7kyjPuTas89QwW/334wdWpDpZ+Yysr0rV+kG1JS6Ay2bQsVe1OV/qpVDZd1JgweHCr3igqYPn3XSl/f9EUkAyWFfLBpU+OmnPSmnTVrGpcvKAiXdA4fDkcdtWuFv+++eqKXiLSJkkJHSHTirljRUNGnvq5f37h8cXGo2IcPh099qnGFP2IEDB2q4RxEJBaqWXaXe6jUM1X2ifkPPmj8nt69Q+U+fDhMntxQ2SdeBw3STVwikhNKCi1xh5qazJV94vXDDxu/p1+/hkr+qKMaV/jDh4cx+9WJKyJ5SEmhvj602TdX6W/d2vg9/fuHCn7UKDjuuMaV/ogRYbsqfRHphLpPUli2DJ5/ftcmnrfeCg9qSVVaGir5MWNg2rSGyl5X7ohIF9d9ksIjj8BXvhLmE5drTpwYBmRLbdoZPjyM1iki0g11n6Rw/vnhcYy6XFNEpEndJynsvbcezCIi0gJd9ygiIklKCiIikqSkICIiSUoKIiKSpKQgIiJJSgoiIpKkpCAiIkmxJgUzO9HMXjOzZWZ2bYbtl5rZS2a2wMyeMbPRccYjIiLNiy0pmFkhcAdwEjAamJGh0v9fdx/n7hXALcDsuOIREZGWxXmmMAlY5u5vuvsOYA4wPbWAu29MWewDeIzxiIhIC+Ic5mIo8HbKcjUwOb2QmX0B+E+gJ3BMph2Z2UxgJsC+++7b7oGKiEgQ55lCpgcK7HIm4O53uPv+wDXAdZl25O53unulu1eWlZW1c5giIpIQZ1KoBoalLJcDq5spPwc4NcZ4RESkBXEmhfnAKDMbaWY9gbOBuakFzGxUyuIngaUxxiMi0unU1dfxSs0rzHl5DsvWL4v982LrU3D3WjO7HPgzUAjc7e6LzWwWUOXuc4HLzew4YCfwPnBhXPGIiOS7D7Z9wKI1i1i4ZiEL3l3AwjULefm9l9lWuw2A2SfM5qrDroo1BnPvXBf8VFZWelVVVa7DEBFps3qvZ8WGFSx8t6HyX7hmISs2rEiWKe1dyoTBE8K0d3g9pOwQehb2bNNnmtkL7l7ZUrnu85AdEZEc2LJzCy+teSlU/O+Gyn/RmkVs2rEJgAIr4MCBBzJ56GRmfmQmFXtXMGHvCezTdx/MMl2vEy8lBRGRduDurN60utE3/4XvLmTp+qXUez0AexTvwfjB47lgwgWh8h88gTGDxtC7R+8cR99ASUFEpJV21O1gSc2S5Df/RAJYt3VdsszI/iOZsPcEZoydkWz+GdF/RE6+/beGkoKISDNqNtc0avpZuGYhS2qWUFtfC0Cvol6MHTSW0w4+Ldn0M37wePYo3iPHkbeNkoKICOHSz9fXvb5LAli9qeH2qiH9hjBh8AQ+OeqTyQ7gUXuNorCgMIeRty8lBRHpsmrra1m3ZR01W2p4b/N71Gyu2WU+sbxiw4rkpZ89CnpwSNkhHLffcY2uACrtXZrjnyh+Sgoi0mnU1teydsvahgp9c1TBJ+a3vNdo2/qt6/EM42waxsDeAynrXUZZnzLGDhrLtAOmtculn52dkoKI5ExqJZ9auSe/zadV/Ou3rs+4n0QlP6jPIMp6h0p+UO9BlPUpS1b8iW1lfcoY2Gtgl2ryaU9KCiLSJHdnR90OttdtZ3vtdrbXbWdb7bbk/PbaaDnDfGrZrbVbQ+Wf1nTTVCVfYAUM7DUwWZmPHzw+Y+WemN+r116q5NuJkoK0irvn/SV1+cTdqfd6autrqfO68FofXrNdl1hubt2Ouh1NVsjJCrtuW1YVe/r29lBgBZT2Lk1W5hP2nhDmm6joB5QMUCWfI0oK3YC7s3nnZjZu38jG7RvZtH1Tw/yOTbuu39F0mS07t2AYhQWFFFph8rWooKhj1qUsFxZkXmdYdpVuFpVte6zLlQIroLiwmOKiYkqKSjLO9yrqxYCSARQXFVNc2HhbcjnDfHP7zPS+ooIifZnoJLpNUnho8UPcveBuehT0oGdhT3oURq/Rcup8+rbEcnNlsn1ftv8c7s72uu0tVuTJbTuaLrdp+6aMnW3pehT0YI/iPZJTv+J+DO47mFHFo+jXsx97FO9Bnx59cLxRpVdXX5d8Ta5LWZ+oJJtaV1tfy/a67buUa3b/zawDkskikVyKCoratK64qLjRcmq5Ruva4bPauq6woJCehT13qaSLCrrNv7e0o27zV7O9bjvrt65nR90OdtbtDK/14TV9XeKmlLj0KOjRZKLZUbcjWalnE4dhjSrxPYr3YM+SPSnfo7xhfVShp5dL31ZcVBzrz91R1MQl0nYaJTWDeq9nZ93OZNJIJIzURNJSYmlqXZP7qg/riguLs67E+xX3o0+PPqoARaRFGiV1NxRYQWgPpWt8cxYRyVacT14TEZFORklBRESSlBRERCQp1qRgZiea2WtmtszMrs2w/T/NbImZLTKzx8xseJzxiIhI82JLCmZWCNwBnASMBmaY2ei0Yi8Cle4+HngYuCWueEREpGVxnilMApa5+5vuvgOYA0xPLeDu89x9S7T4PFAeYzwiItKCOJPCUODtlOXqaF1TLgb+mGmDmc00syozq6qpqWnHEEVEJFWcSSHTHVUZ75Qzs/OASuC7mba7+53uXunulWVlZe0YooiIpIrz5rVqYFjKcjmwOr2QmR0HfA042t1bHJLxhRdeWGtmb7UxplJgbRvf21HyPcZ8jw8UY3vI9/gg/2PMt/iyupAntmEuzKwIeB04FlgFzAfOcffFKWUmEjqYT3T3pbEE0jimqmxu886lfI8x3+MDxdge8j0+yP8Y8z2+psTWfOTutcDlwJ+BV4AH3X2xmc0ys1OiYt8F+gIPmdkCM5sbVzwiItKyWMc+cvdHgUfT1l2fMn9cnJ8vIiKt093uaL4z1wFkId9jzPf4QDG2h3yPD/I/xnyPL6NON3S2iIjEp7udKYiISDOUFEREJKlLJoXOMBBfFjFeamYvRVdlPZNh3KicxpdS7kwzczPr8EvvsjiGF5lZTXQMF5jZ5/IpvqjMv0V/i4vN7H87Mr5sYjSzW1OO3+tmtiEPY9zXzOaZ2YvR//S0PItveFTPLDKzJ8wsv4fzcfcuNQGFwBvAfkBPYCEwOq3MVKB3NP954IE8jHGPlPlTgD/lU3xRuX7AU4Rxqyrz8BheBPwwj/8ORxEGhRwQLQ/KtxjTyn8RuDvfYiR06H4+mh8NrMiz+B4CLozmjwHuzcXfZLZTVzxT6AwD8WUT48aUxT40MURIruKL3EgY2XZbB8aWkG2MuZJNfJcAd7j7+wDu/l4exphqBvCrDomsQTYxOrBHNL8nGUZOyHF8o4HHovl5Gbbnla6YFNptIL4YZRWjmX3BzN4gVLxXdFBskEV80d3ow9z99x0YV6psf89nRKftD5vZsAzb45JNfAcCB5rZs2b2vJmd2GHRBVn/r0RNrCOBxzsgrlTZxHgDcJ6ZVRPui/pix4QGZBffQuCMaP40oJ+ZDeyA2NqkKyaFdhuIL0ZZxejud7j7/sA1wHWxR9Wg2fjMrAC4FfhSh0W0q2yO4e+AER6e1/E34J7Yo2qQTXxFhCakKYRv4T83s/4xx5Uq6/8V4GzgYXevizGeTLKJcQbwC3cvB6YB90Z/ox0hm/iuBo42sxeBownD/tTGHVhbdcWk0NqB+E7xLAbia2dZxZhiDnBqrBE11lJ8/YCxwBNmtgL4GDC3gzubWzyG7r4u5Xf7M+CjHRQbZPc7rgZ+6+473X058BohSXSU1vwdnk3HNx1BdjFeDDwI4O7PASWEweg6QjZ/h6vd/XR3n0ioc3D3DzoovtbLdadGe0+Eb19vEk51Ex0/Y9LKTCR0Do3K4xhHpcyfDFTlU3xp5Z+g4zuaszmG+6TMnwY8n2fxnQjcE82XEpohBuZTjFG5g4AVRDe75uHv+Y/ARdH8IYRKuUNizTK+UqAgmv8WMKujj2OrfqZcBxDTL2oaYYTWN4CvRetmEc4KIDQlrAEWRNPcPIzxNmBxFN+85irlXMSXVrbDk0KWx/A70TFcGB3Dg/MsPgNmA0uAl4Cz8+0YRss3ADd1dGytOI6jgWej3/MC4IQ8i+9MYGlU5udAca6OZTaThrkQEZGkrtinICIibaSkICIiSUoKIiKSpKQgIiJJSgoiIpKkpCDdhpkNTBnx810zWxXNbzCzJTF83hQza9UwINEomrvcBBiN+PrD9otOJDMlBek2PNzhXOHuFcBPgFuj+QqgvqX3m1mszzQXyQdKCiJBoZn9LHquwV/MrBckv7l/28yeBP7DzMrM7P/MbH40HR6VOzrlLORFM+sX7bdvNBjfq2Z2v5lZVP7YqNxLZna3mRWnB2Rmn4meYfAkcHgHHQfp5pQURIJRhGGsxwAbaBjVEqC/ux/t7t8n3Gl+q7sfGpX5eVTmauAL0ZnHkcDWaP1E4ErCXbf7AYebWQnwC+Asdx9HGCrh86nBmNk+wDcIyeD46P0isVNSEAmWu/uCaP4FYETKtgdS5o8DfmhmC4C5wB7RWcGzwGwzu4KQRBKjYP7T3avdvZ4wBMMIwlhCy9399ajMPcBRafFMBp5w9xoP4/Q/gEgHUBupSJA6Um4d0CtleXPKfAFwmLtvpbGbzOwPhHFwno9G4c203yIyD7ecicagkQ6nMwWR1vkLcHliwcwqotf93f0ld78ZqAIObmYfrwIjzOyAaPl84Mm0Mv8ApkRXTPUAPt1eP4BIc5QURFrnCqAyeprbEuDSaP2VZvaymS0k9Cc0+TQ/d98GfAZ4yMxeIlz59JO0Mu8QRid9jjCq77/a+wcRyUSjpIqISJLOFEREJElJQUREkpQUREQkSUlBRESSlBRERCRJSUFERJKUFEREJOn/A6TapnNNY1HkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x27434fb5c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(contrast_1.Threshold, contrast_1[\"Test Acc\"], color=\"blue\", label=\"Contrast 1.0\")\n",
    "plt.plot(contrast_15.Threshold, contrast_15[\"Test Acc\"], color=\"red\", label=\"Contrast 1.5\")\n",
    "plt.plot(contrast_2.Threshold, contrast_2[\"Test Acc\"], color=\"green\", label=\"Contrast 2.0\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.xlabel(\"Threshold\")\n",
    "plt.legend()\n",
    "plt.title(\"Threshold vs Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGXax/Hvj9BUQEoQwSiwUpRm0IgdRCyIIqhr4bWvytpXXfZVdy0s6tpRWX1t6yq2VXTtZbFQbKgEaQKCKK6EogiiUgSR+/3jOWMmISRDksmZJPfnuuZi5rS5Z4D88jznnOeRmeGcc86VV524C3DOOVe9eZA455yrEA8S55xzFeJB4pxzrkI8SJxzzlWIB4lzzrkK8SBx1ZKk4ZIeq4L3aSfJJNUtx74HSiooZf3Dkq6rWIWZR9Lpkt5Nem2SOsRZk0svDxKXkSStSnpslLQ26fVJcddXXUQ/xFdH39siSSMlZcVdl6tZPEhcRjKzRokH8BUwMGnZ41tyrPK0JmqY3aLvsQ9wAvC7mOtxNYwHiavO6kt6RNKPkmZJykuskPSlpMskzQBWS6orqY2kf0taJmmBpIuStu8lKV/SD5K+ljSy2HudJOkrSd9K+kvSfg0k3SFpcfS4Q1KDkoqV1FPSx1G9TwENN7NdA0krJXVLWtYyapVtJylb0svRNiskvSOpzP/LZjYfeA/ITTrutpIelLQkarFcl9xikXS2pDlRzbMl7R4tv1zS50nLjy7r/V3N5UHiqrOjgCeBpsCLwF3F1g8BjojWbwReAqYDOwD9gIslHRZteydwp5k1AXYGxhQ71v5A52i/qyXtGi3/C7A34YfzbkAv4MrihUqqDzwPPAo0B54Gji3pQ5nZOuDZqP6E44GJZvYN8EegAGgJtAL+DJQ51pGkXYADgPlJi0cDG4AOQE/gUOCsaPvjgOHAqUATwve9PNrv8+hY2wJ/BR6T1LqsGlzN5EHiqrN3zexVM/uF8AN6t2LrR5nZQjNbC+wJtDSzEWa23sy+AB4AToy2/RnoICnbzFaZ2QfFjvVXM1trZtMJYZR4r5OAEWb2jZktI/xQPaWEWvcG6gF3mNnPZvYMMLmUz/YERYPkf6JliVpbA22jY71jpQ+a97Gk1cAcYALwfwCSWgGHAxeb2eoopG5P+k7OAm42s8kWzDez/wKY2dNmttjMNprZU8BnhBB1tZAHiavOliY9XwM0LHY+ZGHS87ZAm6g7aKWklYTf5FtF688EOgGfSpos6cgy3qtR9LwN8N+kdf+NlhXXBlhU7Af+f0vYLmEcsJWkvSS1JbR4novW3UJoVbwu6QtJl5dyHIDdo3pPAPYCtomWtyWE25Kk7+Q+YLto/Y6ElscmJJ0qaVrSft2A7DLqcDVUbT8J6Wq25B/aC4EFZtaxxA3NPgOGROcajgGekdQihfdYTPiBPCt6vVO0rLglwA6SlBQmO7GZH9RmtlHSGEKr5GvgZTP7MVr3I6F764+SugLjJU02s7c2V2T0nmMkDQKuBi4mfCfrgGwz21DCbgsJ3XxFRMH2AKGbb5KZ/SJpGqDNvb+r2bxF4mqLj4AfohPwW0nKktRN0p4Akk6W1NLMNgIro31+SeG4/wKujE6GZxN+SJd0f8skwrmIi6IT/8dQdlfQE4RWxEkUdmsh6UhJHSQJ+CGqM5VaAW4Ehkra3syWAK8Dt0lqIqmOpJ0l9Ym2/QcwTNIeCjpEIbINIaSXRfWcQWiRuFrKg8TVCtF5lIGELqIFwLeEH5TbRpv0B2ZJWkU48X6imf2UwqGvA/KBGcBM4ONoWfH3X09o6ZwOfEcIiGfLqPlDYDWhW+y1pFUdgTeBVYSA+j8zm5BCrZjZTGAi8Kdo0alAfWB2VNczhPMvmNnTwPWEEPuRcLFAczObDdwWvffXQHfC1WCulpJPbOWcc64ivEXinHOuQjxInHPOVYgHiXPOuQrxIHHOOVchteI+kuzsbGvXrl3cZTjnXLUyZcqUb82sZVnb1YogadeuHfn5+XGX4Zxz1Yqk0kZf+JV3bTnnnKsQDxLnnHMV4kHinHOuQjxInHPOVYgHiXPOuQpJa5BI+qekbyR9spn1kjRK0nxJMxLTeEbrTpP0WfQ4LWn5HpJmRvuMikZAdc45F5N0t0geJoyqujmHE0Yy7QgMBe4BkNQcuIYwCU8v4BpJzaJ97om2TexX2vGdc86lWVrvIzGztyW1K2WTQcAj0aQ7H0hqGs37fCDwhpmtAJD0BtBf0gSgiZlNipY/Agym6BDblebi/1zMtKXT0nFo55xLu9ztc7mj/x1pf5+4z5HsQNHpUAuiZaUtLyhh+SYkDZWULyl/2bJllVq0c865QnHf2V7S+Q0rx/JNF5rdD9wPkJeXV65JV6oiyZ1zrrqLO0gKgB2TXucQ5rsuIHRvJS+fEC3PKWH79Pj2W/j5Z6hXD+rWDY/E8zp1wM/zO+dc7EHyInCBpCcJJ9a/N7MlksYCf0s6wX4ocIWZrZD0o6S9gQ8J04T+PW3VnXYavPrq5teXFDAlvU7leXn3qRv9FZrBxo0l/xn3uq23ho4doVMn6NwZdt4Z6tdP21+bc65qpTVIJP2L0LLIllRAuBKrHoCZ3Qu8CgwA5gNrgDOidSskXQtMjg41InHiHTiXcDXYVoST7Gk50Q7ABRfAwIGwYUNomWzYUPhIfr2556WtW7t2y/f5+efK/4xSeCRaWMX/rIx1P/4I33xT+J516kD79iFUOncuDJjOnaF1a2/pufBLyIYN4Rcnl/FqxZzteXl5ViNG/038hl88YKD8P+yrysqVMG9eeMydGx6J12vXFm7XqFHRYEk879QprHM105o1kJ8PkybB++/DBx/Ad9/B/vvDoYeGR25u+LfrqoykKWaWV+Z2HiQuVhs3QkHBpgEzdy78978hPBPatCm5FdO2bWEXn8t8ZuHvdtKkwse0aYW/FHXsCPvuC82bw7hxMH16WN6yJRxySAiVQw4J/x5cWnmQJPEgqabWroXPP980YObODb+tJtSrBx06bBownTpBdrZ3lcXtp5/g449DSyMRHEuWhHVbbw29esE++4Tw2Hvv8HeWbOlSeOMNeP318Eh0k3brFkLlsMPggANgq62q9nPVAh4kSTxIahgzWL685ICZP7/ouaRmzUoOmI4doWHD+D5DTVZQUBgY778fQiTxd/Kb34TQSDx69Niy1uTGjTBzZgiUsWPhnXdg/Xpo0AB69y7sBuve3X+BqAQeJEk8SGqRDRtCt0nxgJk3DxYtKtxOCl1iiYBp0wZatCh8ZGeHP5s39yvMSrN+PUydWrSbamF0L3HDhpCXF1oaieBo1apy33/NGnj77cLWyqxZYfn22xeGyiGHwHbbVe771hIeJEk8SBwAq1Zt/oT/jz9ufr/GjYuGTPFHInSSH40a1czfiJcuLWxpTJoEU6aEriuAnXYq7KLaZx/YbbeqD+GCgsJusDfeCC1XgJ49C4Nlv/1CC8aVyYMkiQeJK9OaNeGHTuLx7bdFX5f0WLly88erVy+1wEl+NG8OWVlV95nL8vPPMGNG0eD48suwrn592GOPot1UO5Q4WlF8Nm4MraWxY0OwvPdeaLFuvTX06RPOrRx6KOyyS80M/UrgQZLEg8SlxYYNsGJF2YFTPJgSVyeVpGnTsgOneDBV1knmZcuKdlF99FHhpdlt2hTtotp99+r3W/2PP8LEiYXnV+bNC8tzcgpbKwcfHL5TB3iQFOFB4jKGWfiBlmroJB6rVm3+mFtttWXdbi1ahO66WbOKtjbmzw/Hq1s3dAUld1PtuGPN+639yy8Lu8HefDO0MKVwXicRLHvvXavPkXmQJPEgcdXeunWbtn7K6n5bsSJ075SlVauiXVR5ebXvUtpffoHJkwtP2n/wQVjWqBH07VvYDdahQ80L1FJ4kCTxIHG10saN8P33JYfOypXhirV99gnD1dSiH44p+f57GD8+dIGNHQsLFoTl7doVtlb69QtdkTWYB0kSDxLnXIV8/nnhuZVx40L3ZJ064X6V3XcvfOy2G2yzTdzVVhoPkiQeJM65SvPzz/Dhh+H8ykcfhUugE5Pn1akTrgJLDpfcXNh223hrLqdUg8QHKHLOuS1Rr14YTHL//cNrM1i8OATKxx+Hx/jx8Nhjhft06FAYLHvsES5mqEFXh3mQOOdcRUjhHpoddoCjjipc/vXX4T6Wjz8OIfPRRzBmTOH6tm2Ltlx23z3ckV8NeZA451w6tGoF/fuHR8KKFYXhkng891zh+tatQ4slOVxycjL+YggPEuecqyrNm4ervfr1K1z2ww9hGP3kcHn11cJLt7Ozi3aL7b57xl1p50HinHNxatIkjFzcu3fhsjVrwvA0iW6xjz+GW28tHBVh22037Rbr2DG2IXY8SErx6adhWJ6ddoq7EudcrbL11uGu+r33Lly2bh188knRlstdd4XlEC477tmzaLjsumuVTPrml/+WYv/9w9QHt98OZ5yRUS1J55wLlyLPmVM0XKZNg9Wrw/qGDcOlyj16lOvwqV7+m9YJkCX1lzRX0nxJl5ewvq2ktyTNkDRBUk60vK+kaUmPnyQNjtY9LGlB0rrcdNX/yCPhEvAzz4SBAwsndXPOuYxQr14IidNPh1Gj4N13w135c+bA44/DeefBzjunvYy0tUgkZQHzgEOAAmAyMMTMZidt8zTwspmNlnQQcIaZnVLsOM2B+UCOma2R9HC0zzOp1lKRGxI3bgx/P1dcEYYfuusuGDLEWyfOuZovE1okvYD5ZvaFma0HngQGFdumC/BW9Hx8CesBfgu8ZmZr0lZpKerUgYsvDq3Fzp3hpJPguOMKb2R1zrnaLp1BsgOwMOl1QbQs2XTg2Oj50UBjScVv9zwR+FexZddH3WG3S6qSSRE6dw7TQ99wA7z0EnTtCs8+WxXv7JxzmS2dQVJS50/xfrRhQB9JU4E+wCLg11l/JLUGugNjk/a5AtgF2BNoDlxW4ptLQyXlS8pfVknNh7p14fLLw9V4OTlw7LFw8snw3XeVcnjnnKuW0hkkBcCOSa9zgMXJG5jZYjM7xsx6An+Jln2ftMnxwHNm9nPSPkssWAc8ROhC24SZ3W9meWaW17Jly8r5RJFu3cKFENdcA089FVonr75aqW/hnHPVRjqDZDLQUVJ7SfUJXVQvJm8gKVtSooYrgH8WO8YQinVrRa0UJAkYDHyShtrLVK8eDB8eAqV5czjiCDjrrHCTqnPO1SZpCxIz2wBcQOiWmgOMMbNZkkZISoxsdiAwV9I8oBVwfWJ/Se0ILZqJxQ79uKSZwEwgG7guXZ8hFbvvHrq6Lr8cHnooTE/w1ltl7+ecczWF35BYiT74AE47DebNg/PPh5tuqlFz3DjnaplMuPy31tl77zCw58UXw913h8nS3n037qqccy69PEgq2dZbhyFVJkwINzP27g3DhsHatXFX5pxz6eFBkiZ9+sD06TB0KNx2WziX8tFHcVflnHOVz4MkjRo3hnvvhbFjYdUq2HdfuPJKWL8+7sqcc67yeJBUgUMPDaMIn3wyXH897LlnaK0451xN4EFSRZo2hYcfhhdeCFM577knXHdd4Tw1zjlXXXmQVLGjjoJZs+CYY+Cqq2CffWD27LL3c865TOVBEoMWLeDJJ8PwKgsWhBPxt94Kv/wSd2XOObflPEhidPzxoXXSvz/86U/hSq/58+OuyjnntowHScxatYLnnguzMX7ySbiJ8a67wj0ozjlXHXiQZAAJTjkltE5694YLL4RDDoH//jfuypxzrmweJBlkhx3CcPT33x9uXuzeHf7xD6gFw6E556oxD5IMI8HZZ4f7TvLywvMjjoBFi+KuzDnnSuZBkqHatYM334RRo8K4Xd26wWOPeevEOZd5PEgyWJ064XzJ9Omw667hPMoxx4QbGp1zLlN4kFQDHTvCO+/AzTeHcyjdusEzz8RdlXPOBR4k1URWVrjX5OOPoW1bOO44GDIEli+PuzLnXG3nQVLNdO0KkybBiBGhVdKtWxiza+7cuCtzztVWHiTVUL16YZyuyZOhU6fwfJddws2MHirOuarmQVKN5ebCxImwcCHccUeY/yQRKj16eKg456pGWoNEUn9JcyXNl3R5CevbSnpL0gxJEyTlJK37RdK06PFi0vL2kj6U9JmkpyTVT+dnqA5ycuAPfwjzwydCpUmToqFy7bXw6adxV+qcq4lkaboxQVIWMA84BCgAJgNDzGx20jZPAy+b2WhJBwFnmNkp0bpVZtaohOOOAZ41sycl3QtMN7N7SqslLy/P8vPzK+2zVRcFBfDvf8PTT8N774Vl3buHE/XHHRdCxjnnNkfSFDPLK2u7dLZIegHzzewLM1sPPAkMKrZNF+Ct6Pn4EtYXIUnAQUDi4tfRwOBKq7iGSW6pFBTAnXeGlsrVV4f7Urp395aKc67i0hkkOwALk14XRMuSTQeOjZ4fDTSW1CJ63VBSvqQPJCXCogWw0swS8wqWdEwAJA2N9s9ftmxZRT9LtbfDDnDRRUVDpWlTDxXnXMWlM0hUwrLi/WjDgD6SpgJ9gEVAIiR2ippU/wPcIWnnFI8ZFprdb2Z5ZpbXsmXLcn2AmioRKu+8UzRUrrmmMFRGjIA5c+Ku1DlXHaQzSAqAHZNe5wCLkzcws8VmdoyZ9QT+Ei37PrEu+vMLYALQE/gWaCqp7uaO6bZM8VAZNSqEyvDh0KWLh4pzrmzpDJLJQMfoKqv6wInAi8kbSMqWlKjhCuCf0fJmkhoktgH2A2ZbuDJgPPDbaJ/TgBfS+BlqlTZtwtheyaHSrFlhqHTr5qHinNtU2oIkOo9xATAWmAOMMbNZkkZIOira7EBgrqR5QCvg+mj5rkC+pOmE4Lgx6Wqvy4BLJc0nnDN5MF2foTZLhMrbbxeGSvPmRUPlr3+F2bPLPJRzroZL2+W/maS2Xv6bDosXw7PPwpgx4cS9WRi2JXFJcZcucVfonKssqV7+60Hiyi0RKk8/HbrDzEKQHH+8h4pzNYEHSRIPkvRbsqTw5sfkUDnuOOjXDxo0CPOrJB5ZWUVfV3RdVlaYXVIlXdfnnCsXD5IkHiRVq6RQqSpS+YKpSxc45JDw6N7dA8k58CApwoMkPkuWwIwZsHEj/PJL+LOkR2Wv25J91q8PIyknrkbbfns4+ODCYGndOt7v0Lm4pBokdcvawLmKaN26+vwgLiiAN94Ij7Fj4bHHwvJu3QpDpXdv2GabeOt0LtN4i8S5EmzcCNOnFwbLO+/AunVQvz7st19hsPTsGbrHnKuJvGsriQeJq6i1a8Plzq+/HoJl+vSwvHnzot1gbdvGW6dzlcm7tpyrRFttVRgWAF9/DW++WdhiGTMmLO/YEQ49NGzXt28Ybdm5ms5bJM5VkFk4UZ9orUycCKtXhy6vvfYqDJZevaCu/+rmqhHv2kriQeKq0vr1MGlSCJXXX4f8/BA2TZqEVkoiWDp08MuMXWbzIEniQeLitGIFjBtX2GL58suwvG3bwu6yfv2gRYtSD+NclfMgSeJB4jKFGXz+eWFrZdw4+OGH0DLZY48QKoceCvvsE0YDcC5OHiRJPEhcptqwIdwMmQiWDz4IN0xuvTX06VPYYuna1bvBXNXzIEniQeKqix9+gAkTCq8Gmzs3LG/dujBUDj443H3vXLp5kCTxIHHV1VdfFYbKm2/C8uVheY8ehcFywAGhBeNcZfMgSeJB4mqCjRth6tTCYHn33XCFWIMGsP/+hcGSmxsGonSuojxIkniQuJpozZowg2UiWGbODMuzs4vebb/jjvHW6aovD5IkHiSuNliypOjd9kuXhuWdOxfeu3LggdC4caxlumrEgySJB4mrbczgk08KQ2XixDBeWN26sPfeIViOOCIMOulXg7nNSTVI0tqTKqm/pLmS5ku6vIT1bSW9JWmGpAmScqLluZImSZoVrTshaZ+HJS2QNC165KbzMzhXHUlhgq5LL4XXXoPvvoO33oJhw0KgXHNNuG+lXTu48MKw7uef467aVVdpa5FIygLmAYcABcBkYIiZzU7a5mngZTMbLekg4AwzO0VSJ8DM7DNJbYApwK5mtlLSw9E+z6Rai7dInCtq2TJ4+WV4/vlw/8pPP0HTpqGVMmgQ9O/vXWAuM1okvYD5ZvaFma0HngQGFdumC/BW9Hx8Yr2ZzTOzz6Lni4FvgJZprNW5WqVlSzjjDHjhBfj2W3juORg8GP7zHzj++HDCfsAAuP/+cO7FudKkM0h2ABYmvS6IliWbDhwbPT8aaCypyIhDknoB9YHPkxZfH3V53S6pxIEkJA2VlC8pf9myZRX5HM7VaNtsE0LkoYfCCfqJE+H888PNkL//PbRpE4ZsufFG+PTTuKt1mSidQVLSKbzi/WjDgD6SpgJ9gEXAhl8PILUGHiV0eW2MFl8B7ALsCTQHLivpzc3sfjPLM7O8li29MeNcKurWDdMJjxwJ8+fDjBlw7bXh/MkVV8Cuu4arwC67DN5/P9zb4lyp50gkXVrazmY2spR99wGGm9lh0esron1u2Mz2jYBPzSxxwr0JMAG4wcye3sw+BwLDzOzI0ur0cyTOVdzChfDii6E7bPz4ME5Yq1YwcGBo0fTrBw0bxl2lq0yVdY6kcRmP0kwGOkpqL6k+cCLwYrEisyUlargC+Ge0vD7wHPBI8RCJWilIEjAY+KSMOpxzlWDHHUOX1+uvh5P1TzwRBpZ86ik48shwXuW3v4VHHw1D57vaI633kUgaANwBZAH/NLPrJY0A8s3sRUm/BW4gdHm9DZxvZusknQw8BMxKOtzpZjZN0jjCiXcB04BzzGxVaXV4i8S59Fm3LrRQXnghPJYsCbND9u4dWiqDBvlc9tVVpdyQKGlUaTub2UXlqK3KeZA4VzU2bgwzQj7/fAiV2dHF/rm5IVAGD4bddvObIKuLygqS00rb2cxGl6O2KudB4lw8PvssBMrzz4eT82ahdTJoUHgccADUqxd3lW5zfIiUJB4kzsXvm2/gpZdCsLzxRrgJslmzcBPk4MFw2GHQqFHcVbpklRokkloSLrPtAvx6XYaZHVSRIquKB4lzmWX16nDS/vnnwx32K1aE4fAPPji0VAYO9Mm7MkFl39n+ODAHaA/8FfiScFWWc85tsW22gaOPhtGj4euvw8n6c8+FWbNg6NBwE+Rhh4W77l3mSzVIWpjZg8DPZjbRzH4H7J3GupxztUTdumF4+9tvhy++gOnT4eqrw1wrffr4EC3VQapBkhgXdImkIyT1BHLSVJNzrpaSwjTCw4eHUYu/+iqckP/yy7grc6VJNUiuk7Qt8EfCsCb/AC5JW1XOuVrvwAML56k/4IAw9pfLTCkFiZm9bGbfm9knZtbXzPYwsxfL3tM558pvr73CIJLr14cbHKdPj7siV5KUgkTSaElNk143k/TP9JXlnHNBjx7hfEn9+qGV8uGHcVfkiku1a6uHma1MvDCz74Ce6SnJOeeK6twZ3nkHWrQIlwhPmBB3RS5ZqkFSR1KzxAtJzYG66SnJOec21a5dCJO2beHww+HVV+OuyCWkGiS3Ae9LujYadPF94Ob0leWcc5tq3Tq0Rrp0CXfDP13iBBOuqqV6sv0RwkyGXwPLgGPM7NF0FuaccyXJzoZx46BXLzjxRHj44bgrclsyQ2JzYLWZ/R1YJql9mmpyzrlSbbstjB0bJtM64wy46664K6rdUr1q6xrCWFtXRIvqAY+lqyjnnCvLNtuEQSAHD4YLL4QbSpx71VWFVFskRwNHAasBzGwxZc+Q6JxzadWgAYwZAyedBH/+c5hXvhYMaJ5xUr3yar2ZmSQDkLRNGmtyzrmU1asHjzwShqC/8UZYtQruvBPqbEnHvauQVINkjKT7gKaSzgZ+RxgmxTnnYlenDtxzTwiT224LYfLAA2FASJd+KX3NZnarpEOAH4DOwNVm9kZaK3POuS0gwS23QJMmcM01IUwefzzcEe/SK+W8joLjDQBJWZJOMrPH01aZc85tISkMQd+4MVx6KaxZA888A1ttFXdlNVupvYiSmki6QtJdkg5VcAHwBXB8WQeX1F/SXEnzJV1ewvq2kt6SNEPSBEk5SetOk/RZ9DgtafkekmZGxxwlSVv2kZ1zNd0ll8D994eh6AcMgB9/jLuimq2s01GPErqyZgJnAa8DxwGDzGxQaTtKygLuBg4nTNE7RFKXYpvdCjxiZj2AEcAN0b7NgWuAvYBewDVJQ7TcAwwFOkaP/mV/TOdcbXP22aFr6513wvhcK1bEXVHNVVbX1m/MrDuApH8A3wI7mVkq+d4LmG9mX0T7PwkMAmYnbdOFwnlNxgPPR88PA94wsxXRvm8A/SVNAJqY2aRo+SPAYOC1FOpxztUyQ4aE+02OOw769g3zxLdqFXdVNU9ZLZLEzIiY2S/AghRDBGAHYGHS64JoWbLphKFXINyr0lhSi1L23SF6XtoxAZA0VFK+pPxly5alWLJzrqY56ih45RWYPz/MabJwYdn7uC1TVpDsJumH6PEj0CPxXNIPZexb0rmL4rcKDQP6SJoK9AEWARtK2TeVY4aFZvebWZ6Z5bVs2bKMUp1zNdnBB4fWyNKlYbbF+fPjrqhmKTVIzCzLzJpEj8ZmVjfpeZMyjl0A7Jj0OgdYXOz4i83sGDPrCfwlWvZ9KfsWUHSu+E2O6ZxzJdlvPxg/PlwWfMAB8MkncVdUc6Tz3s/JQEdJ7SXVB04EikzPKylbUqKGK4DErItjgUOjmRibAYcCY81sCfCjpL2jq7VOBV5I42dwztUgu+8eZluUoE8fyM+Pu6KaIW1BYmYbgAsIoTAHGGNmsySNkHRUtNmBwFxJ84BWwPXRviuAawlhNBkYkTjxDpxLuKt+PvA5fqLdObcFunSBd98NNy4edFC4qstVjKwWjHCWl5dn+f6rh3MuSUFBOHfy1Vfw/PNw6KFxV5R5JE0xs7yytvNhzZxztVJOTujm6tQJBg4MYeLKx4PEOVdrbbddOAG/++7w29+GGxjdlvMgcc7Vas2awRtvhJPvp5wC990Xd0XVjweJc67Wa9Qo3LR4xBFwzjlw661xV1S9eJA45xzQsCE8+yyccAL86U9hKPpacC1SpfBpX5xzLlKvXjhPss02MGJEGDXrU8KcAAAYSklEQVT4ttvCfSdu8zxInHMuSVZWmF2xUSO4/fYQJvfeG5a7knmQOOdcMXXqwB13hAmyrr8eVq+G0aNDi8VtyoPEOedKIMF114UwufzyECZPPRXOpbii/GS7c86V4rLL4O674cUX4cgjQ6C4ojxInHOuDOedF7q2xo8PQ6msXBl3RZnFg8Q551Jw6qkwZgxMnhwGe/T58gp5kDjnXIqOPTZ0cc2ZE+6EX+yzIQEeJM45t0X694exY8OUvUOGwMaNcVcUPw8S55zbQr17h8uD334bHnoo7mri50HinHPl8LvfhUD505/gm2/iriZeHiTOOVcOUhgpePVquOSSuKuJlweJc86V0y67wBVXwBNPhPMmtZUHiXPOVcAVV0DnznDuubBmTdzVxCOtQSKpv6S5kuZLuryE9TtJGi9pqqQZkgZEy0+SNC3psVFSbrRuQnTMxLrt0vkZnHOuNA0ahC6uBQvgr3+Nu5p4pC1IJGUBdwOHA12AIZK6FNvsSmCMmfUETgT+D8DMHjezXDPLBU4BvjSzaUn7nZRYb2a1/DSXcy5uffqEk++33QbTp8ddTdVLZ4ukFzDfzL4ws/XAk8CgYtsY0CR6vi1Q0u09Q4B/pa1K55yrBLfcAs2bw9Ch8MsvcVdTtdIZJDsAC5NeF0TLkg0HTpZUALwKXFjCcU5g0yB5KOrWukoqecoZSUMl5UvKX+ZjGTjn0qx58zB/yUcfwT33xF1N1UpnkJT0A774xJVDgIfNLAcYADwq6deaJO0FrDGzT5L2OcnMugMHRI9TSnpzM7vfzPLMLK9ly5YV+RzOOZeS//mfMKjjn/8MixbFXU3VSWeQFAA7Jr3OYdOuqzOBMQBmNgloCGQnrT+RYq0RM1sU/fkj8AShC80552InhdbIhg1wYUn9KzVUOoNkMtBRUntJ9Qmh8GKxbb4C+gFI2pUQJMui13WA4wjnVoiW1ZWUHT2vBxwJfIJzzmWI3/wGrrkGnnsOXngh7mqqRtqCxMw2ABcAY4E5hKuzZkkaIemoaLM/AmdLmk5oeZxuZonur95AgZl9kXTYBsBYSTOAacAi4IF0fQbnnCuPSy+FHj3gggvCnO81nQp/btdceXl5lp+fH3cZzrla5MMPYZ99QhfXnXfGXU35SJpiZnllbed3tjvnXBrstVeYWfHvfw9XctVkHiTOOZcmf/sbtG4d7i35+ee4q0kfDxLnnEuTJk1Ci2T69DB/SU3lQeKcc2l09NFw1FHhSq4FC+KuJj08SJxzLo0kuOsuyMoK50xq4vVNHiTOOZdmO+4I110H//kPPPVU3NVUPg8S55yrAhdcAHl58Ic/wHffxV1N5aobdwFx+fnnnykoKOCnn36Ku5RaqWHDhuTk5FCvXr24S3GuSmRlwQMPhDC57DK4//64K6o8tTZICgoKaNy4Me3atWMzAwi7NDEzli9fTkFBAe3bt4+7HOeqTG5umN/91lvhlFPggAPirqhy1NqurZ9++okWLVp4iMRAEi1atPDWoKuVhg+Htm3h97+HdevirqZy1NogATxEYuTfvautttkmjBA8Zw7cfHPc1VSOWh0kzjkXh8MPhxNOCFdyzZ0bdzUV50ESo6VLl3LiiSey884706VLFwYMGMC8efPKdaw77riDNWvWVEpdEyZM4P333y9x3aeffso+++xDgwYNuPXWWzd7jAULFrDXXnvRsWNHTjjhBNavX18ptTlXU9xxB2y9NZxzTvW/t8SDJCZmxtFHH82BBx7I559/zuzZs/nb3/7G119/Xa7jlRYkv2zhBNKlBUnz5s0ZNWoUw4YNK/UYl112GZdccgmfffYZzZo148EHH9yiGpyr6bbfHm66CSZMgIcfjruaiqm1V20lu/himDatco+Zm1v62Drjx4+nXr16nHPOOUn75AIhZP73f/+X1157DUlceeWVnHDCCUyYMIHhw4eTnZ3NJ598wh577MFjjz3G3//+dxYvXkzfvn3Jzs5m/PjxNGrUiEsvvZSxY8dy2223MW7cOF566SXWrl3Lvvvuy3333YckRo0axb333kvdunXp0qULN954I/feey9ZWVm/HvuApEtLtttuO7bbbjteeeWVzX42M2PcuHE88cQTAJx22mkMHz6cc889t4LfqnM1y1lnwSOPwLBhcOSRUF1nBfcgiUkiCEry7LPPMm3aNKZPn863337LnnvuSe/evQGYOnUqs2bNok2bNuy333689957XHTRRYwcOZLx48eTnR1mKl69ejXdunVjxIgRAHTp0oWrr74agFNOOYWXX36ZgQMHcuONN7JgwQIaNGjAypUradq0Keeccw6NGjUqs9WxOcuXL6dp06bUrRv+eeXk5LCoNk1g7VyK6tQJ95Pk5obJsB59NO6KyseDhMwblfPdd99lyJAhZGVl0apVK/r06cPkyZNp0qQJvXr1IicnBwgtmC+//JL9999/k2NkZWVx7LHH/vp6/Pjx3HzzzaxZs4YVK1bQtWtXBg4cSI8ePTjppJMYPHgwgwcPrpT6S5osza/Scq5kXbrA5ZfDtdfCqafCIYfEXdGW83MkMenatStTpkwpcV1ps1Y2aNDg1+dZWVls2LChxO0aNmxIVlYWEO6ZOe+883jmmWeYOXMmZ5999q/3cLzyyiucf/75TJkyhT322GOzx9sS2dnZrFy58tdjFRQU0KZNmwof17ma6s9/hk6d4NxzYe3auKvZch4kMTnooINYt24dDzxQOOX85MmTmThxIr179+app57il19+YdmyZbz99tv06tWr1OM1btyYHzczOXQiNLKzs1m1ahXPPPMMABs3bmThwoX07duXm2++mZUrV7Jq1apSj5UKSfTt2/fX9xk9ejSDBg0q9/Gcq+kaNoR774XPPw8tk+omrUEiqb+kuZLmS7q8hPU7SRovaaqkGZIGRMvbSVoraVr0uDdpnz0kzYyOOUrVtM9EEs899xxvvPEGO++8M127dmX48OG0adOGo48+mh49erDbbrtx0EEHcfPNN7P99tuXeryhQ4dy+OGH07dv303WNW3alLPPPpvu3bszePBg9txzTyBczXXyySfTvXt3evbsySWXXELTpk0ZOHAgzz33HLm5ubzzzjtFjrV06VJycnIYOXIk1113HTk5Ofzwww8ADBgwgMWLFwNw0003MXLkSDp06MDy5cs588wzK+Nrc67G6tsXTj8dbrkFZs6Mu5oto9K6USp0YCkLmAccAhQAk4EhZjY7aZv7galmdo+kLsCrZtZOUjvgZTPrVsJxPwL+AHwAvAqMMrPXSqslLy/P8vPziyybM2cOu+66awU+oaso/ztwrqjly2GXXaBjR3j33XAyPk6SpphZXlnbpbPMXsB8M/vCzNYDTwLF+zcMaBI93xZYXNoBJbUGmpjZJAsJ+AhQOWeInXMuZi1awMiRMGkS3Hdf3NWkLp1BsgOwMOl1QbQs2XDgZEkFhNbFhUnr2kddXhMlJW5k2CE6TmnHdM65auvkk6Ffv3Al1+JSf7XOHOkMkpLOXRTvRxsCPGxmOcAA4FFJdYAlwE5m1hO4FHhCUpMUjxneXBoqKV9S/rJly8r9IZxzripJ4cT7+vVw0UVxV5OadAZJAbBj0uscNu26OhMYA2Bmk4CGQLaZrTOz5dHyKcDnQKfomDllHJNov/vNLM/M8lpW19tFnXO1UocOcNVV8O9/w0svxV1N2dIZJJOBjpLaS6oPnAi8WGybr4B+AJJ2JQTJMkkto5P1SPoN0BH4wsyWAD9K2ju6WutU4IU0fgbnnIvFsGHQtSucfz6sWhV3NaVLW5CY2QbgAmAsMAcYY2azJI2QdFS02R+BsyVNB/4FnB6dRO8NzIiWPwOcY2Yron3OBf4BzCe0VEq9Yss556qj+vXD8CkLF4bWSSZL68VlZvaqmXUys53N7Ppo2dVm9mL0fLaZ7Wdmu5lZrpm9Hi3/t5l1jZbvbmYvJR0z38y6Rce8wNJ1/XIVqMnDyJ9++um0b9+e3NxccnNzmVbZo2I6Vwvsu2+4233UKNjMQBgZwe9sj0lNH0Ye4JZbbmHatGlMmzbt15GNnXNb5oYboFUrOPtsqIQRjNLCB22EWMaRr8nDyDvnKs+224YWyXHHhT8vvTTuijblQRKTmjyMfMJf/vIXRowYQb9+/bjxxhuLDDjpnEvdsceG+Uquuio8b9s27oqK8iCBjBtHvroPIw9www03sP3227N+/XqGDh3KTTfd9GuQOee2jAR33x2GnD///HBJcCaNMujnSGJSk4eRB2jdujWSaNCgAWeccQYfffRRpRzXudpqp53CyMCvvALRwNoZw4MkJjV5GHmAJUuWACEUn3/+ebp122T8TefcFrrwQth993DH+8qVcVdTyIMkJjV9GPmTTjqJ7t270717d7799luuvPLKyvjanKvV6taFBx6Ab74JY3FlirQNI59JfBj5zOR/B86Vz6WXwu23h6Hm99svfe+TCcPIO+ecS4MRI8I5k6FDw+COcfMgcc65aqZRo3AV1+zZYUbFuHmQOOdcNXTkkeEmxWuvhc8+i7cWDxLnnKum7rwTGjaEc86BOE93e5A451w11bo13HgjjBsHjz4aXx0eJM45V40NHRpGCb70Uvj223hq8CCJUXUcRv7xxx+nR48e9OjRg3333Zfp06eXuN2CBQvYa6+96NixIyeccALrM+HSEudqoDp14L774Pvvw2RYsdQQz9u66jqMfPv27Zk4cSIzZszgqquuYujQoSVud9lll3HJJZfw2Wef0axZMx588MEtqsE5l7pu3eB//xdGjw7dXFXNb0gELv7PxUxbWrnDyOdun8sd/Tc/GOS4ceMYPnw4b7/99ibryjOM/LBhw+jcuXOlDCO/9957k5WVRcuWLTcZRj7Zd999R7du3Vi0aNEm9bds2ZKlS5dSt25dJk2axPDhwxk7dmyR7fyGROcqz9q10KNHeD5jBmy1VcWPmeoNiT76b0xqwjDyDz74IIcffvgmy5cvX07Tpk2pWzf888rJydkkbJxzlWurreDee+Hgg+H66+G666ruvT1IoNSWQxyqwzDy48eP58EHH+Tdd9/dZF1JrVxl0pjXztVQ/frBKafATTfBkCHQtWvVvK+fI4lJdR5GfsaMGZx11lm88MILtGjRYpP12dnZrFy58tdjFRQU0KZNmzKP65yruNtuC7MqDh0KGzdWzXumNUgk9Zc0V9J8SZuMVSlpJ0njJU2VNEPSgGj5IZKmSJoZ/XlQ0j4TomNOix7bpfMzpEt1HUb+q6++4phjjuHRRx+lU6dOJW4jib59+/76PqNHj2bQoEGlfyHOuUrRsiXceiu8/34YKbgqpC1IJGUBdwOHA12AIZK6FNvsSmCMmfUETgT+L1r+LTDQzLoDpwHFb7U5ycxyo8c36foM6VRdh5EfMWIEy5cv57zzziM3N5e8vMLzcMnDyN90002MHDmSDh06sHz5cs4888yKfmXOuRSddhr07QuXXQbR1EBplbartiTtAww3s8Oi11cAmNkNSdvcB3xhZjdF299mZvsWO44IwdLGzNZJmgAMM7Oil2GVwoeRz0z+d+Bc+nz2GfzhD3DPPeWf4z0TrtraAViY9LoA2KvYNsOB1yVdCGwDHFzCcY4FpprZuqRlD0n6Bfg3cJ2VkIaShgJDAXbaaafyfgbnnKuWOnaEV1+tmvdK5zmSki7TKf4DfwjwsJnlAAOARyX9WpOkrsBNwO+T9jkp6vI6IHqcUtKbm9n9ZpZnZnktW7aswMdwzjlXmnQGSQGwY9LrHGBxsW3OBMYAmNkkoCGQDSApB3gOONXMPk/sYGaLoj9/BJ4ASj8LXYracDNmpvLv3rmaI51BMhnoKKm9pPqEk+kvFtvmK6AfgKRdCUGyTFJT4BXgCjN7L7GxpLqSEkFTDzgS+KQ8xTVs2JDly5f7D7QYmBnLly+nYcOGcZfinKsEaTtHYmYbJF0AjAWygH+a2SxJI4B8M3sR+CPwgKRLCN1ep5uZRft1AK6SdFV0yEOB1cDYKESygDeBcl3glpOTQ0FBAcuWLavIx3Tl1LBhw19vrHTOVW+1dqwt55xzpUv1qi2/s90551yFeJA455yrEA8S55xzFVIrzpFIWgb8t5y7ZxPurM9kmV5jptcHmV9jptcHXmNlyLT62ppZmTfi1YogqQhJ+amcbIpTpteY6fVB5teY6fWB11gZMr2+zfGuLeeccxXiQeKcc65CPEjKdn/cBaQg02vM9Pog82vM9PrAa6wMmV5fifwciXPOuQrxFolzzrkK8SBxzjlXIR4kkRTml79U0uxobvm3JJVzzrG01XdONMf9NEnvljCtcew1Jm33W0kmqUovc0zhOzxd0rLoO5wm6ayqrC+VGqNtjo/+Lc6S9ESm1Sjp9qTvcJ6klRlW306SxkuaGv1/HlCV9aVYY9vo58wMSROiaTUyl5nV+gdhJOHPgd8A9YHpQJdi2/QFto6enws8lWH1NUl6fhTwn0z7DqPtGgNvAx8AeZlUH3A6cFeG/zvsCEwFmkWvt8u0GottfyFh5O+MqY9wQvvc6HkX4MtM+w6Bp4HToucHAY/G8W8y1Ye3SIJewHwz+8LM1gNPAoOSNzCz8Wa2Jnr5AWGirkyq74ekl9uw6WyU6VZmjZFrgZuBn6qyOFKvL06p1Hg2cLeZfQdgZt9kYI3JhgD/qpLKglTqM6BJ9HxbNp1wL91SqbEL8Fb0fHwJ6zOKB0lQ0vzyO5Sy/ZnAa2mtqKiU6pN0vqTPCT+oL6qi2hLKrFFST2BHM3u5KguLpPp3fGzUnfCMpB1LWJ9OqdTYCegk6T1JH0jqX2XVBSn/X4m6f9sD46qgroRU6hsOnCypAHiV0GqqSqnUOB04Nnp+NNBYUosqqK1cPEiCVOaXDxtKJwN5wC1prajY25awbJP6zOxuM9sZuAy4Mu1VFVVqjZLqALcTJjOLQyrf4UtAOzPrQZg0bXTaqyoqlRrrErq3DiT8tv+PaEbRqpLy/xXCrKjPmNkvaaynuFTqGwI8bGY5wADg0ejfZ1VJpcZhQB9JU4E+wCJgQ7oLKy8PkiCV+eWRdDDwF+AoM1tXRbVBivUleRIYnNaKNlVWjY2BbsAESV8CewMvVuEJ9zK/QzNbnvT3+gCwRxXVlpDK33MB8IKZ/WxmC4C5hGCpKlvyb/FEqrZbC1Kr70xgDICZTSJM8Z1dJdUFqfxbXGxmx5hZT8LPHMzs+6orcQvFfZImEx6E3/K+IDTDEye/uhbbpifhBFnHDK2vY9LzgYTpjDOqxmLbT6BqT7an8h22Tnp+NPBBpn2HQH9gdPQ8m9BF0iKTaoy26wx8SXTTcybVR+iWPj16vivhh3iV1ZlijdlAnej59cCIqvwet/gzxV1ApjwITdx5UVj8JVo2gtD6gNDV8TUwLXq8mGH13QnMimobX9oP8bhqLLZtlQZJit/hDdF3OD36DnfJtO+Q0C0yEpgNzAROzLQao9fDgRururYUv8MuwHvR3/M04NAMrPG3wGfRNv8AGsTxXab68CFSnHPOVYifI3HOOVchHiTOOecqxIPEOedchXiQOOecqxAPEueccxXiQeLcZkhqkTSK7VJJi6LnKyXNTsP7HShpi4aPiUaG3eSmzmgk47sqrzrnNs+DxLnNsHCne66Z5QL3ArdHz3OBjWXtL6luumt0LhN4kDhXPlmSHojmBHld0lbwawvhb5ImAn+Q1FLSvyVNjh77Rdv1SWrtTJXUODpuo2jAyE8lPS5J0fb9ou1mSvqnpAbFC5J0RjT/x0Rgvyr6HpzzIHGunDoShnPvCqykcKRWgKZm1sfMbiOMOHC7me0ZbfOPaJthwPlRC+cAYG20vCdwMeHu698A+0lqCDwMnGBm3QlDbJybXIyk1sBfCQFySLS/c1XCg8S58llgZtOi51OAdknrnkp6fjBwl6RpwItAk6j18R4wUtJFhOBJjOz6kZkVmNlGwvAd7QjjVi0ws3nRNqOB3sXq2QuYYGbLLMxx8RTOVRHvw3WufJJHf/4F2Crp9eqk53WAfcxsLUXdKOkVwphLH0QjS5d03LqUPOx4SXy8IxcLb5E4l16vAxckXkjKjf7c2cxmmtlNQD6wSynH+BRoJ6lD9PoUYGKxbT4EDoyuNKsHHFdZH8C5sniQOJdeFwF50ayLs4FzouUXS/pE0nTC+ZHNzrhpZj8BZwBPS5pJuGLs3mLbLCGMuDuJMFL1x5X9QZzbHB/91znnXIV4i8Q551yFeJA455yrEA8S55xzFeJB4pxzrkI8SJxzzlWIB4lzzrkK8SBxzjlXIf8PRTdhsW6e53cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x27434f980f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(contrast_1.Threshold, contrast_1[\"Test Re\"], color=\"blue\", label=\"Contrast 1.0\")\n",
    "plt.plot(contrast_15.Threshold, contrast_15[\"Test Re\"], color=\"red\", label=\"Contrast 1.5\")\n",
    "plt.plot(contrast_2.Threshold, contrast_2[\"Test Re\"], color=\"green\", label=\"Contrast 2.0\")\n",
    "plt.ylabel(\"Recall\")\n",
    "plt.xlabel(\"Threshold\")\n",
    "plt.legend()\n",
    "plt.title(\"Threshold vs Recall\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}