test_ntrees.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9877b553-b802-493d-ba45-abd7b471d16b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import contextlib\n",
    "import joblib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.metrics import roc_curve\n",
    "from tqdm.notebook import trange, tqdm\n",
    "\n",
    "from sktree.ensemble import HonestForestClassifier\n",
    "from sktree.stats import build_hyppo_oob_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4dfdd46-8a30-4ecd-980e-2be1ce1e32c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "@contextlib.contextmanager\n",
    "def tqdm_joblib(tqdm_object):\n",
    "    \"\"\"Context manager to patch joblib to report into tqdm progress bar given as argument\"\"\"\n",
    "    class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):\n",
    "        def __call__(self, *args, **kwargs):\n",
    "            tqdm_object.update(n=self.batch_size)\n",
    "            return super().__call__(*args, **kwargs)\n",
    "\n",
    "    old_batch_callback = joblib.parallel.BatchCompletionCallBack\n",
    "    joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback\n",
    "    try:\n",
    "        yield tqdm_object\n",
    "    finally:\n",
    "        joblib.parallel.BatchCompletionCallBack = old_batch_callback\n",
    "        tqdm_object.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0f4a8a6d-25e4-4bd5-8877-7a132827af9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE = \"mendseq.featurematrix.csv.pkl\"\n",
    "\n",
    "N_ESTIMATORS = list(range(100, 4001, 100))\n",
    "# N_ESTIMATORS[0] = 10\n",
    "REPS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e253450-658b-498b-8991-f23b636c5c83",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sensitivity_at_specificity(y_true, y_score, target_specificity=0.98, pos_label=1):\n",
    "    n_trees, n_samples, n_classes = y_score.shape\n",
    "\n",
    "    # Compute nan-averaged y_score along the trees axis\n",
    "    y_score_avg = np.nanmean(y_score, axis=0)\n",
    "\n",
    "    # Extract true labels and nan-averaged predicted scores for the positive class\n",
    "    y_true = y_true.ravel()\n",
    "    y_score_binary = y_score_avg[:, 1]\n",
    "\n",
    "    # Identify rows with NaN values in y_score_binary\n",
    "    nan_rows = np.isnan(y_score_binary)\n",
    "\n",
    "    # Remove NaN rows from y_score_binary and y_true\n",
    "    y_score_binary = y_score_binary[~nan_rows]\n",
    "    y_true = y_true[~nan_rows]\n",
    "\n",
    "    # Compute ROC curve\n",
    "    fpr, tpr, thresholds = roc_curve(y_true, y_score_binary, pos_label=pos_label)\n",
    "\n",
    "    # Find the threshold corresponding to the target specificity\n",
    "    index = np.argmax(fpr >= (1 - target_specificity))\n",
    "    threshold_at_specificity = thresholds[index]\n",
    "\n",
    "    # Compute sensitivity at the chosen specificity\n",
    "    # sensitivity = tpr[index]\n",
    "    # return sensitivity\n",
    "\n",
    "    # Use the threshold to classify predictions\n",
    "    y_pred_at_specificity = (y_score_binary >= threshold_at_specificity).astype(int)\n",
    "\n",
    "    # Compute sensitivity at the chosen specificity\n",
    "    sensitivity = np.sum((y_pred_at_specificity == 1) & (y_true == 1)) / np.sum(\n",
    "        y_true == 1\n",
    "    )\n",
    "\n",
    "    return sensitivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7429ee9d-966d-4ce3-ab2e-03318ab9c335",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(file=\"mendseq.featurematrix.csv.pkl\"):\n",
    "    data = np.load(file, allow_pickle=True)\n",
    "    columns_to_remove = [\n",
    "        \"Run\",\n",
    "        \"Sample\",\n",
    "        \"Library\",\n",
    "        \"Cancer Status\",\n",
    "        \"Tumor type\",\n",
    "        \"MAF\",\n",
    "        \"Stage\",\n",
    "        \"P7\",\n",
    "        \"P7 primer\",\n",
    "        \"P7 Primer\",\n",
    "        \"Library volume\",\n",
    "        \"Library Volume\",\n",
    "        \"UIDs Used\",\n",
    "        \"Avg GC\",\n",
    "        \"Library volume (uL)\",\n",
    "        \"Total Reads\",\n",
    "        \"Total Alu\",\n",
    "    ]\n",
    "    y = data[\"Cancer Status\"].to_numpy()\n",
    "    features = data.loc[:, ~data.columns.isin(columns_to_remove)].columns.tolist()\n",
    "    X = data.loc[:, features].to_numpy()\n",
    "    X = np.nan_to_num(X)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a401ae55-0d1c-4dfc-ba0a-14d46b2564ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1992, 1024)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = generate_data()\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07468b2e-679e-4f6a-bdf1-d3abb892b81d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _parallel_counts(seed):\n",
    "    X, y = generate_data()\n",
    "    est = HonestForestClassifier(\n",
    "        n_estimators=100000,\n",
    "        random_state=seed,\n",
    "        honest_fraction=0.5,\n",
    "        n_jobs=1,\n",
    "        bootstrap=True,\n",
    "        stratify=True,\n",
    "        max_samples=1.6,\n",
    "        # permute_per_tree=True,\n",
    "    )\n",
    "    _, posterior_arr = build_hyppo_oob_forest(\n",
    "        est,\n",
    "        X,\n",
    "        y,\n",
    "        verbose=False\n",
    "    )\n",
    "    sas98 = sensitivity_at_specificity(\n",
    "        y, posterior_arr, target_specificity=0.98\n",
    "    )\n",
    "    return sas98"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "80f25dac-9fe4-4389-81d3-9355bc9811e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fatal Python error: init_sys_streams: can't initialize sys standard streams\n",
      "Python runtime state: core initialized\n",
      "OSError: [Errno 9] Bad file descriptor\n",
      "\n",
      "Current thread 0x00007ff84b8207c0 (most recent call first):\n",
      "  <no Python frame>\n",
      "Fatal Python error: init_sys_streams: can't initialize sys standard streams\n",
      "Python runtime state: core initialized\n",
      "OSError: [Errno 9] Bad file descriptor\n",
      "\n",
      "Current thread 0x00007ff84b8207c0 (most recent call first):\n",
      "  <no Python frame>\n"
     ]
    },
    {
     "ename": "BrokenProcessPool",
     "evalue": "A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31m_RemoteTraceback\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;31m_RemoteTraceback\u001b[0m: \n\"\"\"\nTraceback (most recent call last):\n  File \"/Users/spanda/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/externals/loky/process_executor.py\", line 426, in _process_worker\n    call_item = call_queue.get(block=True, timeout=timeout)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/Users/spanda/miniconda3/envs/cancer/lib/python3.11/multiprocessing/queues.py\", line 122, in get\n    return _ForkingPickler.loads(res)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"<frozen importlib._bootstrap>\", line 1176, in _find_and_load\n  File \"<frozen importlib._bootstrap>\", line 1138, in _find_and_load_unlocked\n  File \"<frozen importlib._bootstrap>\", line 1078, in _find_spec\n  File \"/Users/spanda/miniconda3/envs/cancer/lib/python3.11/site-packages/_scikit_tree_editable_loader.py\", line 271, in find_spec\n    tree = self.rebuild()\n           ^^^^^^^^^^^^^^\n  File \"/Users/spanda/miniconda3/envs/cancer/lib/python3.11/site-packages/_scikit_tree_editable_loader.py\", line 312, in rebuild\n    subprocess.run(self._build_cmd, cwd=self._build_path, env=env, stdout=stdout, check=True)\n  File \"/Users/spanda/miniconda3/envs/cancer/lib/python3.11/subprocess.py\", line 571, in run\n    raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command '['/Users/spanda/miniconda3/envs/cancer/bin/ninja']' returned non-zero exit status 1.\n\"\"\"",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mBrokenProcessPool\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[23], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# with tqdm_joblib(tqdm(desc=\"Reps Histogram S@98\", total=100)) as progress_bar:\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m sas98s \u001b[38;5;241m=\u001b[39m \u001b[43mParallel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn_jobs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprefer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m    \u001b[49m\u001b[43m[\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdelayed\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_parallel_counts\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[43mseed\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mrange\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1952\u001b[0m, in \u001b[0;36mParallel.__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1946\u001b[0m \u001b[38;5;66;03m# The first item from the output is blank, but it makes the interpreter\u001b[39;00m\n\u001b[1;32m   1947\u001b[0m \u001b[38;5;66;03m# progress until it enters the Try/Except block of the generator and\u001b[39;00m\n\u001b[1;32m   1948\u001b[0m \u001b[38;5;66;03m# reach the first `yield` statement. This starts the aynchronous\u001b[39;00m\n\u001b[1;32m   1949\u001b[0m \u001b[38;5;66;03m# dispatch of the tasks to the workers.\u001b[39;00m\n\u001b[1;32m   1950\u001b[0m \u001b[38;5;28mnext\u001b[39m(output)\n\u001b[0;32m-> 1952\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m output \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_generator \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43moutput\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1595\u001b[0m, in \u001b[0;36mParallel._get_outputs\u001b[0;34m(self, iterator, pre_dispatch)\u001b[0m\n\u001b[1;32m   1592\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m\n\u001b[1;32m   1594\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backend\u001b[38;5;241m.\u001b[39mretrieval_context():\n\u001b[0;32m-> 1595\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_retrieve()\n\u001b[1;32m   1597\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mGeneratorExit\u001b[39;00m:\n\u001b[1;32m   1598\u001b[0m     \u001b[38;5;66;03m# The generator has been garbage collected before being fully\u001b[39;00m\n\u001b[1;32m   1599\u001b[0m     \u001b[38;5;66;03m# consumed. This aborts the remaining tasks if possible and warn\u001b[39;00m\n\u001b[1;32m   1600\u001b[0m     \u001b[38;5;66;03m# the user if necessary.\u001b[39;00m\n\u001b[1;32m   1601\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1699\u001b[0m, in \u001b[0;36mParallel._retrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1692\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wait_retrieval():\n\u001b[1;32m   1693\u001b[0m \n\u001b[1;32m   1694\u001b[0m     \u001b[38;5;66;03m# If the callback thread of a worker has signaled that its task\u001b[39;00m\n\u001b[1;32m   1695\u001b[0m     \u001b[38;5;66;03m# triggered an exception, or if the retrieval loop has raised an\u001b[39;00m\n\u001b[1;32m   1696\u001b[0m     \u001b[38;5;66;03m# exception (e.g. `GeneratorExit`), exit the loop and surface the\u001b[39;00m\n\u001b[1;32m   1697\u001b[0m     \u001b[38;5;66;03m# worker traceback.\u001b[39;00m\n\u001b[1;32m   1698\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_aborting:\n\u001b[0;32m-> 1699\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raise_error_fast\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1700\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m   1702\u001b[0m     \u001b[38;5;66;03m# If the next job is not ready for retrieval yet, we just wait for\u001b[39;00m\n\u001b[1;32m   1703\u001b[0m     \u001b[38;5;66;03m# async callbacks to progress.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1734\u001b[0m, in \u001b[0;36mParallel._raise_error_fast\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1730\u001b[0m \u001b[38;5;66;03m# If this error job exists, immediatly raise the error by\u001b[39;00m\n\u001b[1;32m   1731\u001b[0m \u001b[38;5;66;03m# calling get_result. This job might not exists if abort has been\u001b[39;00m\n\u001b[1;32m   1732\u001b[0m \u001b[38;5;66;03m# called directly or if the generator is gc'ed.\u001b[39;00m\n\u001b[1;32m   1733\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m error_job \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1734\u001b[0m     \u001b[43merror_job\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_result\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:736\u001b[0m, in \u001b[0;36mBatchCompletionCallBack.get_result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    730\u001b[0m backend \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparallel\u001b[38;5;241m.\u001b[39m_backend\n\u001b[1;32m    732\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m backend\u001b[38;5;241m.\u001b[39msupports_retrieve_callback:\n\u001b[1;32m    733\u001b[0m     \u001b[38;5;66;03m# We assume that the result has already been retrieved by the\u001b[39;00m\n\u001b[1;32m    734\u001b[0m     \u001b[38;5;66;03m# callback thread, and is stored internally. It's just waiting to\u001b[39;00m\n\u001b[1;32m    735\u001b[0m     \u001b[38;5;66;03m# be returned.\u001b[39;00m\n\u001b[0;32m--> 736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_return_or_raise\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    738\u001b[0m \u001b[38;5;66;03m# For other backends, the main thread needs to run the retrieval step.\u001b[39;00m\n\u001b[1;32m    739\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:754\u001b[0m, in \u001b[0;36mBatchCompletionCallBack._return_or_raise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    752\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    753\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m TASK_ERROR:\n\u001b[0;32m--> 754\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_result\n\u001b[1;32m    755\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_result\n\u001b[1;32m    756\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "\u001b[0;31mBrokenProcessPool\u001b[0m: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable."
     ]
    }
   ],
   "source": [
    "# with tqdm_joblib(tqdm(desc=\"Reps Histogram S@98\", total=100)) as progress_bar:\n",
    "sas98s = Parallel(n_jobs=2, verbose=100, prefer=None)(\n",
    "    [\n",
    "        delayed(_parallel_counts)(seed)\n",
    "        for seed in range(100)\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5da67e2b-30c4-4196-b758-489149d37623",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sas98s' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m fig, ax \u001b[38;5;241m=\u001b[39m plt\u001b[38;5;241m.\u001b[39msubplots()\n\u001b[0;32m----> 2\u001b[0m ax\u001b[38;5;241m.\u001b[39mhist(\u001b[43msas98s\u001b[49m)\n\u001b[1;32m      3\u001b[0m ax\u001b[38;5;241m.\u001b[39mset(\n\u001b[1;32m      4\u001b[0m     xlabel\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mS@98\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      5\u001b[0m     ylabel\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCounts\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      6\u001b[0m     title\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHistogram of S@98 for 10000 Trees\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      7\u001b[0m )\n\u001b[1;32m      8\u001b[0m plt\u001b[38;5;241m.\u001b[39mshow()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sas98s' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGiCAYAAADA0E3hAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAcw0lEQVR4nO3db2zdVf3A8U/b0VsItEzn2m0WKyiiAhturBYkiKk2gUz3wDjBbHPhj+AkuEZlY7CK6DoRyKIrLkwQH6ibEDDGLUOsLgapWdjWBGSDwMBNYwsT184iLWu/vweG+qvrYLf0z077eiX3wY7n3O+5Hkbf3H8tyLIsCwCABBSO9QYAAI6VcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSkXe4/OEPf4h58+bF9OnTo6CgIH75y1++5Zpt27bFRz7ykcjlcvG+970v7r///iFsFQCY6PIOl66urpg5c2Y0NTUd0/wXXnghLrvssrjkkkuitbU1vvrVr8ZVV10VjzzySN6bBQAmtoK380sWCwoK4uGHH4758+cfdc6NN94Ymzdvjqeeeqp/7POf/3wcPHgwtm7dOtRLAwAT0KSRvkBLS0vU1tYOGKurq4uvfvWrR13T3d0d3d3d/X/u6+uLV155Jd75zndGQUHBSG0VABhGWZbFoUOHYvr06VFYODxvqx3xcGlra4vy8vIBY+Xl5dHZ2Rn//ve/48QTTzxiTWNjY9x6660jvTUAYBTs378/3v3udw/LfY14uAzFihUror6+vv/PHR0dcdppp8X+/fujtLR0DHcGAByrzs7OqKysjFNOOWXY7nPEw6WioiLa29sHjLW3t0dpaemgz7ZERORyucjlckeMl5aWChcASMxwvs1jxL/HpaamJpqbmweMPfroo1FTUzPSlwYAxpm8w+Vf//pXtLa2Rmtra0T85+POra2tsW/fvoj4z8s8ixYt6p9/7bXXxt69e+Mb3/hG7NmzJ+6+++74xS9+EcuWLRueRwAATBh5h8sTTzwR5513Xpx33nkREVFfXx/nnXderFq1KiIi/v73v/dHTETEe9/73ti8eXM8+uijMXPmzLjzzjvjRz/6UdTV1Q3TQwAAJoq39T0uo6WzszPKysqio6PDe1wAIBEj8fPb7yoCAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZQwqXpqamqKqqipKSkqiuro7t27e/6fy1a9fGBz7wgTjxxBOjsrIyli1bFq+99tqQNgwATFx5h8umTZuivr4+GhoaYufOnTFz5syoq6uLl156adD5P/vZz2L58uXR0NAQu3fvjnvvvTc2bdoUN91009vePAAwseQdLnfddVdcffXVsWTJkvjQhz4U69evj5NOOinuu+++Qec//vjjceGFF8YVV1wRVVVV8alPfSouv/zyt3yWBgDgf+UVLj09PbFjx46ora397x0UFkZtbW20tLQMuuaCCy6IHTt29IfK3r17Y8uWLXHppZce9Trd3d3R2dk54AYAMCmfyQcOHIje3t4oLy8fMF5eXh579uwZdM0VV1wRBw4ciI997GORZVkcPnw4rr322jd9qaixsTFuvfXWfLYGAEwAI/6pom3btsXq1avj7rvvjp07d8ZDDz0Umzdvjttuu+2oa1asWBEdHR39t/3794/0NgGABOT1jMuUKVOiqKgo2tvbB4y3t7dHRUXFoGtuueWWWLhwYVx11VUREXHOOedEV1dXXHPNNbFy5cooLDyynXK5XORyuXy2BgBMAHk941JcXByzZ8+O5ubm/rG+vr5obm6OmpqaQde8+uqrR8RJUVFRRERkWZbvfgGACSyvZ1wiIurr62Px4sUxZ86cmDt3bqxduza6urpiyZIlERGxaNGimDFjRjQ2NkZExLx58+Kuu+6K8847L6qrq+O5556LW265JebNm9cfMAAAxyLvcFmwYEG8/PLLsWrVqmhra4tZs2bF1q1b+9+wu2/fvgHPsNx8881RUFAQN998c/ztb3+Ld73rXTFv3rz4zne+M3yPAgCYEAqyBF6v6ezsjLKysujo6IjS0tKx3g4AcAxG4ue331UEACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhhQuTU1NUVVVFSUlJVFdXR3bt29/0/kHDx6MpUuXxrRp0yKXy8WZZ54ZW7ZsGdKGAYCJa1K+CzZt2hT19fWxfv36qK6ujrVr10ZdXV0888wzMXXq1CPm9/T0xCc/+cmYOnVqPPjggzFjxoz4y1/+Eqeeeupw7B8AmEAKsizL8llQXV0d559/fqxbty4iIvr6+qKysjKuv/76WL58+RHz169fH9/73vdiz549ccIJJwxpk52dnVFWVhYdHR1RWlo6pPsAAEbXSPz8zuulop6entixY0fU1tb+9w4KC6O2tjZaWloGXfOrX/0qampqYunSpVFeXh5nn312rF69Onp7e496ne7u7ujs7BxwAwDIK1wOHDgQvb29UV5ePmC8vLw82traBl2zd+/eePDBB6O3tze2bNkSt9xyS9x5553x7W9/+6jXaWxsjLKysv5bZWVlPtsEAMapEf9UUV9fX0ydOjXuueeemD17dixYsCBWrlwZ69evP+qaFStWREdHR/9t//79I71NACABeb05d8qUKVFUVBTt7e0Dxtvb26OiomLQNdOmTYsTTjghioqK+sc++MEPRltbW/T09ERxcfERa3K5XORyuXy2BgBMAHk941JcXByzZ8+O5ubm/rG+vr5obm6OmpqaQddceOGF8dxzz0VfX1//2LPPPhvTpk0bNFoAAI4m75eK6uvrY8OGDfGTn/wkdu/eHdddd110dXXFkiVLIiJi0aJFsWLFiv751113Xbzyyitxww03xLPPPhubN2+O1atXx9KlS4fvUQAAE0Le3+OyYMGCePnll2PVqlXR1tYWs2bNiq1bt/a/YXffvn1RWPjfHqqsrIxHHnkkli1bFueee27MmDEjbrjhhrjxxhuH71EAABNC3t/jMhZ8jwsApGfMv8cFAGAsCRcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIxpDCpampKaqqqqKkpCSqq6tj+/btx7Ru48aNUVBQEPPnzx/KZQGACS7vcNm0aVPU19dHQ0ND7Ny5M2bOnBl1dXXx0ksvvem6F198Mb72ta/FRRddNOTNAgATW97hctddd8XVV18dS5YsiQ996EOxfv36OOmkk+K+++476pre3t74whe+ELfeemucfvrpb3mN7u7u6OzsHHADAMgrXHp6emLHjh1RW1v73zsoLIza2tpoaWk56rpvfetbMXXq1LjyyiuP6TqNjY1RVlbWf6usrMxnmwDAOJVXuBw4cCB6e3ujvLx8wHh5eXm0tbUNuuaxxx6Le++9NzZs2HDM11mxYkV0dHT03/bv35/PNgGAcWrSSN75oUOHYuHChbFhw4aYMmXKMa/L5XKRy+VGcGcAQIryCpcpU6ZEUVFRtLe3Dxhvb2+PioqKI+Y///zz8eKLL8a8efP6x/r6+v5z4UmT4plnnokzzjhjKPsGACagvF4qKi4ujtmzZ0dzc3P/WF9fXzQ3N0dNTc0R888666x48skno7W1tf/26U9/Oi655JJobW313hUAIC95v1RUX18fixcvjjlz5sTcuXNj7dq10dXVFUuWLImIiEWLFsWMGTOisbExSkpK4uyzzx6w/tRTT42IOGIcAOCt5B0uCxYsiJdffjlWrVoVbW1tMWvWrNi6dWv/G3b37dsXhYW+kBcAGH4FWZZlY72Jt9LZ2RllZWXR0dERpaWlY70dAOAYjMTPb0+NAADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQjCGFS1NTU1RVVUVJSUlUV1fH9u3bjzp3w4YNcdFFF8XkyZNj8uTJUVtb+6bzAQCOJu9w2bRpU9TX10dDQ0Ps3LkzZs6cGXV1dfHSSy8NOn/btm1x+eWXx+9///toaWmJysrK+NSnPhV/+9vf3vbmAYCJpSDLsiyfBdXV1XH++efHunXrIiKir68vKisr4/rrr4/ly5e/5fre3t6YPHlyrFu3LhYtWjTonO7u7uju7u7/c2dnZ1RWVkZHR0eUlpbms10AYIx0dnZGWVnZsP78zusZl56entixY0fU1tb+9w4KC6O2tjZaWlqO6T5effXVeP311+Md73jHUec0NjZGWVlZ/62ysjKfbQIA41Re4XLgwIHo7e2N8vLyAePl5eXR1tZ2TPdx4403xvTp0wfEz/9asWJFdHR09N/279+fzzYBgHFq0mhebM2aNbFx48bYtm1blJSUHHVeLpeLXC43ijsDAFKQV7hMmTIlioqKor29fcB4e3t7VFRUvOnaO+64I9asWRO//e1v49xzz81/pwDAhJfXS0XFxcUxe/bsaG5u7h/r6+uL5ubmqKmpOeq622+/PW677bbYunVrzJkzZ+i7BQAmtLxfKqqvr4/FixfHnDlzYu7cubF27dro6uqKJUuWRETEokWLYsaMGdHY2BgREd/97ndj1apV8bOf/Syqqqr63wtz8sknx8knnzyMDwUAGO/yDpcFCxbEyy+/HKtWrYq2traYNWtWbN26tf8Nu/v27YvCwv8+kfPDH/4wenp64rOf/eyA+2loaIhvfvObb2/3AMCEkvf3uIyFkfgcOAAwssb8e1wAAMaScAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkCBcAIBnCBQBIhnABAJIhXACAZAgXACAZwgUASIZwAQCSIVwAgGQIFwAgGcIFAEiGcAEAkiFcAIBkDClcmpqaoqqqKkpKSqK6ujq2b9/+pvMfeOCBOOuss6KkpCTOOeec2LJly5A2CwBMbHmHy6ZNm6K+vj4aGhpi586dMXPmzKirq4uXXnpp0PmPP/54XH755XHllVfGrl27Yv78+TF//vx46qmn3vbmAYCJpSDLsiyfBdXV1XH++efHunXrIiKir68vKisr4/rrr4/ly5cfMX/BggXR1dUVv/71r/vHPvrRj8asWbNi/fr1g16ju7s7uru7+//c0dERp512Wuzfvz9KS0vz2S4AMEY6OzujsrIyDh48GGVlZcNyn5PymdzT0xM7duyIFStW9I8VFhZGbW1ttLS0DLqmpaUl6uvrB4zV1dXFL3/5y6Nep7GxMW699dYjxisrK/PZLgBwHPjHP/4xNuFy4MCB6O3tjfLy8gHj5eXlsWfPnkHXtLW1DTq/ra3tqNdZsWLFgNg5ePBgvOc974l9+/YN2wNnaN6oZ89+jT1ncfxwFscX53H8eOMVk3e84x3Ddp95hctoyeVykcvljhgvKyvzD+FxorS01FkcJ5zF8cNZHF+cx/GjsHD4PsSc1z1NmTIlioqKor29fcB4e3t7VFRUDLqmoqIir/kAAEeTV7gUFxfH7Nmzo7m5uX+sr68vmpubo6amZtA1NTU1A+ZHRDz66KNHnQ8AcDR5v1RUX18fixcvjjlz5sTcuXNj7dq10dXVFUuWLImIiEWLFsWMGTOisbExIiJuuOGGuPjii+POO++Myy67LDZu3BhPPPFE3HPPPcd8zVwuFw0NDYO+fMTochbHD2dx/HAWxxfncfwYibPI++PQERHr1q2L733ve9HW1hazZs2K73//+1FdXR0RER//+Mejqqoq7r///v75DzzwQNx8883x4osvxvvf//64/fbb49JLLx22BwEATAxDChcAgLHgdxUBAMkQLgBAMoQLAJAM4QIAJOO4CZempqaoqqqKkpKSqK6uju3bt7/p/AceeCDOOuusKCkpiXPOOSe2bNkySjsd//I5iw0bNsRFF10UkydPjsmTJ0dtbe1bnh3HLt+/F2/YuHFjFBQUxPz580d2gxNIvmdx8ODBWLp0aUybNi1yuVyceeaZ/j01TPI9i7Vr18YHPvCBOPHEE6OysjKWLVsWr7322ijtdvz6wx/+EPPmzYvp06dHQUHBm/4Owjds27YtPvKRj0Qul4v3ve99Az6BfMyy48DGjRuz4uLi7L777sv+/Oc/Z1dffXV26qmnZu3t7YPO/+Mf/5gVFRVlt99+e/b0009nN998c3bCCSdkTz755CjvfPzJ9yyuuOKKrKmpKdu1a1e2e/fu7Itf/GJWVlaW/fWvfx3lnY8/+Z7FG1544YVsxowZ2UUXXZR95jOfGZ3NjnP5nkV3d3c2Z86c7NJLL80ee+yx7IUXXsi2bduWtba2jvLOx598z+KnP/1plsvlsp/+9KfZCy+8kD3yyCPZtGnTsmXLlo3yzsefLVu2ZCtXrsweeuihLCKyhx9++E3n7927NzvppJOy+vr67Omnn85+8IMfZEVFRdnWrVvzuu5xES5z587Nli5d2v/n3t7ebPr06VljY+Og8z/3uc9ll1122YCx6urq7Etf+tKI7nMiyPcs/tfhw4ezU045JfvJT34yUlucMIZyFocPH84uuOCC7Ec/+lG2ePFi4TJM8j2LH/7wh9npp5+e9fT0jNYWJ4x8z2Lp0qXZJz7xiQFj9fX12YUXXjii+5xojiVcvvGNb2Qf/vCHB4wtWLAgq6ury+taY/5SUU9PT+zYsSNqa2v7xwoLC6O2tjZaWloGXdPS0jJgfkREXV3dUedzbIZyFv/r1Vdfjddff31YfxPoRDTUs/jWt74VU6dOjSuvvHI0tjkhDOUsfvWrX0VNTU0sXbo0ysvL4+yzz47Vq1dHb2/vaG17XBrKWVxwwQWxY8eO/peT9u7dG1u2bPElqGNguH52j/lvhz5w4ED09vZGeXn5gPHy8vLYs2fPoGva2toGnd/W1jZi+5wIhnIW/+vGG2+M6dOnH/EPJ/kZylk89thjce+990Zra+so7HDiGMpZ7N27N373u9/FF77whdiyZUs899xz8eUvfzlef/31aGhoGI1tj0tDOYsrrrgiDhw4EB/72Mciy7I4fPhwXHvttXHTTTeNxpb5f472s7uzszP+/e9/x4knnnhM9zPmz7gwfqxZsyY2btwYDz/8cJSUlIz1diaUQ4cOxcKFC2PDhg0xZcqUsd7OhNfX1xdTp06Ne+65J2bPnh0LFiyIlStXxvr168d6axPOtm3bYvXq1XH33XfHzp0746GHHorNmzfHbbfdNtZbY4jG/BmXKVOmRFFRUbS3tw8Yb29vj4qKikHXVFRU5DWfYzOUs3jDHXfcEWvWrInf/va3ce65547kNieEfM/i+eefjxdffDHmzZvXP9bX1xcREZMmTYpnnnkmzjjjjJHd9Dg1lL8X06ZNixNOOCGKior6xz74wQ9GW1tb9PT0RHFx8YjuebwaylnccsstsXDhwrjqqqsiIuKcc86Jrq6uuOaaa2LlypVRWOi/30fL0X52l5aWHvOzLRHHwTMuxcXFMXv27Ghubu4f6+vri+bm5qipqRl0TU1NzYD5ERGPPvroUedzbIZyFhERt99+e9x2222xdevWmDNnzmhsddzL9yzOOuusePLJJ6O1tbX/9ulPfzouueSSaG1tjcrKytHc/rgylL8XF154YTz33HP98RgR8eyzz8a0adNEy9swlLN49dVXj4iTN4Iy86v6RtWw/ezO733DI2Pjxo1ZLpfL7r///uzpp5/OrrnmmuzUU0/N2trasizLsoULF2bLly/vn//HP/4xmzRpUnbHHXdku3fvzhoaGnwcepjkexZr1qzJiouLswcffDD7+9//3n87dOjQWD2EcSPfs/hfPlU0fPI9i3379mWnnHJK9pWvfCV75plnsl//+tfZ1KlTs29/+9tj9RDGjXzPoqGhITvllFOyn//859nevXuz3/zmN9kZZ5yRfe5znxurhzBuHDp0KNu1a1e2a9euLCKyu+66K9u1a1f2l7/8JcuyLFu+fHm2cOHC/vlvfBz661//erZ79+6sqakp3Y9DZ1mW/eAHP8hOO+20rLi4OJs7d272pz/9qf9/u/jii7PFixcPmP+LX/wiO/PMM7Pi4uLswx/+cLZ58+ZR3vH4lc9ZvOc978ki4ohbQ0PD6G98HMr378X/J1yGV75n8fjjj2fV1dVZLpfLTj/99Ow73/lOdvjw4VHe9fiUz1m8/vrr2Te/+c3sjDPOyEpKSrLKysrsy1/+cvbPf/5z9Dc+zvz+978f9N//b/z/v3jx4uziiy8+Ys2sWbOy4uLi7PTTT89+/OMf533dgizzXBkAkIYxf48LAMCxEi4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJCM/wM9kKRvAVrZIAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.hist(sas98s)\n",
    "# ax.set(\n",
    "#     xlabel=\"S@98\",\n",
    "#     ylabel=\"Counts\",\n",
    "#     title=\"Histogram of S@98 for 10000 Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7308b086-050b-4c5c-9634-f70ee5da77f5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# total_sas98s = []\n",
    "# for n_estimators in tqdm(N_ESTIMATORS, desc=\"Estimators\"):\n",
    "#     sas98s = []\n",
    "#     for reps in trange(REPS, desc=\"REPS\"):\n",
    "#         est = HonestForestClassifier(\n",
    "#             n_estimators=n_estimators,\n",
    "#             # random_state=seed,\n",
    "#             honest_fraction=0.5,\n",
    "#             n_jobs=-1,\n",
    "#             bootstrap=True,\n",
    "#             stratify=True,\n",
    "#             max_samples=1.6,\n",
    "#             # permute_per_tree=True,\n",
    "#         )\n",
    "        \n",
    "#         _, posterior_arr = build_hyppo_oob_forest(\n",
    "#             est,\n",
    "#             X,\n",
    "#             y,\n",
    "#             verbose=False,\n",
    "#         )\n",
    "#         sas98 = sensitivity_at_specificity(\n",
    "#             y, posterior_arr, target_specificity=0.98\n",
    "#         )\n",
    "#         sas98s.append(sas98)\n",
    "#     total_sas98s.append(sas98s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ae342888-ed14-4dfa-a50d-e29358137315",
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.array(total_sas98s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "62e12546-572d-4da4-8756-edba421dd69c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.plot(N_ESTIMATORS, np.var(total_sas98s, axis=1))\n",
    "# ax.set(\n",
    "#     xlabel=\"Number of Trees\",\n",
    "#     ylabel=\"Variance of S@98 Statistic\",\n",
    "#     title=\"Variance of S@98 Stat vs. Number of Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f3deb826-7e08-40a5-963b-4b4cb1f0a5e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.plot(N_ESTIMATORS, np.mean(total_sas98s, axis=1))\n",
    "# ax.set(\n",
    "#     xlabel=\"Number of Trees\",\n",
    "#     ylabel=\"Mean of S@98 Statistic\",\n",
    "#     title=\"Mean of S@98 Stat vs. Number of Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d232ff3e-90bc-4288-9042-c1f48a3ecdca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# total_sas98s = []\n",
    "# for seed in trange(10, desc=\"Reps S@98\"):\n",
    "#     est = HonestForestClassifier(\n",
    "#         n_estimators=N_ESTIMATORS[-1],\n",
    "#         random_state=seed,\n",
    "#         honest_fraction=0.5,\n",
    "#         n_jobs=-1,\n",
    "#         bootstrap=True,\n",
    "#         stratify=True,\n",
    "#         max_samples=1.6,\n",
    "#         # permute_per_tree=True,\n",
    "#     )\n",
    "#     _, posterior_arr = build_hyppo_oob_forest(\n",
    "#         est,\n",
    "#         X,\n",
    "#         y,\n",
    "#         verbose=False\n",
    "#     )\n",
    "\n",
    "#     sas98s = []\n",
    "#     for tree in N_ESTIMATORS:\n",
    "#         sas98 = sensitivity_at_specificity(\n",
    "#             y, posterior_arr[:tree, :, :], target_specificity=0.98\n",
    "#         )\n",
    "#         sas98s.append(sas98)\n",
    "#     total_sas98s.append(sas98s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47099010-87a6-4f75-8b92-dcd6c6d603da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.plot(N_ESTIMATORS, np.var(total_sas98s, axis=0))\n",
    "# ax.set(\n",
    "#     xlabel=\"Number of Trees\",\n",
    "#     ylabel=\"Variance of S@98 Statistic\",\n",
    "#     title=\"Variance of S@98 Stat vs. Number of Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c8528322-2ba6-42de-b0b3-728f95530fce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.plot(N_ESTIMATORS, sas98s)\n",
    "# ax.set(\n",
    "#     xlabel=\"Number of Trees\",\n",
    "#     ylabel=\"Variance of S@98 Statistic\",\n",
    "#     title=\"Variance of S@98 Stat vs. Number of Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1f0b709-50c3-4867-8382-a2a42b378bcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4154ca4153484678a3f4ce786dfa85a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Reps Histogram S@98:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# sas98s = []\n",
    "# for seed in trange(100, desc=\"Reps Histogram S@98\"):\n",
    "#     est = HonestForestClassifier(\n",
    "#         n_estimators=1600,\n",
    "#         random_state=seed,\n",
    "#         honest_fraction=0.5,\n",
    "#         n_jobs=-1,\n",
    "#         bootstrap=True,\n",
    "#         stratify=True,\n",
    "#         max_samples=1.6,\n",
    "#         # permute_per_tree=True,\n",
    "#     )\n",
    "#     _, posterior_arr = build_hyppo_oob_forest(\n",
    "#         est,\n",
    "#         X,\n",
    "#         y,\n",
    "#         verbose=False\n",
    "#     )\n",
    "#     sas98 = sensitivity_at_specificity(\n",
    "#         y, posterior_arr, target_specificity=0.98\n",
    "#     )\n",
    "#     sas98s.append(sas98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "059f638a-37a9-40d7-8baf-7446a4ada64a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHkklEQVR4nO3dd3hU1d728XvSKSkgISESulSBQKSKFAkEDiIooqJHihQfBSwISkSqJSi280AOxQI8ehQbAkcQgVCEQ1FKFBBpUiUJBklCKCEm6/2DN3MYUyDJTAr7+7mufcmsvfae31pMyO1uYzPGGAEAAFiIW0kXAAAAUNwIQAAAwHIIQAAAwHIIQAAAwHIIQAAAwHIIQAAAwHIIQAAAwHIIQAAAwHIIQAAAwHIIQLCcWrVqafDgwSVdxg1vxowZqlOnjtzd3RUWFlbS5RS7gwcPqnv37vL395fNZtOSJUtKuiQAVyEAoUxbsGCBbDabtm/fnuv6zp0769Zbby3y+6xYsUJTpkwp8n6sYtWqVXruued0++23a/78+Xr11Vfz7f/vf/9bnTp1UtWqVVW+fHnVqVNH999/v1auXHld73f+/HllZmZeV9/Vq1erQ4cOKl++vCpVqqT77rtPR48ezdHv0qVLio6OVuPGjVW+fHndfPPN6t+/v/bu3Xtd7zNo0CDt3r1br7zyij788EPddttt17VdYc2ePVv9+/dXjRo1ZLPZrhny16xZozvvvFP+/v7y9fVVeHi4Pv300xz9li1bppYtW8rHx0c1atTQ5MmT9eeff+bol5ycrBEjRigwMFAVKlRQly5dtHPnznxryP75vdZSq1atgkwFcF08SroAoLjt379fbm4Fy/4rVqxQTEwMIeg6rV27Vm5ubnr//ffl5eWVb9833nhD48aNU6dOnRQVFaXy5cvr0KFDWrNmjRYtWqQePXrkut3q1as1Z84crV27VsnJyXJ3d1ft2rV133336amnnlJwcHCObb7++mv16dNHLVu21PTp05Wamqp//OMf6tChg3bt2qXAwEB734cffljLli3T8OHD1bJlS506dUoxMTFq166ddu/erZo1a+Y5posXL2rLli2aMGGCRo0adZ2zVjSvvfaazp07p9atWys+Pj7fvvPnz9fQoUPVrVs3vfrqq3J3d9f+/ft14sQJh37ffPON+vbtq86dO2vmzJnavXu3Xn75ZZ0+fVqzZ8+298vKylKvXr30448/aty4capSpYr++c9/qnPnztqxY4duueWWXOvo2LGjPvzwQ4e2YcOGqXXr1hoxYoS9rWLFigWdDuDaDFCGzZ8/30gyP/zwQ67rO3XqZJo0aVLk9xk5cqQpyR+XtLS0EnvvwhgyZIipUKHCNftlZGQYPz8/061bt1zXJyYm5mhLS0sz/fr1MzabzfTs2dPMnDnTfP311+azzz4zkyZNMrfccosJCAgwX3zxRY5tGzdubOrVq2fS09PtbXFxccbNzc2MGTPG3nby5EkjyYwdO9Zh+7Vr1xpJ5q233sp3XMeOHTOSzIwZM/LtVxDX+gwcPXrUZGVlGWOMqVChghk0aFCu/Y4cOWLKlStnnnzyyWu+Z+PGjU3z5s1NRkaGvW3ChAnGZrOZffv22ds+/fRTI8l8/vnn9rbTp0+bgIAAM2DAgGu+z9Xyqz1bRkaGw98hUBgEIJRphQlANWvWdPgH9vLly2bKlCmmXr16xtvb21SuXNncfvvtZtWqVcYYYwYNGmQk5ViypaWlmTFjxpjq1asbLy8vU79+fTNjxgz7L6NsFy5cMKNHjzY33XSTqVixoundu7f9F+3kyZPt/SZPnmwkmb1795oBAwaYgIAAExYWZowx5scffzSDBg0ytWvXNt7e3iYoKMgMGTLEJCUlObxX9j72799vHn74YePn52eqVKliXnzxRZOVlWWOHz9u7r77buPr62uCgoLMG2+8cV3znZGRYaZNm2bq1KljvLy8TM2aNU1UVJS5dOmSvU9uczV//vxc9xcfH28kmSlTplz3+3fu3NnUqFHDfP/993n2ee2114yXl5f5+uuv7e1nzpwxksy4ceNybNOkSRMTEhJif71v375cA0x2++zZs/OsMXvur15q1qxpX79z507To0cP4+vraypUqGDuvPNOs2XLFod9ZH+u169fbx5//HETGBhoAgIC8p2bq+UXIp5//nnj5eVlkpOTjTHGnDt3Lsdn1Rhj9u7daySZmJgYh/bffvvNSDIvvfSSva1///4mKCjIZGZmOvQdMWKEKV++vMPno6C1HzlyxP538fbbb5s6deoYNzc3s2vXLmPMlb+Tfv36mUqVKhlvb28THh5uli5dmmO/Z8+eNU899ZT957Ru3bpm+vTpOWr+5JNPTMuWLU3FihWNr6+vufXWW80777xz3fWj7OAUGG4IKSkpSkpKytGekZFxzW2nTJmi6Oho+6H31NRUbd++XTt37lS3bt302GOP6dSpU1q9enWOw/XGGN19991at26dhg4dqrCwMH377bcaN26cfvvtN7399tv2voMHD9Znn32mRx55RG3bttWGDRvUq1evPOvq37+/brnlFr366qsyxki6ctrn119/1ZAhQxQcHKy9e/dq3rx52rt3r7Zu3SqbzeawjwceeECNGjXS9OnTtXz5cr388suqXLmy5s6dqzvvvFOvvfaa/vWvf2ns2LFq1aqVOnbsmO9cDRs2TAsXLtR9992nZ599Vtu2bVN0dLT27dunr776SpL04Ycfat68efr+++/13nvvSZLat2+f6/6qVq2qcuXK6d///rdGjx6typUr5/v+0dHR2r9/v3bs2KFq1apJunL65eLFi6pQoYKysrKUnJys5557Tr6+vnr00Ud16NAh+fr6Kj09XZJUrly5HPstX7689u7dq4SEBAUHB6tu3bqqXr263nzzTTVo0EAtWrTQqVOn9Nxzz6l27dp68MEH86zx3nvvVUBAgJ555hkNGDBAf/vb3+yncPbu3as77rhDfn5+eu655+Tp6am5c+eqc+fO2rBhg9q0aeOwryeeeEKBgYGaNGmSzp8/n+/cXK81a9aoYcOGWrFihf1zWqlSJY0cOVJTp061nx7etWuXJOW4dikkJETVq1e3r8/u27Jlyxynllu3bq158+bpwIEDatq0aZHqnj9/vi5duqQRI0bI29tblStX1t69e3X77bfr5ptv1vjx41WhQgV99tln6tu3r7788kvdc889kqQLFy6oU6dO+u233/TYY4+pRo0a2rx5s6KiohQfH6933nlH0pWfrwEDBqhr16567bXXJEn79u3Tf/7zHz311FNFqh+lUEknMKAosv9POb/lWkeAmjdvbnr16pXv++R1CmzJkiVGknn55Zcd2u+77z5js9nMoUOHjDHG7Nixw0gyTz/9tEO/wYMH53kEKLdTBxcuXMjR9sknnxhJ5rvvvsuxjxEjRtjb/vzzT1O9enVjs9nM9OnT7e1nz5415cqVu+Zph7i4OCPJDBs2zKF97NixRpJZu3atvW3QoEHXdQrMGGMmTZpkJJkKFSqYnj17mldeecXs2LEjR7+UlBTj5+dnlixZYm+bN2+eqVSpkv3v+csvv3T4e2rZsqWZN2+eMcaYzMxMExAQYLp27eqw36SkJFOhQgUjyWzfvt3evm3bNlO3bl2Hz1J4eLiJj4+/5piuPmpxtb59+xovLy9z+PBhe9upU6eMr6+v6dixo70t+3PdoUMH8+eff17z/f4qvyNAfn5+9qMlEydONF988YV56KGHjCQzfvx4e78ZM2YYSeb48eM59tGqVSvTtm1bh/d79NFHc/Rbvny5kWRWrlxZ6Nqz59LPz8+cPn3aoW/Xrl1N06ZNHY4wZWVlmfbt25tbbrnF3vbSSy+ZChUqmAMHDjhsP378eOPu7m4f41NPPWX8/PwKNecoe7gLDDeEmJgYrV69OsfSrFmza24bEBCgvXv36uDBgwV+3xUrVsjd3V1PPvmkQ/uzzz4rY4y++eYbSbLfzfTEE0849Bs9enSe+/6f//mfHG1XH724dOmSkpKS1LZtW0nK9Y6bYcOG2f/s7u6u2267TcYYDR061N4eEBCgBg0a6Ndff82zFunKWCVpzJgxDu3PPvusJGn58uX5bp+XqVOn6uOPP1aLFi307bffasKECQoPD1fLli21b98+e79Vq1apcuXKuvvuuyVdGe9jjz2mfv366auvvtIDDzyg4cOHO+y7T58+Wr9+vSTJzc1Njz32mGJjYxUVFaWDBw9qx44duv/++3X58mVJVy5ezlapUiWFhYVp/PjxWrJkid544w0dPXpU/fv316VLlwo8zszMTK1atUp9+/ZVnTp17O3VqlXTQw89pE2bNik1NdVhm+HDh8vd3b3A75WftLQ0nT17VlOnTtW0adPUr18//etf/1KPHj30j3/8Q+fOnZP037nw9vbOsQ8fHx+Hubp48WKe/a7eV1H069fP4SL1P/74Q2vXrtX999+vc+fOKSkpSUlJSTpz5owiIyN18OBB/fbbb5Kkzz//XHfccYcqVapk75eUlKSIiAhlZmbqu+++k3TlZ+H8+fNavXp1ketF6UcAwg2hdevWioiIyLFUqlTpmttOmzZNycnJql+/vpo2bapx48bpp59+uq73PXbsmEJCQuTr6+vQ3qhRI/v67P+6ubmpdu3aDv3q1auX577/2le68o/+U089paCgIJUrV06BgYH2fikpKTn616hRw+G1v7+/fHx8VKVKlRztZ8+ezbOWq8fw15qDg4MVEBBgH2thDBgwQBs3btTZs2e1atUqPfTQQ9q1a5d69+5tDxs7duxQp06d7Kf53nvvPXXu3Fnvvvuu+vbtq4kTJ+YIlEFBQfr999/tr6dNm6ahQ4fq9ddfV/369XXbbbfJw8PDHgizT1WlpKTojjvuULt27RQdHa0+ffro2Wef1ZdffqlNmzZp/vz5BR7j77//rgsXLqhBgwY51jVq1EhZWVk57sLK7TNQVNkhesCAAQ7tAwYM0MWLF+2ntrL7ZZ86vNqlS5ccwni5cuXy7Hf1vorir3Nx6NAhGWM0ceJEBQYGOiyTJ0+WJJ0+fVrSlWcyrVy5Mke/iIgIh35PPPGE6tevr549e6p69ep69NFHr/tRDCh7uAYIltexY0cdPnxYS5cu1apVq/Tee+/p7bff1pw5cxyOoBS33H5p3H///dq8ebPGjRunsLAwVaxYUVlZWerRo4eysrJy9M/t6EFeRxTM/7/O6Fr+ep2RM/n5+albt27q1q2bPD09tXDhQm3btk2dOnXSmTNnFBISYu979OhRtWrVymH71q1bO7w+ceKEbrrpJvtrLy8vvffee3rllVd04MABBQUFqX79+nrooYccwt2XX36pxMRE+9GmbJ06dZKfn5/+85//6PHHH3f28HNwRnD4q5CQEB08eFBBQUEO7VWrVpUkexDOvsYqPj5eoaGhDn3j4+Md5rpatWq53nqf3Xb131th/XUusj/vY8eOVWRkZK7bZP99ZmVlqVu3bnruuedy7Ve/fn1JV+YgLi5O3377rb755ht98803mj9/vgYOHKiFCxcWeQwoXQhAgKTKlStryJAhGjJkiNLS0tSxY0dNmTLFHoDy+qVfs2ZNrVmzRufOnXM4CvTLL7/Y12f/NysrS0eOHHF4JsqhQ4euu8azZ88qNjZWU6dO1aRJk+zthTl1VxjZYzh48KD9CJckJSYmKjk5Od/n4hTGbbfdpoULF9p/ifr5+Tkc5QoODtbhw4cdtrn6NN6lS5f04YcfOsxVtqCgIHsAyMzM1Pr169WmTRv7EaDExET7uqsZY5SZmZnrgwCvJTAwUOXLl9f+/ftzrPvll1/k5uaWI2i4Qnh4uP300NWn4k6dOmWvU5L96d3bt293CDunTp3SyZMnHZ7TExYWpo0bNyorK8vhQuht27apfPny9oDhTNm1e3p62o/k5KVu3bpKS0u7Zj/pSkju3bu3evfuraysLD3xxBOaO3euJk6cmO8RW5Q9nAKD5Z05c8bhdcWKFVWvXj2HQ/oVKlSQdOVpt1f729/+pszMTM2aNcuh/e2335bNZlPPnj0lyf5/qP/85z8d+s2cOfO668w+cvPXIzXZd7C42t/+9rdc3++tt96SpHzvaMvLhQsXtGXLllzXZV8/lX3KqFGjRtq2bZt9/T333KOvvvpKMTExOnbsmFasWGF/4vTGjRvVvXt3VapUSX//+9/zreGNN95QfHy8/Vom6b9HBBYtWuTQd9myZTp//rxatGhRwJFe+fvr3r27li5d6vDk6cTERH388cfq0KGD/Pz8CrzfgnrggQckSe+//769LSsrS/Pnz1flypUVHh4uSWrSpIkaNmyoefPmOQTB2bNny2az6b777rO33XfffUpMTNTixYvtbUlJSfr888/Vu3fvXK8PKqqqVauqc+fOmjt3bq5Hn64+9Xn//fdry5Yt+vbbb3P0S05Otgfav/5b4ObmZr+OMLdTfCjbOAIEy2vcuLE6d+6s8PBwVa5cWdu3b9cXX3zh8ATf7F8KTz75pCIjI+Xu7q4HH3xQvXv3VpcuXTRhwgQdPXpUzZs316pVq7R06VI9/fTTqlu3rn37fv366Z133tGZM2fst8EfOHBA0vWdVvLz81PHjh31+uuvKyMjQzfffLNWrVqlI0eOuGBWcmrevLkGDRqkefPmKTk5WZ06ddL333+vhQsXqm/fvurSpUuB93nhwgW1b99ebdu2VY8ePRQaGqrk5GQtWbJEGzduVN++fe1ho0ePHvqf//kf7dq1Sy1atFDv3r312GOPadSoURo1apTKly+vqVOnaty4cercubPuu+8+LV682OGX70cffaQvv/xSHTt2VMWKFbVmzRp99tlnGjZsmPr162fv17t3bzVp0kTTpk3TsWPH1LZtWx06dEizZs1StWrVHC4iL4iXX37Z/lUcTzzxhDw8PDR37lylp6fr9ddfL9Q+s/373//Wjz/+KOnK4x9++uknvfzyy5Kku+++2/6LvE+fPuratauio6OVlJSk5s2ba8mSJdq0aZPmzp3rMF8zZszQ3Xffre7du+vBBx/Unj17NGvWLA0bNszhKOB9992ntm3basiQIfr555/tT4LOzMzU1KlTizSu/MTExKhDhw5q2rSphg8frjp16igxMVFbtmzRyZMn7fMxbtw4LVu2THfddZcGDx6s8PBwnT9/Xrt379YXX3yho0ePqkqVKho2bJj++OMP3XnnnapevbqOHTummTNnKiwszGG8uEGU5C1oQFE540GIL7/8smndurUJCAgw5cqVMw0bNjSvvPKKuXz5sr3Pn3/+aUaPHm0CAwONzWZzuNX63Llz5plnnjEhISHG09PT3HLLLbk+CPH8+fNm5MiRpnLlyqZixYqmb9++Zv/+/UaSw23p2bew//777znGc/LkSXPPPfeYgIAA4+/vb/r3729OnTqV5630f91HXrenX+8TszMyMszUqVNN7dq1jaenpwkNDc3xIMT83ie3/b377rumb9++pmbNmsbb29uUL1/etGjRwsyYMSPH034HDRpk2rRp49B++PBhs3HjRnP27Flz8eJFs2XLFvtD/v5q27ZtpmPHjqZSpUrGx8fHNG/e3MyZMyfXBwH+8ccf5plnnjH169c33t7epkqVKubBBx80v/766zXHlddt8MZceRBiZGSkqVixoilfvrzp0qWL2bx5s0Ofa32uc5PXAzuVy4Moz507Z5566ikTHBxsvLy8TNOmTc1HH32U636/+uorExYWZry9vU316tXNiy++6PCzke2PP/4wQ4cONTfddJMpX7686dSpU4Hqz5bfgxBzc/jwYTNw4EATHBxsPD09zc0332zuuuuuHE8CP3funImKijL16tUzXl5epkqVKqZ9+/bmjTfesI/niy++MN27dzdVq1Y1Xl5epkaNGuaxxx67rkcfoOyxGXOdVz4CcLq4uDi1aNFCH330kR5++OGSLqfUS0pKUnh4uG699VZ98sknuZ4yyszM1FdffeVwigYA/ooABBSTixcv5riTZfDgwfrwww919OjRYrkA9kZw4MAB9erVS6mpqRo1apS6deumkJAQpaamatOmTZo1a5YSEhK0c+fOHI8BAIBsBCCgmEydOlU7duxQly5d5OHhYb/NdsSIEZo7d25Jl1emnDt3TjNmzNB7773ncAGsr6+vHn74YU2aNMl+GzcA5IYABBST1atXa+rUqfr555+VlpamGjVq6JFHHtGECRPk4cH9CIVhjNGhQ4eUkJAgPz8/NWrUSF5eXiVdFoAygAAEAAAsh+cAAQAAyyEAAQAAy+HCg1xkZWXp1KlT8vX1den3HgEAAOcxxujcuXMKCQlx+FqW3BCAcnHq1CluSQYAoIw6ceKEqlevnm8fAlAusr/U8sSJE8Xy3TwAAKDoUlNTFRoa6vDl1HkhAOUi+7SXn58fAQgAgDLmei5f4SJoAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOQQgAABgOR4lXQCAsq3W+OUlXYIlHJ3eq6RLAG4oHAECAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWQwACAACWU6IB6LvvvlPv3r0VEhIim82mJUuWOKy32Wy5LjNmzMhzn1OmTMnRv2HDhi4eCQAAKEtKNACdP39ezZs3V0xMTK7r4+PjHZYPPvhANptN/fr1y3e/TZo0cdhu06ZNrigfAACUUR4l+eY9e/ZUz54981wfHBzs8Hrp0qXq0qWL6tSpk+9+PTw8cmwLAACQrcxcA5SYmKjly5dr6NCh1+x78OBBhYSEqE6dOnr44Yd1/PjxYqgQAACUFSV6BKggFi5cKF9fX91777359mvTpo0WLFigBg0aKD4+XlOnTtUdd9yhPXv2yNfXN9dt0tPTlZ6ebn+dmprq1NoBAEDpUmYC0AcffKCHH35YPj4++fa7+pRas2bN1KZNG9WsWVOfffZZnkePoqOjNXXqVKfWCwAASq8ycQps48aN2r9/v4YNG1bgbQMCAlS/fn0dOnQozz5RUVFKSUmxLydOnChKuQAAoJQrEwHo/fffV3h4uJo3b17gbdPS0nT48GFVq1Ytzz7e3t7y8/NzWAAAwI2rRANQWlqa4uLiFBcXJ0k6cuSI4uLiHC5aTk1N1eeff57n0Z+uXbtq1qxZ9tdjx47Vhg0bdPToUW3evFn33HOP3N3dNWDAAJeOBQAAlB0leg3Q9u3b1aVLF/vrMWPGSJIGDRqkBQsWSJIWLVokY0yeAebw4cNKSkqyvz558qQGDBigM2fOKDAwUB06dNDWrVsVGBjouoEAAIAyxWaMMSVdRGmTmpoqf39/paSkcDoMuIZa45eXdAmWcHR6r5IuASj1CvL7u0xcAwQAAOBMBCAAAGA5BCAAAGA5ZeZBiIAVcD0NABQPjgABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLKdEA9N1336l3794KCQmRzWbTkiVLHNYPHjxYNpvNYenRo8c19xsTE6NatWrJx8dHbdq00ffff++iEQAAgLKoRAPQ+fPn1bx5c8XExOTZp0ePHoqPj7cvn3zySb77/PTTTzVmzBhNnjxZO3fuVPPmzRUZGanTp087u3wAAFBGeZTkm/fs2VM9e/bMt4+3t7eCg4Ove59vvfWWhg8friFDhkiS5syZo+XLl+uDDz7Q+PHji1QvAAC4MZT6a4DWr1+vqlWrqkGDBnr88cd15syZPPtevnxZO3bsUEREhL3Nzc1NERER2rJlS57bpaenKzU11WEBAAA3rhI9AnQtPXr00L333qvatWvr8OHDeuGFF9SzZ09t2bJF7u7uOfonJSUpMzNTQUFBDu1BQUH65Zdf8nyf6OhoTZ061en1A4Cz1Bq/vKRLKLCj03uVdAlAnkp1AHrwwQftf27atKmaNWumunXrav369eratavT3icqKkpjxoyxv05NTVVoaKjT9g8AAEqXUn8K7Gp16tRRlSpVdOjQoVzXV6lSRe7u7kpMTHRoT0xMzPc6Im9vb/n5+TksAADgxlWmAtDJkyd15swZVatWLdf1Xl5eCg8PV2xsrL0tKytLsbGxateuXXGVCQAASrkSDUBpaWmKi4tTXFycJOnIkSOKi4vT8ePHlZaWpnHjxmnr1q06evSoYmNj1adPH9WrV0+RkZH2fXTt2lWzZs2yvx4zZozeffddLVy4UPv27dPjjz+u8+fP2+8KAwAAKNFrgLZv364uXbrYX2dfhzNo0CDNnj1bP/30kxYuXKjk5GSFhISoe/fueumll+Tt7W3f5vDhw0pKSrK/fuCBB/T7779r0qRJSkhIUFhYmFauXJnjwmgAAGBdNmOMKekiSpvU1FT5+/srJSWF64FQrMrinT5AXrgLDMWtIL+/y9Q1QAAAAM5AAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZDAAIAAJZTogHou+++U+/evRUSEiKbzaYlS5bY12VkZOj5559X06ZNVaFCBYWEhGjgwIE6depUvvucMmWKbDabw9KwYUMXjwQAAJQlJRqAzp8/r+bNmysmJibHugsXLmjnzp2aOHGidu7cqcWLF2v//v26++67r7nfJk2aKD4+3r5s2rTJFeUDAIAyyqMk37xnz57q2bNnruv8/f21evVqh7ZZs2apdevWOn78uGrUqJHnfj08PBQcHOzUWgEAwI2jTF0DlJKSIpvNpoCAgHz7HTx4UCEhIapTp44efvhhHT9+PN/+6enpSk1NdVgAAMCNq8wEoEuXLun555/XgAED5Ofnl2e/Nm3aaMGCBVq5cqVmz56tI0eO6I477tC5c+fy3CY6Olr+/v72JTQ01BVDAAAApUSZCEAZGRm6//77ZYzR7Nmz8+3bs2dP9e/fX82aNVNkZKRWrFih5ORkffbZZ3luExUVpZSUFPty4sQJZw8BAACUIiV6DdD1yA4/x44d09q1a/M9+pObgIAA1a9fX4cOHcqzj7e3t7y9vYtaKgAAKCNK9RGg7PBz8OBBrVmzRjfddFOB95GWlqbDhw+rWrVqLqgQAACURSUagNLS0hQXF6e4uDhJ0pEjRxQXF6fjx48rIyND9913n7Zv365//etfyszMVEJCghISEnT58mX7Prp27apZs2bZX48dO1YbNmzQ0aNHtXnzZt1zzz1yd3fXgAEDint4AACglCrRU2Dbt29Xly5d7K/HjBkjSRo0aJCmTJmiZcuWSZLCwsIctlu3bp06d+4sSTp8+LCSkpLs606ePKkBAwbozJkzCgwMVIcOHbR161YFBga6djAAAKDMKNEA1LlzZxlj8lyf37psR48edXi9aNGiopYFAABucKX6GiAAAABXIAABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLKVQA2rlzp3bv3m1/vXTpUvXt21cvvPCCLl++7LTiAAAAXKFQAeixxx7TgQMHJEm//vqrHnzwQZUvX16ff/65nnvuOacWCAAA4GyFCkAHDhxQWFiYJOnzzz9Xx44d9fHHH2vBggX68ssvnVkfAACA03kUZiNjjLKysiRJa9as0V133SVJCg0NVVJSkvOqA4qg1vjlJV0CAKCUKtQRoNtuu00vv/yyPvzwQ23YsEG9evWSJB05ckRBQUFOLRAAAMDZChWA3n77be3cuVOjRo3ShAkTVK9ePUnSF198ofbt2zu1QAAAAGcr1Cmw5s2bO9wFlm3GjBny8CjULgEAAIpNoY4A1alTR2fOnMnRfunSJdWvX7/IRQEAALhSoQLQ0aNHlZmZmaM9PT1dJ0+eLHJRAAAArlSg81XLli2z//nbb7+Vv7+//XVmZqZiY2NVu3Zt51UHAADgAgUKQH379pUk2Ww2DRo0yGGdp6enatWqpTfffNNpxQEAALhCgQJQ9rN/ateurR9++EFVqlRxSVEAAACuVKhbto4cOeLsOgAAAIpNoe9Zj42NVWxsrE6fPm0/MpTtgw8+KHJhAAAArlKoADR16lRNmzZNt912m6pVqyabzebsugAAAFymUAFozpw5WrBggR555BFn1wMAAOByhXoO0OXLl/nKCwAAUGYVKgANGzZMH3/8cZHf/LvvvlPv3r0VEhIim82mJUuWOKw3xmjSpEmqVq2aypUrp4iICB08ePCa+42JiVGtWrXk4+OjNm3a6Pvvvy9yrQAA4MZRqFNgly5d0rx587RmzRo1a9ZMnp6eDuvfeuut69rP+fPn1bx5cz366KO69957c6x//fXX9b//+79auHChateurYkTJyoyMlI///yzfHx8ct3np59+qjFjxmjOnDlq06aN3nnnHUVGRmr//v2qWrVqwQcLAABuODZjjCnoRl26dMl7hzab1q5dW/BCbDZ99dVX9octGmMUEhKiZ599VmPHjpUkpaSkKCgoSAsWLNCDDz6Y637atGmjVq1aadasWZKuPLsoNDRUo0eP1vjx46+rltTUVPn7+yslJUV+fn4FHgtKh1rjl5d0CYClHZ3eq6RLgMUU5Pd3oY4ArVu3rlCFFcSRI0eUkJCgiIgIe5u/v7/atGmjLVu25BqALl++rB07digqKsre5ubmpoiICG3ZsiXP90pPT1d6err9dWpqqpNGAQAASqNCXQNUHBISEiRJQUFBDu1BQUH2dX+VlJSkzMzMAm0jSdHR0fL397cvoaGhRaweAACUZoU6AtSlS5d8n/1TmFNgJSkqKkpjxoyxv05NTSUEAQBwAytUAAoLC3N4nZGRobi4OO3ZsyfHl6QWVnBwsCQpMTFR1apVs7cnJibmeP9sVapUkbu7uxITEx3aExMT7fvLjbe3t7y9vYteNAAAKBMKFYDefvvtXNunTJmitLS0IhWUrXbt2goODlZsbKw98KSmpmrbtm16/PHHc93Gy8tL4eHhio2NtV9MnZWVpdjYWI0aNcopdQEAgLLPqdcA/f3vfy/Q94ClpaUpLi5OcXFxkq5c+BwXF6fjx4/LZrPp6aef1ssvv6xly5Zp9+7dGjhwoEJCQuzhRpK6du1qv+NLksaMGaN3331XCxcu1L59+/T444/r/PnzGjJkiLOGCQAAyrhCfxlqbrZs2ZLn83lys337dodb6rOvwxk0aJAWLFig5557TufPn9eIESOUnJysDh06aOXKlQ7vcfjwYSUlJdlfP/DAA/r99981adIkJSQkKCwsTCtXrsxxYTQAALCuQj0H6K8PLTTGKD4+Xtu3b9fEiRM1efJkpxVYEngO0I2B5wABJYvnAKG4ufw5QP7+/g6v3dzc1KBBA02bNk3du3cvzC4BAACKTaEC0Pz5851dBwAAQLEp0jVAO3bs0L59+yRJTZo0UYsWLZxSFAAAgCsVKgCdPn1aDz74oNavX6+AgABJUnJysrp06aJFixYpMDDQmTUCAAA4VaFugx89erTOnTunvXv36o8//tAff/yhPXv2KDU1VU8++aSzawQAAHCqQh0BWrlypdasWaNGjRrZ2xo3bqyYmBguggYAAKVeoY4AZWVlydPTM0e7p6ensrKyilwUAACAKxUqAN1555166qmndOrUKXvbb7/9pmeeeUZdu3Z1WnEAAACuUKgANGvWLKWmpqpWrVqqW7eu6tatq9q1ays1NVUzZ850do0AAABOVahrgEJDQ7Vz506tWbNGv/zyiySpUaNGioiIcGpxAAAArlCgI0Br165V48aNlZqaKpvNpm7dumn06NEaPXq0WrVqpSZNmmjjxo2uqhUAAMApChSA3nnnHQ0fPjzX79fw9/fXY489prfeestpxQEAALhCgU6B/fjjj3rttdfyXN+9e3e98cYbRS4KpQ9fLAoAuJEU6AhQYmJirre/Z/Pw8NDvv/9e5KIAAABcqUAB6Oabb9aePXvyXP/TTz+pWrVqRS4KAADAlQoUgP72t79p4sSJunTpUo51Fy9e1OTJk3XXXXc5rTgAAABXKNA1QC+++KIWL16s+vXra9SoUWrQoIEk6ZdfflFMTIwyMzM1YcIElxQKAADgLAUKQEFBQdq8ebMef/xxRUVFyRgjSbLZbIqMjFRMTIyCgoJcUigAAICzFPhBiDVr1tSKFSt09uxZHTp0SMYY3XLLLapUqZIr6gMAAHC6Qj0JWpIqVaqkVq1aObMWAACAYlGo7wIDAAAoywhAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAckp9AKpVq5ZsNluOZeTIkbn2X7BgQY6+Pj4+xVw1AAAozTxKuoBr+eGHH5SZmWl/vWfPHnXr1k39+/fPcxs/Pz/t37/f/tpms7m0RgAAULaU+gAUGBjo8Hr69OmqW7euOnXqlOc2NptNwcHBri4NAACUUaX+FNjVLl++rI8++kiPPvpovkd10tLSVLNmTYWGhqpPnz7au3dvMVYJAABKuzIVgJYsWaLk5GQNHjw4zz4NGjTQBx98oKVLl+qjjz5SVlaW2rdvr5MnT+a5TXp6ulJTUx0WAABw4ypTAej9999Xz549FRISkmefdu3aaeDAgQoLC1OnTp20ePFiBQYGau7cuXluEx0dLX9/f/sSGhrqivIBAEApUWYC0LFjx7RmzRoNGzasQNt5enqqRYsWOnToUJ59oqKilJKSYl9OnDhR1HIBAEApVmYC0Pz581W1alX16tWrQNtlZmZq9+7dqlatWp59vL295efn57AAAIAbV5kIQFlZWZo/f74GDRokDw/HG9cGDhyoqKgo++tp06Zp1apV+vXXX7Vz5079/e9/17Fjxwp85AgAANy4Sv1t8JK0Zs0aHT9+XI8++miOdcePH5eb239z3NmzZzV8+HAlJCSoUqVKCg8P1+bNm9W4cePiLBkAAJRiNmOMKekiSpvU1FT5+/srJSWF02H/X63xy0u6BABlzNHpBbtkASiqgvz+LhOnwAAAAJyJAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACyHAAQAACynVAegKVOmyGazOSwNGzbMd5vPP/9cDRs2lI+Pj5o2baoVK1YUU7UAAKCsKNUBSJKaNGmi+Ph4+7Jp06Y8+27evFkDBgzQ0KFDtWvXLvXt21d9+/bVnj17irFiAABQ2pX6AOTh4aHg4GD7UqVKlTz7/uMf/1CPHj00btw4NWrUSC+99JJatmypWbNmFWPFAACgtCv1AejgwYMKCQlRnTp19PDDD+v48eN59t2yZYsiIiIc2iIjI7Vly5Z83yM9PV2pqakOCwAAuHGV6gDUpk0bLViwQCtXrtTs2bN15MgR3XHHHTp37lyu/RMSEhQUFOTQFhQUpISEhHzfJzo6Wv7+/vYlNDTUaWMAAAClT6kOQD179lT//v3VrFkzRUZGasWKFUpOTtZnn33m1PeJiopSSkqKfTlx4oRT9w8AAEoXj5IuoCACAgJUv359HTp0KNf1wcHBSkxMdGhLTExUcHBwvvv19vaWt7e30+oEAAClW6k+AvRXaWlpOnz4sKpVq5br+nbt2ik2NtahbfXq1WrXrl1xlAcAAMqIUh2Axo4dqw0bNujo0aPavHmz7rnnHrm7u2vAgAGSpIEDByoqKsre/6mnntLKlSv15ptv6pdfftGUKVO0fft2jRo1qqSGAAAASqFSfQrs5MmTGjBggM6cOaPAwEB16NBBW7duVWBgoCTp+PHjcnP7b4Zr3769Pv74Y7344ot64YUXdMstt2jJkiW69dZbS2oIAACgFLIZY0xJF1HapKamyt/fXykpKfLz8yvpckqFWuOXl3QJAMqYo9N7lXQJsJiC/P4u1afAAAAAXIEABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALMejpAsAANyYao1fXtIlFNjR6b1KuoQCY54LhyNAAADAcghAAADAcghAAADAcghAAADAckp1AIqOjlarVq3k6+urqlWrqm/fvtq/f3++2yxYsEA2m81h8fHxKaaKAQBAWVCqA9CGDRs0cuRIbd26VatXr1ZGRoa6d++u8+fP57udn5+f4uPj7cuxY8eKqWIAAFAWlOrb4FeuXOnwesGCBapatap27Nihjh075rmdzWZTcHCwq8sDAABlVKk+AvRXKSkpkqTKlSvn2y8tLU01a9ZUaGio+vTpo7179+bbPz09XampqQ4LAAC4cZWZAJSVlaWnn35at99+u2699dY8+zVo0EAffPCBli5dqo8++khZWVlq3769Tp48mec20dHR8vf3ty+hoaGuGAIAACglykwAGjlypPbs2aNFixbl269du3YaOHCgwsLC1KlTJy1evFiBgYGaO3dunttERUUpJSXFvpw4ccLZ5QMAgFKkVF8DlG3UqFH6+uuv9d1336l69eoF2tbT01MtWrTQoUOH8uzj7e0tb2/vopYJAADKiFJ9BMgYo1GjRumrr77S2rVrVbt27QLvIzMzU7t371a1atVcUCEAACiLSvURoJEjR+rjjz/W0qVL5evrq4SEBEmSv7+/ypUrJ0kaOHCgbr75ZkVHR0uSpk2bprZt26pevXpKTk7WjBkzdOzYMQ0bNqzExgEAAEqXUh2AZs+eLUnq3LmzQ/v8+fM1ePBgSdLx48fl5vbfA1lnz57V8OHDlZCQoEqVKik8PFybN29W48aNi6tsAABQypXqAGSMuWaf9evXO7x+++239fbbb7uoIgAAcCMo1dcAAQAAuAIBCAAAWA4BCAAAWE6pvgboRlVr/PKSLgEAkAv+fbYOjgABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLIQABAADLKRMBKCYmRrVq1ZKPj4/atGmj77//Pt/+n3/+uRo2bCgfHx81bdpUK1asKKZKAQBAWVDqA9Cnn36qMWPGaPLkydq5c6eaN2+uyMhInT59Otf+mzdv1oABAzR06FDt2rVLffv2Vd++fbVnz55irhwAAJRWNmOMKeki8tOmTRu1atVKs2bNkiRlZWUpNDRUo0eP1vjx43P0f+CBB3T+/Hl9/fXX9ra2bdsqLCxMc+bMua73TE1Nlb+/v1JSUuTn5+ecgVyl1vjlTt8nAABlxdHpvVyy34L8/i7VR4AuX76sHTt2KCIiwt7m5uamiIgIbdmyJddttmzZ4tBfkiIjI/PsDwAArMejpAvIT1JSkjIzMxUUFOTQHhQUpF9++SXXbRISEnLtn5CQkOf7pKenKz093f46JSVF0pUk6QpZ6Rdcsl8AAMoCV/1+zd7v9ZzcKtUBqLhER0dr6tSpOdpDQ0NLoBoAAG5s/u+4dv/nzp2Tv79/vn1KdQCqUqWK3N3dlZiY6NCemJio4ODgXLcJDg4uUH9JioqK0pgxY+yvs7Ky9Mcff+imm26SzWbLt8bU1FSFhobqxIkTLrleqCxhLq5gHq5gHv6LubiCefgv5uIKZ8+DMUbnzp1TSEjINfuW6gDk5eWl8PBwxcbGqm/fvpKuhJPY2FiNGjUq123atWun2NhYPf300/a21atXq127dnm+j7e3t7y9vR3aAgICClSrn5+fpT/EV2MurmAermAe/ou5uIJ5+C/m4gpnzsO1jvxkK9UBSJLGjBmjQYMG6bbbblPr1q31zjvv6Pz58xoyZIgkaeDAgbr55psVHR0tSXrqqafUqVMnvfnmm+rVq5cWLVqk7du3a968eSU5DAAAUIqU+gD0wAMP6Pfff9ekSZOUkJCgsLAwrVy50n6h8/Hjx+Xm9t+b2dq3b6+PP/5YL774ol544QXdcsstWrJkiW699daSGgIAAChlSn0AkqRRo0blecpr/fr1Odr69++v/v37u7iqK7y9vTV58uQcp9CsiLm4gnm4gnn4L+biCubhv5iLK0pyHkr9gxABAACcrVQ/CBEAAMAVCEAAAMByCEAAAMByCEAAAMByCEC5iImJUa1ateTj46M2bdro+++/z7Pvu+++qzvuuEOVKlVSpUqVFBERkaP/4MGDZbPZHJYePXq4ehhF5ux5kKR9+/bp7rvvlr+/vypUqKBWrVrp+PHjrhyGUzh7Lv76echeZsyY4eqhFImz5yEtLU2jRo1S9erVVa5cOTVu3Fhz5sxx9TCKzNnzkJiYqMGDByskJETly5dXjx49dPDgQVcPwykKMheLFy/WbbfdpoCAAFWoUEFhYWH68MMPHfoYYzRp0iRVq1ZN5cqVU0RERJmYC2fPw+LFi9W9e3f7NxLExcW5eATO48y5yMjI0PPPP6+mTZuqQoUKCgkJ0cCBA3Xq1KmiF2rgYNGiRcbLy8t88MEHZu/evWb48OEmICDAJCYm5tr/oYceMjExMWbXrl1m3759ZvDgwcbf39+cPHnS3mfQoEGmR48eJj4+3r788ccfxTWkQnHFPBw6dMhUrlzZjBs3zuzcudMcOnTILF26NM99lhaumIurPwvx8fHmgw8+MDabzRw+fLi4hlVgrpiH4cOHm7p165p169aZI0eOmLlz5xp3d3ezdOnS4hpWgTl7HrKyskzbtm3NHXfcYb7//nvzyy+/mBEjRpgaNWqYtLS04hxagRV0LtatW2cWL15sfv75Z3Po0CHzzjvvGHd3d7Ny5Up7n+nTpxt/f3+zZMkS8+OPP5q7777b1K5d21y8eLG4hlVgrpiH//u//zNTp0417777rpFkdu3aVUyjKRpnz0VycrKJiIgwn376qfnll1/Mli1bTOvWrU14eHiRayUA/UXr1q3NyJEj7a8zMzNNSEiIiY6Ovq7t//zzT+Pr62sWLlxobxs0aJDp06ePs0t1KVfMwwMPPGD+/ve/O71WV3PFXPxVnz59zJ133lnkWl3JFfPQpEkTM23aNId+LVu2NBMmTHBO0S7g7HnYv3+/kWT27NnjsM/AwEDz7rvvOrd4JyvqXBhjTIsWLcyLL75ojLkSBoODg82MGTPs65OTk423t7f55JNPnFe4kzl7Hq525MiRMhWAXDkX2b7//nsjyRw7dqxItXIK7CqXL1/Wjh07FBERYW9zc3NTRESEtmzZcl37uHDhgjIyMlS5cmWH9vXr16tq1apq0KCBHn/8cZ05c8aptTuTK+YhKytLy5cvV/369RUZGamqVauqTZs2WrJkiSuG4DSu/ExkS0xM1PLlyzV06FCn1OwKrpqH9u3ba9myZfrtt99kjNG6det04MABde/e3eljcAZXzEN6erokycfHx2Gf3t7e2rRpkxOrd66izoUxRrGxsdq/f786duwoSTpy5IgSEhIc9unv7682bdpc9/wWN1fMQ1lVXHORkpIim81W4O/s/CsC0FWSkpKUmZlp/5qNbEFBQUpISLiufTz//PMKCQlx+AD06NFD//d//6fY2Fi99tpr2rBhg3r27KnMzEyn1u8srpiH06dPKy0tTdOnT1ePHj20atUq3XPPPbr33nu1YcMGp4/BWVz1mbjawoUL5evrq3vvvbfI9bqKq+Zh5syZaty4sapXry4vLy/16NFDMTExpfYXgSvmoWHDhqpRo4aioqJ09uxZXb58Wa+99ppOnjyp+Ph4p4/BWQo7FykpKapYsaK8vLzUq1cvzZw5U926dZMk+3ZFmd/i5op5KKuKYy4uXbqk559/XgMGDCjyl6eWia/CKCumT5+uRYsWaf369Q7/N/fggw/a/9y0aVM1a9ZMdevW1fr169W1a9eSKNWlcpuHrKwsSVKfPn30zDPPSJLCwsK0efNmzZkzR506dSqxel0pr8/E1T744AM9/PDDea6/EeQ1DzNnztTWrVu1bNky1axZU999951GjhyZb2Asy3KbB09PTy1evFhDhw5V5cqV5e7uroiICPXs2VPmBnxQv6+vr+Li4pSWlqbY2FiNGTNGderUUefOnUu6tGLFPPzX9c5FRkaG7r//fhljNHv27CK/LwHoKlWqVJG7u7sSExMd2hMTExUcHJzvtm+88YamT5+uNWvWqFmzZvn2rVOnjqpUqaJDhw6VygDkinmoUqWKPDw81LhxY4f+jRo1KtWH+V39mdi4caP279+vTz/91Gk1u4Ir5uHixYt64YUX9NVXX6lXr16SpGbNmikuLk5vvPFGqQxArvo8hIeHKy4uTikpKbp8+bICAwPVpk0b3XbbbU4fg7MUdi7c3NxUr149SVf+J2jfvn2Kjo5W586d7dslJiaqWrVqDvsMCwtz/iCcwBXzUFa5ci6yw8+xY8e0du3aIh/9kTgF5sDLy0vh4eGKjY21t2VlZSk2Nlbt2rXLc7vXX39dL730klauXHld/2CdPHlSZ86ccfgBL01cMQ9eXl5q1aqV9u/f79B+4MAB1axZ07kDcCJXfybef/99hYeHq3nz5k6t29lcMQ8ZGRnKyMiQm5vjP0Pu7u72I4aljas/D/7+/goMDNTBgwe1fft29enTx6n1O1Nh5+KvsrKy7NdB1a5dW8HBwQ77TE1N1bZt2wq0z+Lkinkoq1w1F9nh5+DBg1qzZo1uuukm5xRcpEuob0CLFi0y3t7eZsGCBebnn382I0aMMAEBASYhIcEYY8wjjzxixo8fb+8/ffp04+XlZb744guH25rPnTtnjDHm3LlzZuzYsWbLli3myJEjZs2aNaZly5bmlltuMZcuXSqRMV4PZ8+DMcYsXrzYeHp6mnnz5pmDBw+amTNnGnd3d7Nx48ZiH19BuGIujDEmJSXFlC9f3syePbtYx1NYrpiHTp06mSZNmph169aZX3/91cyfP9/4+PiYf/7zn8U+vuvlinn47LPPzLp168zhw4fNkiVLTM2aNc29995b7GMrqILOxauvvmpWrVplDh8+bH7++WfzxhtvGA8PD4e73aZPn24CAgLM0qVLzU8//WT69OlTJm6Dd/Y8nDlzxuzatcssX77cSDKLFi0yu3btMvHx8cU+voJw9lxcvnzZ3H333aZ69eomLi7O4WcoPT29SLUSgHIxc+ZMU6NGDePl5WVat25ttm7dal/XqVMnM2jQIPvrmjVrGkk5lsmTJxtjjLlw4YLp3r27CQwMNJ6enqZmzZpm+PDh9g9DaebMecj2/vvvm3r16hkfHx/TvHlzs2TJkmIaTdG4Yi7mzp1rypUrZ5KTk4tpFEXn7HmIj483gwcPNiEhIcbHx8c0aNDAvPnmmyYrK6sYR1Vwzp6Hf/zjH6Z69erG09PT1KhRw7z44otF/se9uBRkLiZMmGD/+a9UqZJp166dWbRokcP+srKyzMSJE01QUJDx9vY2Xbt2Nfv37y+u4RSas+dh/vz51/XvSGnkzLnIfgxAbsu6deuKVKfNmBvwKjsAAIB8cA0QAACwHAIQAACwHAIQAACwHAIQAACwHAIQAACwHAIQAACwHAIQAACwHAIQAACwHAIQgDLj999/1+OPP64aNWrI29tbwcHBioyM1H/+8x+HfmfOnNGUKVPUqlUrBQYGqkaNGurVq5cWLVqU6zes79y5U926dVNAQIBuuukmjRgxQmlpaQ59fvjhB3Xt2lUBAQGqVKmSIiMj9eOPP7p0vABchwAEoMzo16+fdu3apYULF+rAgQNatmyZOnfurDNnztj7rFq1SvXr19cPP/ygsWPHatWqVVq8eLHuuusuvfTSS4qMjNT58+ft/U+dOqWIiAjVq1dP27Zt08qVK7V3714NHjzY3ictLU09evRQjRo1tG3bNm3atEm+vr6KjIxURkZGcU4BACfhqzAAlAnJycmqVKmS1q9fr06dOuXaZ/v27YqMjNSCBQvUu3fvHOv//PNPjRgxQklJSVq2bJkkad68eZo4caLi4+Pt30y/e/duNWvWTAcPHlS9evW0fft2tWrVSsePH1doaGiufQCULRwBAlAmVKxYURUrVtSSJUuUnp6ea5/Ro0frlVdeUe/evfXzzz+rU6dOCgwM1P33368xY8bo9ddf15w5c/Tzzz9r3bp1kqT09HR5eXnZw48klStXTpK0adMmSVKDBg1000036f3339fly5d18eJFvf/++2rUqJFq1arl2oEDcAkCEIAywcPDQwsWLNDChQsVEBCg22+/XS+88IJ++uknSdLBgwd19OhRDRs2TJmZmbrnnnvUsGFDrV69WnfccYdmzpypy5cvy8vLSwMGDNC3334rSbrzzjuVkJCgGTNm6PLlyzp79qzGjx8vSYqPj5ck+fr6av369froo49Urlw5VaxYUStXrtQ333wjDw+PkpkQAEVCAAJQZvTr10+nTp3SsmXL1KNHD61fv14tW7bUggULtHv3brVq1UoeHh7av3+/fvvtN82aNUthYWEaPXq0OnfubN9PtWrVdPbsWUlSkyZNtHDhQr355psqX768goODVbt2bQUFBdmPCl28eFFDhw7V7bffrq1bt+o///mPbr31VvXq1UsXL14siakAUEQEIABlio+Pj7p166aJEydq8+bNGjx4sCZPnqw///zTfurq8uXL8vT0lKenp327ihUr2v+8c+dOh+t2HnroISUkJOi3336z30H2+++/q06dOpKkjz/+WEePHtX8+fPVqlUrtW3bVh9//LGOHDmipUuXFtPIATgTAQhAmda4cWOdP39e9erV0+7duyVduWbH09NTs2bNUmZmprZu3apvv/1WGRkZWrhwob755huHu7yyBQUFqWLFivr000/tQUuSLly4IDc3N9lsNnvf7NdZWVnFMk4AzkUAAlAmnDlzRnfeeac++ugj/fTTTzpy5Ig+//xzvf766+rTp49atGihixcvat26dSpXrpwWLFigSZMmydvbW0OGDFHfvn312muvaf78+Vq1apUCAwPt+541a5Z27typAwcOKCYmRqNGjVJ0dLQCAgIkSd26ddPZs2c1cuRI7du3T3v37tWQIUPk4eGhLl26lNCMACgKboMHUCakp6drypQpWrVqlQ4fPqyMjAyFhoaqf//+euGFF1SuXDl99tlnGjt2rL777jvVqlVLmZmZSkhIUEhIiFJSUuTh4eFwKizbwIEDtXz5cqWlpalhw4YaO3asHnnkEYc+q1ev1tSpU7Vnzx65ubmpRYsWeuWVV9S2bdvimgIATkQAAnBDefXVV/XWW28pKipKDzzwgKpXr6709HRt2LBBL730ksaMGaN77rmnpMsEUMIIQABuOBs3btTLL7+s9evXyxijP//8U40bN9aTTz6pYcOGOTzzB4A1EYAA3LDS09N1+vRp+fr62q/nAQCJAAQAACyI48AAAMByCEAAAMByCEAAAMByCEAAAMByCEAAAMByCEAAAMByCEAAAMByCEAAAMByCEAAAMBy/h8k/JT6bgzgQwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.hist(sas98s)\n",
    "# ax.set(\n",
    "#     xlabel=\"S@98\",\n",
    "#     ylabel=\"Counts\",\n",
    "#     title=\"Histogram of S@98 for 1600 Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c0a8fe2b-1cd9-4585-aa92-02b8a38571a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f61d81da66c440919b5169c874091016",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Reps Histogram S@98:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# sas98s = []\n",
    "# for seed in trange(100, desc=\"Reps Histogram S@98\"):\n",
    "#     est = HonestForestClassifier(\n",
    "#         n_estimators=4000,\n",
    "#         random_state=seed,\n",
    "#         honest_fraction=0.5,\n",
    "#         n_jobs=-1,\n",
    "#         bootstrap=True,\n",
    "#         stratify=True,\n",
    "#         max_samples=1.6,\n",
    "#         # permute_per_tree=True,\n",
    "#     )\n",
    "#     _, posterior_arr = build_hyppo_oob_forest(\n",
    "#         est,\n",
    "#         X,\n",
    "#         y,\n",
    "#         verbose=False\n",
    "#     )\n",
    "#     sas98 = sensitivity_at_specificity(\n",
    "#         y, posterior_arr, target_specificity=0.98\n",
    "#     )\n",
    "#     sas98s.append(sas98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "aec09315-5383-4363-a871-c962b85de95f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/cklEQVR4nO3deVyVdf7//ycgIMgmiiC5r4mOG65Z7grmmOaSS/VRR21Tm3S0ySwV0zC1bEbJpUwdG7PNrcwV93EpNTJNcUnLFXcQFyB4//7wy/l1BFQQPOeyx/12O7c87+t9vc/renslT67tuBhjjAAAACzI1dEFAAAA5BVBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBpZWrlw59enTx9FlPPAmTZqkChUqyM3NTbVr13Z0OffdoUOH1LZtW/n7+8vFxUVLlixxdEkA/h+CDJzG3Llz5eLiop07d2a7vHnz5qpRo8Y9f863336rMWPG3PM4fxarV6/Wq6++qiZNmmjOnDl6++23b9v/66+/VrNmzVSiRAl5e3urQoUKeuqpp7Ry5cq7+ryrV68qPT39rvquWbNGjz76qLy9vVW0aFF17dpVx44dy9Lvxo0bio6OVlhYmLy9vfXQQw+pW7du2rdv3119Tu/evfXTTz9p/Pjxmj9/vurVq3dX6+WHLVu2yMXFRS4uLjp//nyW5SdPntRTTz2lgIAA+fn5qWPHjvrll1+yHWv27NmqVq2aChcurMqVK2vq1KnZ9svNmJnGjBljq/N2r+bNm+d6DoDbKeToAoB7ER8fL1fX3OXxb7/9VjExMYSZu7Ru3Tq5urpq9uzZ8vDwuG3fyZMna/jw4WrWrJlGjBghb29vHT58WGvXrtXChQsVGRmZ7Xpr1qzRjBkztG7dOl2+fFlubm4qX768unbtqr///e8KCQnJss4333yjjh07qm7dupowYYKSkpL0r3/9S48++qh++OEHBQUF2fo+/fTTWrZsmQYMGKC6devq1KlTiomJUePGjfXTTz+pbNmyOW7T9evXtW3bNo0cOVKDBg26y1nLHxkZGRo8eLCKFCmiq1evZlmenJysFi1aKDExUa+//rrc3d01ZcoUNWvWTHFxcSpWrJit78yZM/XCCy+oS5cuGjp0qDZv3qyXX35Z165d0z//+c88jflHnTt3VqVKlezGefHFF/Xkk0+qc+fOtvbg4OD8mBrg/2cAJzFnzhwjyXz//ffZLm/WrJmpXr36PX/OwIEDjSN3/eTkZId9dl707dvXFClS5I790tLSjJ+fn2nTpk22yxMSErK0JScnmy5duhgXFxfTrl07M3XqVPPNN9+Yzz//3IwaNcpUrlzZBAQEmC+//DLLumFhYaZSpUomJSXF1hYXF2dcXV3N0KFDbW0nTpwwksywYcPs1l+3bp2RZN57773bbtevv/5qJJlJkybdtl9u3O0+MH36dFOsWDHz97//3Ugy586ds1v+zjvvGEnmu+++s7Xt37/fuLm5mREjRtjarl27ZooVK2bat29vt/7TTz9tihQpYi5evJjrMe/k3LlzRpIZPXr0bftdv37dpKen3/W4wK0IMnAaeQkyZcuWNb1797a9T01NNWPGjDGVKlUynp6eJjAw0DRp0sSsXr3aGGNM7969jaQsr0zJyclm6NChplSpUsbDw8NUqVLFTJo0yWRkZNh97rVr18zgwYNNsWLFjI+Pj+nQoYPtB+Yf/+EePXq0kWT27dtnevbsaQICAkzt2rWNMcb8+OOPpnfv3qZ8+fLG09PTBAcHm759+5rz58/bfVbmGPHx8ebpp582fn5+pnjx4uaNN94wGRkZ5rfffjNPPPGE8fX1NcHBwWby5Ml3Nd9paWlm7NixpkKFCsbDw8OULVvWjBgxwty4ccPWJ7u5mjNnTrbjnT592kgyY8aMuevPb968uSlTpozdD81b+7zzzjvGw8PDfPPNN7b2CxcuGElm+PDhWdapXr26CQ0Ntb3fv39/tkEks3369Ok51pg59398lS1b1rZ89+7dJjIy0vj6+poiRYqYli1bmm3bttmNkblfb9iwwbz44osmKCjIBAQE3HZuMrexWLFiJiYmxlbHrUGmfv36pn79+lnWbdu2ralYsaLt/fLly40ks3z5crt+W7duNZLM/Pnzcz3mnWQXZNavX28kmU8//dSMHDnShIaGGhcXF3Pp0iVjjDHbt283ERERxs/Pz3h5eZmmTZuaLVu2ZBn7xIkTpm/fvqZEiRLGw8PDhIWFmdmzZ2fp9+9//9uEhYUZLy8vExAQYMLDw81///vfu94GWAOnluB0EhMTs70WIC0t7Y7rjhkzRtHR0erfv78aNGigpKQk7dy5U7t371abNm30/PPP69SpU1qzZo3mz59vt64xRk888YTWr1+vfv36qXbt2lq1apWGDx+ukydPasqUKba+ffr00eeff65nn31WjRo10saNG9W+ffsc6+rWrZsqV66st99+W8YYSTdPp/zyyy/q27evQkJCtG/fPs2aNUv79u3T9u3b5eLiYjdG9+7dVa1aNU2YMEHLly/XuHHjFBgYqJkzZ6ply5Z655139N///lfDhg1T/fr11bRp09vOVf/+/TVv3jx17dpV//jHP7Rjxw5FR0dr//79Wrx4sSRp/vz5mjVrlr777jt99NFHkqRHHnkk2/FKlCghLy8vff311xo8eLACAwNv+/nR0dGKj4/Xrl27VLJkSUk3T6Vcv35dRYoUUUZGhi5fvqxXX31Vvr6++tvf/qbDhw/L19dXKSkpkiQvL68s43p7e2vfvn06c+aMQkJCVLFiRZUqVUrvvvuuqlatqjp16ujUqVN69dVXVb58efXo0SPHGjt37qyAgAANGTJEPXv21OOPPy4fHx9J0r59+/TYY4/Jz89Pr776qtzd3TVz5kw1b95cGzduVMOGDe3GeumllxQUFKRRo0Zle5roVm+++aZCQkL0/PPP66233sqyPCMjQ3v27NHf/va3LMsaNGig1atX68qVK/L19dUPP/wgSVmu7QkPD5erq6t++OEHPfPMM7ka81689dZb8vDw0LBhw5SSkiIPDw+tW7dO7dq1U3h4uEaPHi1XV1fNmTNHLVu21ObNm9WgQQNJUkJCgho1aiQXFxcNGjRIQUFBWrFihfr166ekpCS98sorkqQPP/xQL7/8su305I0bN7Rnzx7t2LFDvXr1uqf64WQcnaSATJm/ud7udacjMrVq1cpy+PxWOZ1aWrJkiZFkxo0bZ9fetWtX4+LiYg4fPmyMMWbXrl1GknnllVfs+vXp0yfHIzI9e/bM8nnXrl3L0vbpp58aSWbTpk1Zxnjuuedsbb///rspVaqUcXFxMRMmTLC1X7p0yXh5ednNSXbi4uKMJNO/f3+79mHDhhlJZt26dba23r1739WpJWOMGTVqlJFkihQpYtq1a2fGjx9vdu3alaVfYmKi8fPzM0uWLLG1zZo1yxQtWtT29/zVV1/Z/T3VrVvXzJo1yxhjTHp6ugkICDCtWrWyG/f8+fOmSJEiRpLZuXOnrX3Hjh2mYsWKdvtSeHi4OX369B236ejRo9ke0enUqZPx8PAwR44csbWdOnXK+Pr6mqZNm9raMvfrRx991Pz+++93/Dxjbh6tc3NzM6tWrTLGmGyPyGQe8Rg7dmyW9WNiYowkc+DAAWPMzX3ezc0t288KCgoyPXr0yPWYd3K7IzIVKlSw2/8zMjJM5cqVTUREhN3Rz2vXrpny5cvbna7s16+fKVmyZJYjlz169DD+/v62cTt27Jgvp6Lh/LhrCU4nJiZGa9asyfKqWbPmHdcNCAjQvn37dOjQoVx/7rfffis3Nze9/PLLdu3/+Mc/ZIzRihUrJMl2981LL71k12/w4ME5jv3CCy9kafvj0YQbN27o/PnzatSokSRp9+7dWfr379/f9mc3NzfVq1dPxhj169fP1h4QEKCqVave8Q6Tb7/9VpI0dOhQu/Z//OMfkqTly5ffdv2cREVFacGCBapTp45WrVqlkSNHKjw8XHXr1tX+/ftt/VavXq3AwEA98cQTkm5u7/PPP68uXbpo8eLF6t69uwYMGGA3dseOHbVhwwZJkqurq55//nnFxsZqxIgROnTokHbt2qWnnnpKqampkm5epJupaNGiql27tl577TUtWbJEkydP1rFjx9StWzfduHEj19uZnp6u1atXq1OnTqpQoYKtvWTJkurVq5e2bNmipKQku3UGDBggNze3uxr/5ZdfVrt27dS2bdsc+2Run6enZ5ZlhQsXtutz/fr1HC/ULly4sF2/ux3zXvTu3dtu/4+Li9OhQ4fUq1cvXbhwQefPn9f58+d19epVtWrVSps2bVJGRoaMMfrqq6/UoUMHGWNs/c6fP6+IiAglJiba/t8JCAjQiRMn9P33399zvXBunFqC02nQoEG2t7cWLVo021NOfzR27Fh17NhRVapUUY0aNRQZGalnn332rkLQr7/+qtDQ0CyHzatVq2ZbnvlfV1dXlS9f3q7fH+/YuNWtfSXp4sWLioqK0sKFC3X27Fm7ZYmJiVn6lylTxu69v7+/ChcurOLFi2dpv3DhQo61/HEbbq05JCREAQEBtm3Ni549e6pnz55KSkrSjh07NHfuXC1YsEAdOnTQ3r17VbhwYe3atUvNmjWznT776KOP1Lx5c3344YeSpE6dOik9PV1RUVG2cYODg7Vlyxbb+7Fjx+r8+fOaOHGiJkyYIElq27at+vXrpxkzZthOASUmJuqxxx7T8OHDbUFNunmapXnz5pozZ45efPHFXG3juXPndO3aNVWtWjXLsmrVqikjI0PHjx9X9erVbe3Z7QPZ+eyzz7R161bt3bv3tv0yg0DmabY/ygxnmX28vLxsAS+7vn/sd7dj3otb5yLzF4/evXvnuE5iYqLS0tJ0+fJlzZo1S7Nmzcq2X+b/S//85z+1du1aNWjQQJUqVVLbtm3Vq1cvNWnS5J7rh3MhyOCB0rRpUx05ckRLly7V6tWr9dFHH2nKlCmaMWOG3RGN+y27f/yfeuopbd26VcOHD1ft2rXl4+OjjIwMRUZGKiMjI0v/7H6bz+k3fPP/rsO5k1uvw8lPfn5+atOmjdq0aSN3d3fNmzdPO3bsULNmzXThwgWFhoba+h47dkz169e3Wz/zmohMx48ft7v118PDQx999JHGjx+vgwcPKjg4WFWqVFGvXr3sQtpXX32lhIQE29GfTM2aNZOfn5/+97//5TrI5MXdBoDhw4erW7du8vDwsD0T5/Lly5JuzkFqaqpCQ0MVGBgoT09PnT59OssYmW2Zc1yyZEmlp6fr7NmzKlGihK1famqq3d9Fbsa8F7fOReb+PmnSpBwfuOjj42ML6M8880yOoSfzl5Zq1aopPj5e33zzjVauXKmvvvpKH3zwgUaNGmUXkGF9BBk8cAIDA9W3b1/17dtXycnJatq0qcaMGWMLMjn98C5btqzWrl2b5WLGAwcO2JZn/jcjI0NHjx5V5cqVbf0OHz581zVeunRJsbGxioqK0qhRo2zteTkllheZ23Do0CHbESfp5oWUly9fvu1zVfKiXr16mjdvnu2HoZ+fn91Rp5CQEB05csRunT+eHrtx44bmz59vN1eZgoODbc8mSU9P14YNG9SwYUPbEZmEhATbsj8yxig9PV2///57rrcnKChI3t7eio+Pz7LswIEDcnV1VenSpXM9rnQzrCxYsEALFizIsqxu3bqqVauW4uLi5Orqqr/85S/ZPkByx44dqlChgm0/zgwHO3fu1OOPP27rt3PnTmVkZNiW52bM/FSxYkVJN/eL1q1b59gvKChIvr6+Sk9Pv22/TEWKFFH37t3VvXt3paamqnPnzho/frxGjBhhO1UG6+MaGTxQbj2l4uPjo0qVKtkdKi9SpIik//+33EyPP/640tPTNW3aNLv2KVOmyMXFRe3atZMkRURESJI++OADu345PSU1O5lHUm49cvL+++/f9Rj3IvOH2a2f995770nSbe/Aysm1a9e0bdu2bJdlXl+UeSqmWrVq2rFjh235k08+qcWLFysmJka//vqrvv32W9sThDdv3qy2bduqaNGieuaZZ25bw+TJk3X69Gm7U0hVqlSRJC1cuNCu77Jly3T16lXVqVMnl1t68++vbdu2Wrp0qd2ThBMSErRgwQI9+uij8vPzy/W4krR48eIsr+7du0uS/vOf/9jdPde1a1d9//33dsEjPj5e69atU7du3WxtLVu2VGBgoKZPn273WdOnT5e3t7fd3/fdjpmfwsPDVbFiRU2ePFnJyclZlp87d07SzXnv0qWLvvrqq2xPvWX2k7L+W+Dh4aGwsDAZY+7qDkhYB0dk8EAJCwtT8+bNFR4ersDAQO3cuVNffvml3RNZw8PDJd28oDIiIkJubm7q0aOHOnTooBYtWmjkyJE6duyYatWqpdWrV2vp0qV65ZVXbL81hoeHq0uXLnr//fd14cIF2+3XBw8elHR3p2v8/PzUtGlTTZw4UWlpaXrooYe0evVqHT16tABmJatatWqpd+/emjVrli5fvqxmzZrpu+++07x589SpUye1aNEi12Neu3ZNjzzyiBo1aqTIyEiVLl1aly9f1pIlS7R582Z16tTJFhoiIyP1wgsv6IcfflCdOnXUoUMHPf/88xo0aJAGDRokb29vRUVFafjw4WrevLm6du2qRYsW2V2E+sknn+irr75S06ZN5ePjo7Vr1+rzzz9X//791aVLF1u/Dh06qHr16ho7dqx+/fVXNWrUSIcPH9a0adNUsmRJu4ulc2PcuHG2r0h46aWXVKhQIc2cOVMpKSmaOHFinsaUbl4fdKu4uDhJUrt27eyuiXrppZf04Ycfqn379ho2bJjc3d313nvvKTg42C7MeXl56a233tLAgQPVrVs3RUREaPPmzfrkk080fvx4u1vl73bM/OTq6qqPPvpI7dq1U/Xq1dW3b1899NBDOnnypNavXy8/Pz99/fXXkqQJEyZo/fr1atiwoQYMGKCwsDBdvHhRu3fv1tq1a3Xx4kVJN6+XCgkJUZMmTRQcHKz9+/dr2rRpat++fYEcVYIDOe6GKcBefjwQb9y4caZBgwYmICDAeHl5mYcfftiMHz/epKam2vr8/vvvZvDgwSYoKMi4uLjY3eJ75coVM2TIEBMaGmrc3d1N5cqVs30g3tWrV83AgQNNYGCg8fHxMZ06dTLx8fFGkt3t0Dk9yMyYmw/1evLJJ01AQIDx9/c33bp1M6dOncrxFu5bx8jptui7fQJyWlqaiYqKMuXLlzfu7u6mdOnSWR6Id7vPyW68Dz/80HTq1MmULVvWeHp6Gm9vb1OnTh0zadIkuyfwZo7bsGFDu/YjR46YzZs3m0uXLpnr16+bbdu2mcuXL2f7eTt27DBNmzY1RYsWNYULFza1atUyM2bMyPJ3ZYwxFy9eNEOGDDFVqlQxnp6epnjx4qZHjx7ml19+ueN25XT7tTE3H4gXERFhfHx8jLe3t2nRooXZunWrXZ877dd343b70fHjx03Xrl2Nn5+f8fHxMX/961/NoUOHsh1n1qxZpmrVqsbDw8NUrFjRTJkyJdv5ys2YObnd7ddffPFFtuv88MMPpnPnzqZYsWLG09PTlC1b1jz11FMmNjbWrl9CQoIZOHCgKV26tHF3dzchISGmVatWttvzjTFm5syZpmnTpraxKlasaIYPH24SExNztR1wfi7G3OVVgQBuKy4uTnXq1NEnn3yip59+2tHlOL3z588rPDxcNWrU0KeffprtqZj09HQtXrxYXbt2dUCFAKyAIAPkwfXr17PcedGnTx/Nnz9fx44dy/OFnn82Bw8eVPv27ZWUlKRBgwapTZs2Cg0NVVJSkrZs2aJp06bpzJkz2r17d5bbzwFAIsgAeRIVFaVdu3apRYsWKlSokFasWKEVK1boueee08yZMx1dnqVcuXJFkyZN0kcffWR326+vr6+efvppjRo1yvYVBgBwK4IMkAdr1qxRVFSUfv75ZyUnJ6tMmTJ69tlnNXLkSBUqxDX0eWGM0eHDh3XmzBn5+fmpWrVqOT6NFgAyEWQAAIBl8RwZAABgWQQZAABgWQ/8yfyMjAydOnVKvr6+Bfq9MgAAIP8YY3TlyhWFhobK1TXn4y4PfJA5deoUt8ICAGBRx48fV6lSpXJc/sAHmcxHUR8/fjzP330CAADur6SkJJUuXfqOXynxwAeZzNNJfn5+BBkAACzmTpeFcLEvAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIIMAACwLIcGmenTp6tmzZq2rw9o3LixVqxYYVt+48YNDRw4UMWKFZOPj4+6dOmihIQEB1YMAACciUODTKlSpTRhwgTt2rVLO3fuVMuWLdWxY0ft27dPkjRkyBB9/fXX+uKLL7Rx40adOnVKnTt3dmTJAADAibgYY4yji/ijwMBATZo0SV27dlVQUJAWLFigrl27SpIOHDigatWqadu2bWrUqNFdjZeUlCR/f38lJibypZEAAFjE3f78dpprZNLT07Vw4UJdvXpVjRs31q5du5SWlqbWrVvb+jz88MMqU6aMtm3b5sBKAQCAsyjk6AJ++uknNW7cWDdu3JCPj48WL16ssLAwxcXFycPDQwEBAXb9g4ODdebMmRzHS0lJUUpKiu19UlJSQZUOAAAczOFBpmrVqoqLi1NiYqK+/PJL9e7dWxs3bszzeNHR0YqKisrHCgE4s3KvLXd0Cbl2bEJ7R5cAPDAcfmrJw8NDlSpVUnh4uKKjo1WrVi3961//UkhIiFJTU3X58mW7/gkJCQoJCclxvBEjRigxMdH2On78eAFvAQAAcBSHB5lbZWRkKCUlReHh4XJ3d1dsbKxtWXx8vH777Tc1btw4x/U9PT1tt3NnvgAAwIPJoaeWRowYoXbt2qlMmTK6cuWKFixYoA0bNmjVqlXy9/dXv379NHToUAUGBsrPz0+DBw9W48aN7/qOJQAA8GBzaJA5e/as/u///k+nT5+Wv7+/atasqVWrVqlNmzaSpClTpsjV1VVdunRRSkqKIiIi9MEHHziyZAAA4ESc7jky+Y3nyAAPNi72BR5MlnuODAAAQG4RZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGU5NMhER0erfv368vX1VYkSJdSpUyfFx8fb9WnevLlcXFzsXi+88IKDKgYAAM7EoUFm48aNGjhwoLZv3641a9YoLS1Nbdu21dWrV+36DRgwQKdPn7a9Jk6c6KCKAQCAMynkyA9fuXKl3fu5c+eqRIkS2rVrl5o2bWpr9/b2VkhIyP0uDwAAODmnukYmMTFRkhQYGGjX/t///lfFixdXjRo1NGLECF27di3HMVJSUpSUlGT3AgAADyaHHpH5o4yMDL3yyitq0qSJatSoYWvv1auXypYtq9DQUO3Zs0f//Oc/FR8fr0WLFmU7TnR0tKKiou5X2QAAwIFcjDHG0UVI0osvvqgVK1Zoy5YtKlWqVI791q1bp1atWunw4cOqWLFiluUpKSlKSUmxvU9KSlLp0qWVmJgoPz+/AqkdgOOUe225o0vItWMT2ju6BMDpJSUlyd/f/44/v53iiMygQYP0zTffaNOmTbcNMZLUsGFDScoxyHh6esrT07NA6gQAAM7FoUHGGKPBgwdr8eLF2rBhg8qXL3/HdeLi4iRJJUuWLODqAACAs3NokBk4cKAWLFigpUuXytfXV2fOnJEk+fv7y8vLS0eOHNGCBQv0+OOPq1ixYtqzZ4+GDBmipk2bqmbNmo4sHQAAOAGHBpnp06dLuvnQuz+aM2eO+vTpIw8PD61du1bvv/++rl69qtKlS6tLly564403HFAtAABwNg4/tXQ7pUuX1saNG+9TNQAAwGqc6jkyAAAAuUGQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAluXQIBMdHa369evL19dXJUqUUKdOnRQfH2/X58aNGxo4cKCKFSsmHx8fdenSRQkJCQ6qGAAAOBOHBpmNGzdq4MCB2r59u9asWaO0tDS1bdtWV69etfUZMmSIvv76a33xxRfauHGjTp06pc6dOzuwagAA4CwKOfLDV65cafd+7ty5KlGihHbt2qWmTZsqMTFRs2fP1oIFC9SyZUtJ0pw5c1StWjVt375djRo1ckTZAADASTjVNTKJiYmSpMDAQEnSrl27lJaWptatW9v6PPzwwypTpoy2bduW7RgpKSlKSkqyewEAgAeT0wSZjIwMvfLKK2rSpIlq1KghSTpz5ow8PDwUEBBg1zc4OFhnzpzJdpzo6Gj5+/vbXqVLly7o0gEAgIM4TZAZOHCg9u7dq4ULF97TOCNGjFBiYqLtdfz48XyqEAAAOBuHXiOTadCgQfrmm2+0adMmlSpVytYeEhKi1NRUXb582e6oTEJCgkJCQrIdy9PTU56engVdMgAAcAIOPSJjjNGgQYO0ePFirVu3TuXLl7dbHh4eLnd3d8XGxtra4uPj9dtvv6lx48b3u1wAAOBkHHpEZuDAgVqwYIGWLl0qX19f23Uv/v7+8vLykr+/v/r166ehQ4cqMDBQfn5+Gjx4sBo3bswdSwAAwLFBZvr06ZKk5s2b27XPmTNHffr0kSRNmTJFrq6u6tKli1JSUhQREaEPPvjgPlcKAACckUODjDHmjn0KFy6smJgYxcTE3IeKAACAlTjNXUsAAAC5RZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWlacgs3v3bv3000+290uXLlWnTp30+uuvKzU1Nd+KAwAAuJ08BZnnn39eBw8elCT98ssv6tGjh7y9vfXFF1/o1VdfzdcCAQAAcpKnIHPw4EHVrl1bkvTFF1+oadOmWrBggebOnauvvvoqP+sDAADIUZ6CjDFGGRkZkqS1a9fq8ccflySVLl1a58+fz7/qAAAAbiNPQaZevXoaN26c5s+fr40bN6p9+/aSpKNHjyo4ODhfCwQAAMhJnoLMlClTtHv3bg0aNEgjR45UpUqVJElffvmlHnnkkXwtEAAAICeF8rJSrVq17O5ayjRp0iQVKpSnIQEAAHItT6mjQoUK+v7771WsWDG79hs3bqhu3br65Zdf8qU4AHgQlXttuaNLyLVjE9o7ugQgW3k6tXTs2DGlp6dnaU9JSdGJEyfuuSgAAIC7kasjMsuWLbP9edWqVfL397e9T09PV2xsrMqXL59/1QEAANxGroJMp06dJEkuLi7q3bu33TJ3d3eVK1dO7777br4VBwAAcDu5CjKZz44pX768vv/+exUvXrxAigIAALgbebrY9+jRo/ldBwAAQK7l+V7p2NhYxcbG6uzZs7YjNZk+/vjjey4MAADgTvIUZKKiojR27FjVq1dPJUuWlIuLS37XBQAAcEd5CjIzZszQ3Llz9eyzz+Z3PQAAAHctT8+RSU1N5asIAACAw+UpyPTv318LFizI71oAAAByJU+nlm7cuKFZs2Zp7dq1qlmzptzd3e2Wv/fee/lSHAAAwO3kKcjs2bNHtWvXliTt3bvXbhkX/gLWZcXvAALw55anILN+/fr8rgMAACDX8nSNDAAAgDPI0xGZFi1a3PYU0rp16/JcEAAAwN3KU5DJvD4mU1pamuLi4rR3794sXyYJAABQUPIUZKZMmZJt+5gxY5ScnHxPBQEAANytfL1G5plnnuF7lgAAwH2Tr0Fm27ZtKly48F3337Rpkzp06KDQ0FC5uLhoyZIldsv79OkjFxcXu1dkZGR+lgwAACwsT6eWOnfubPfeGKPTp09r586devPNN+96nKtXr6pWrVr629/+lmXMTJGRkZozZ47tvaenZ15KBgAAD6A8BRl/f3+7966urqpatarGjh2rtm3b3vU47dq1U7t27W7bx9PTUyEhIXkpEwAAPODyFGT+eISkoG3YsEElSpRQ0aJF1bJlS40bN07FihW7b58PAACcV56CTKZdu3Zp//79kqTq1aurTp06+VJUpsjISHXu3Fnly5fXkSNH9Prrr6tdu3batm2b3Nzcsl0nJSVFKSkptvdJSUn5WhMAAHAeeQoyZ8+eVY8ePbRhwwYFBARIki5fvqwWLVpo4cKFCgoKypfievToYfvzX/7yF9WsWVMVK1bUhg0b1KpVq2zXiY6OVlRUVL58PgAAcG55umtp8ODBunLlivbt26eLFy/q4sWL2rt3r5KSkvTyyy/nd402FSpUUPHixXX48OEc+4wYMUKJiYm21/HjxwusHgAA4Fh5OiKzcuVKrV27VtWqVbO1hYWFKSYmJlcX++bWiRMndOHCBZUsWTLHPp6entzZBADAn0SegkxGRobc3d2ztLu7uysjI+Oux0lOTrY7unL06FHFxcUpMDBQgYGBioqKUpcuXRQSEqIjR47o1VdfVaVKlRQREZGXsgEAwAMmT6eWWrZsqb///e86deqUre3kyZMaMmRIjteuZGfnzp2qU6eO7SLhoUOHqk6dOho1apTc3Ny0Z88ePfHEE6pSpYr69eun8PBwbd68mSMuAABAUh6PyEybNk1PPPGEypUrp9KlS0uSjh8/rho1auiTTz6563GaN28uY0yOy1etWpWX8gAAwJ9EnoJM6dKltXv3bq1du1YHDhyQJFWrVk2tW7fO1+IAAABuJ1enltatW6ewsDAlJSXJxcVFbdq00eDBgzV48GDVr19f1atX1+bNmwuqVgAAADu5CjLvv/++BgwYID8/vyzL/P399fzzz+u9997Lt+IAAABuJ1dB5scff7ztt0+3bdtWu3btuueiAAAA7kaugkxCQkK2t11nKlSokM6dO3fPRQEAANyNXAWZhx56SHv37s1x+Z49e277sDoAAID8lKsg8/jjj+vNN9/UjRs3siy7fv26Ro8erb/+9a/5VhwAAMDt5Or26zfeeEOLFi1SlSpVNGjQIFWtWlWSdODAAcXExCg9PV0jR44skEIBAABulasgExwcrK1bt+rFF1/UiBEjbA+zc3FxUUREhGJiYhQcHFwghQIAANwq1w/EK1u2rL799ltdunRJhw8fljFGlStXVtGiRQuiPgAAgBzl6cm+klS0aFHVr18/P2sBAADIlTx9aSQAAIAzIMgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLKuToAoAHVbnXlju6BAB44HFEBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWJZDg8ymTZvUoUMHhYaGysXFRUuWLLFbbozRqFGjVLJkSXl5eal169Y6dOiQY4oFAABOx6FB5urVq6pVq5ZiYmKyXT5x4kT9+9//1owZM7Rjxw4VKVJEERERunHjxn2uFAAAOCOHPhCvXbt2ateuXbbLjDF6//339cYbb6hjx46SpP/85z8KDg7WkiVL1KNHj/tZKgAAcEJOe43M0aNHdebMGbVu3drW5u/vr4YNG2rbtm05rpeSkqKkpCS7FwAAeDA57VcUnDlzRpIUHBxs1x4cHGxblp3o6GhFRUUVaG0A8Gdjxa/cODahvaNLwH3gtEdk8mrEiBFKTEy0vY4fP+7okgAAQAFx2iATEhIiSUpISLBrT0hIsC3Ljqenp/z8/OxeAADgweS0QaZ8+fIKCQlRbGysrS0pKUk7duxQ48aNHVgZAABwFg69RiY5OVmHDx+2vT969Kji4uIUGBioMmXK6JVXXtG4ceNUuXJllS9fXm+++aZCQ0PVqVMnxxUNAACchkODzM6dO9WiRQvb+6FDh0qSevfurblz5+rVV1/V1atX9dxzz+ny5ct69NFHtXLlShUuXNhRJQMAACfiYowxji6iICUlJcnf31+JiYlcL4P7yop3eQAPEu5asra7/fnttNfIAAAA3AlBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWJZTB5kxY8bIxcXF7vXwww87uiwAAOAkCjm6gDupXr261q5da3tfqJDTlwwAAO4Tp08FhQoVUkhIiKPLAAAATsipTy1J0qFDhxQaGqoKFSro6aef1m+//Xbb/ikpKUpKSrJ7AQCAB5OLMcY4uoicrFixQsnJyapatapOnz6tqKgonTx5Unv37pWvr2+264wZM0ZRUVFZ2hMTE+Xn51fQJVtCudeWO7qEXDs2ob2jS8g1K84zAMey4r91BSUpKUn+/v53/Pnt1Edk2rVrp27duqlmzZqKiIjQt99+q8uXL+vzzz/PcZ0RI0YoMTHR9jp+/Ph9rBgAANxPTn+NzB8FBASoSpUqOnz4cI59PD095enpeR+rAgAAjuLUR2RulZycrCNHjqhkyZKOLgUAADgBpw4yw4YN08aNG3Xs2DFt3bpVTz75pNzc3NSzZ09HlwYAAJyAU59aOnHihHr27KkLFy4oKChIjz76qLZv366goCBHlwYAAJyAUweZhQsXOroEAADgxJz61BIAAMDtEGQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlEWQAAIBlFXJ0AcDdKPfackeXAABwQhyRAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlsVXFNwDHpsPAIBjcUQGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFt+1BACAk7Did/gdm9DeoZ/PERkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZlggyMTExKleunAoXLqyGDRvqu+++c3RJAADACTh9kPnss880dOhQjR49Wrt371atWrUUERGhs2fPOro0AADgYE4fZN577z0NGDBAffv2VVhYmGbMmCFvb299/PHHji4NAAA4mFMHmdTUVO3atUutW7e2tbm6uqp169batm2bAysDAADOwKm/ouD8+fNKT09XcHCwXXtwcLAOHDiQ7TopKSlKSUmxvU9MTJQkJSUl5Xt9GSnX8n1MAACspCB+vv5xXGPMbfs5dZDJi+joaEVFRWVpL126tAOqAQDgweb/fsGOf+XKFfn7++e43KmDTPHixeXm5qaEhAS79oSEBIWEhGS7zogRIzR06FDb+4yMDF28eFHFihWTi4tLlv5JSUkqXbq0jh8/Lj8/v/zdgD8h5jP/Maf5i/nMf8xp/mI+bzLG6MqVKwoNDb1tP6cOMh4eHgoPD1dsbKw6deok6WYwiY2N1aBBg7Jdx9PTU56ennZtAQEBd/wsPz+/P/UOk9+Yz/zHnOYv5jP/Maf5i/nUbY/EZHLqICNJQ4cOVe/evVWvXj01aNBA77//vq5evaq+ffs6ujQAAOBgTh9kunfvrnPnzmnUqFE6c+aMateurZUrV2a5ABgAAPz5OH2QkaRBgwbleCrpXnl6emr06NFZTkchb5jP/Mec5i/mM/8xp/mL+cwdF3On+5oAAACclFM/EA8AAOB2CDIAAMCyCDIAAMCyCDIAAMCyHsggExMTo3Llyqlw4cJq2LChvvvuuxz7fvjhh3rsscdUtGhRFS1aVK1bt87Sv0+fPnJxcbF7RUZGFvRmOI38nk9J2r9/v5544gn5+/urSJEiql+/vn777beC3Aynkd/zeeu+mfmaNGlSQW+K08jvOU1OTtagQYNUqlQpeXl5KSwsTDNmzCjozXAa+T2fCQkJ6tOnj0JDQ+Xt7a3IyEgdOnSooDfDqeRmThctWqR69eopICBARYoUUe3atTV//ny7PsYYjRo1SiVLlpSXl5dat279p5tTG/OAWbhwofHw8DAff/yx2bdvnxkwYIAJCAgwCQkJ2fbv1auXiYmJMT/88IPZv3+/6dOnj/H39zcnTpyw9endu7eJjIw0p0+ftr0uXrx4vzbJoQpiPg8fPmwCAwPN8OHDze7du83hw4fN0qVLcxzzQVIQ8/nH/fL06dPm448/Ni4uLubIkSP3a7McqiDmdMCAAaZixYpm/fr15ujRo2bmzJnGzc3NLF269H5tlsPk93xmZGSYRo0amccee8x899135sCBA+a5554zZcqUMcnJyfdz0xwmt3O6fv16s2jRIvPzzz+bw4cPm/fff9+4ubmZlStX2vpMmDDB+Pv7myVLlpgff/zRPPHEE6Z8+fLm+vXr92uznMYDF2QaNGhgBg4caHufnp5uQkNDTXR09F2t//vvvxtfX18zb948W1vv3r1Nx44d87tUSyiI+ezevbt55pln8r1WKyiI+bxVx44dTcuWLe+5VqsoiDmtXr26GTt2rF2/unXrmpEjR+ZP0U4sv+czPj7eSDJ79+61GzMoKMh8+OGH+Vu8k7rXOTXGmDp16pg33njDGHMzHIaEhJhJkybZll++fNl4enqaTz/9NP8Kt4gH6tRSamqqdu3apdatW9vaXF1d1bp1a23btu2uxrh27ZrS0tIUGBho175hwwaVKFFCVatW1YsvvqgLFy7ka+3OqCDmMyMjQ8uXL1eVKlUUERGhEiVKqGHDhlqyZElBbIJTKcj9M1NCQoKWL1+ufv365UvNzq6g5vSRRx7RsmXLdPLkSRljtH79eh08eFBt27bN921wJgUxnykpKZKkwoUL243p6empLVu25GP1zule59QYo9jYWMXHx6tp06aSpKNHj+rMmTN2Y/r7+6thw4Z3/ff0IHmggsz58+eVnp6e5esLgoODdebMmbsa45///KdCQ0PtdpDIyEj95z//UWxsrN555x1t3LhR7dq1U3p6er7W72wKYj7Pnj2r5ORkTZgwQZGRkVq9erWefPJJde7cWRs3bsz3bXAmBbV//tG8efPk6+urzp0733O9VlBQczp16lSFhYWpVKlS8vDwUGRkpGJiYmw/SB5UBTGfDz/8sMqUKaMRI0bo0qVLSk1N1TvvvKMTJ07o9OnT+b4Nziavc5qYmCgfHx95eHioffv2mjp1qtq0aSNJtvXu5e/pQWKJryi4XyZMmKCFCxdqw4YNdr899OjRw/bnv/zlL6pZs6YqVqyoDRs2qFWrVo4o1RKym8+MjAxJUseOHTVkyBBJUu3atbV161bNmDFDzZo1c1i9zi6n/fOPPv74Yz399NM5Loe9nOZ06tSp2r59u5YtW6ayZctq06ZNGjhw4G1DJLKfT3d3dy1atEj9+vVTYGCg3Nzc1Lp1a7Vr106GB8vnyNfXV3FxcUpOTlZsbKyGDh2qChUqqHnz5o4uzek8UEGmePHicnNzU0JCgl17QkKCQkJCbrvu5MmTNWHCBK1du1Y1a9a8bd8KFSqoePHiOnz48AMdZApiPosXL65ChQopLCzMrn+1atUe+MPMBb1/bt68WfHx8frss8/yrWZnVxBzev36db3++utavHix2rdvL0mqWbOm4uLiNHny5Ac6yBTUPhoeHq64uDglJiYqNTVVQUFBatiwoerVq5fv2+Bs8jqnrq6uqlSpkqSbv+zt379f0dHRat68uW29hIQElSxZ0m7M2rVr5/9GOLkH6tSSh4eHwsPDFRsba2vLyMhQbGysGjdunON6EydO1FtvvaWVK1fe1f9YJ06c0IULF+x2oAdRQcynh4eH6tevr/j4eLv2gwcPqmzZsvm7AU6moPfP2bNnKzw8XLVq1crXup1ZQcxpWlqa0tLS5Opq/8+jm5ub7Yjig6qg91F/f38FBQXp0KFD2rlzpzp27Jiv9TujvM7prTIyMmzXG5UvX14hISF2YyYlJWnHjh25GvOB4eCLjfPdwoULjaenp5k7d675+eefzXPPPWcCAgLMmTNnjDHGPPvss+a1116z9Z8wYYLx8PAwX375pd0trFeuXDHGGHPlyhUzbNgws23bNnP06FGzdu1aU7duXVO5cmVz48YNh2zj/ZTf82mMMYsWLTLu7u5m1qxZ5tChQ2bq1KnGzc3NbN68+b5v3/1WEPNpjDGJiYnG29vbTJ8+/b5ujzMoiDlt1qyZqV69ulm/fr355ZdfzJw5c0zhwoXNBx98cN+3734riPn8/PPPzfr1682RI0fMkiVLTNmyZU3nzp3v+7Y5Sm7n9O233zarV682R44cMT///LOZPHmyKVSokN1dXhMmTDABAQFm6dKlZs+ePaZjx47cfv0gmTp1qilTpozx8PAwDRo0MNu3b7cta9asmendu7ftfdmyZY2kLK/Ro0cbY4y5du2aadu2rQkKCjLu7u6mbNmyZsCAAbYd8M8gP+cz0+zZs02lSpVM4cKFTa1atcySJUvu09Y4XkHM58yZM42Xl5e5fPnyfdoK55Lfc3r69GnTp08fExoaagoXLmyqVq1q3n33XZORkXEft8px8ns+//Wvf5lSpUoZd3d3U6ZMGfPGG2+YlJSU+7hFjpebOR05cqTt38eiRYuaxo0bm4ULF9qNl5GRYd58800THBxsPD09TatWrUx8fPz92hyn4mIMV1sBAABreqCukQEAAH8uBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkADnHu3Dm9+OKLKlOmjDw9PRUSEqKIiAj973//s+t34cIFjRkzRvXr11dQUJDKlCmj9u3ba+HChdl+e/Lu3bvVpk0bBQQEqFixYnruueeUnJxs1+f7779Xq1atFBAQoKJFiyoiIkI//vhjgW4vgIJBkAHgEF26dNEPP/ygefPm6eDBg1q2bJmaN2+uCxcu2PqsXr1aVapU0ffff69hw4Zp9erVWrRokf7617/qrbfeUkREhK5evWrrf+rUKbVu3VqVKlXSjh07tHLlSu3bt099+vSx9UlOTlZkZKTKlCmjHTt2aMuWLfL19VVERITS0tLu5xQAyAd8RQGA++7y5csqWrSoNmzYoGbNmmXbZ+fOnYqIiNDcuXPVoUOHLMt///13Pffcczp//ryWLVsmSZo1a5befPNNnT592vbt1T/99JNq1qypQ4cOqVKlStq5c6fq16+v3377TaVLl862DwDr4IgMgPvOx8dHPj4+WrJkiVJSUrLtM3jwYI0fP14dOnTQzz//rGbNmikoKEhPPfWUhg4dqokTJ2rGjBn6+eeftX79eklSSkqKPDw8bCFGkry8vCRJW7ZskSRVrVpVxYoV0+zZs5Wamqrr169r9uzZqlatmsqVK1ewGw4g3xFkANx3hQoV0ty5czVv3jwFBASoSZMmev3117Vnzx5J0qFDh3Ts2DH1799f6enpevLJJ/Xwww9rzZo1euyxxzR16lSlpqbKw8NDPXv21KpVqyRJLVu21JkzZzRp0iSlpqbq0qVLeu211yRJp0+fliT5+vpqw4YN+uSTT+Tl5SUfHx+tXLlSK1asUKFChRwzIQDyjCADwCG6dOmiU6dOadmyZYqMjNSGDRtUt25dzZ07Vz/99JPq16+vQoUKKT4+XidPntS0adNUu3ZtDR48WM2bN7eNU7JkSV26dEmSVL16dc2bN0/vvvuuvL29FRISovLlyys4ONh2lOb69evq16+fmjRpou3bt+t///ufatSoofbt2+v69euOmAoA94AgA8BhChcurDZt2ujNN9/U1q1b1adPH40ePVq///677ZRQamqq3N3d5e7ublvPx8fH9ufdu3fbXdfSq1cvnTlzRidPnrTd8XTu3DlVqFBBkrRgwQIdO3ZMc+bMUf369dWoUSMtWLBAR48e1dKlS+/TlgPILwQZAE4jLCxMV69eVaVKlfTTTz9JunlNi7u7u6ZNm6b09HRt375dq1atUlpamubNm6cVK1bY3ZWUKTg4WD4+Pvrss89sgUmSrl27JldXV7m4uNj6Zr7PyMi4L9sJIP8QZADcdxcuXFDLli31ySefaM+ePTp69Ki++OILTZw4UR07dlSdOnV0/fp1rV+/Xl5eXpo7d65GjRolT09P9e3bV506ddI777yjOXPmaPXq1QoKCrKNPW3aNO3evVsHDx5UTEyMBg0apOjoaAUEBEiS2rRpo0uXLmngwIHav3+/9u3bp759+6pQoUJq0aKFg2YEQF5x+zWA+y4lJUVjxozR6tWrdeTIEaWlpal06dLq1q2bXn/9dXl5eenzzz/XsGHDtGnTJpUrV07p6ek6c+aMQkNDlZiYqEKFCtmdYsr0f//3f1q+fLmSk5P18MMPa9iwYXr22Wft+qxZs0ZRUVHau3evXF1dVadOHY0fP16NGjW6X1MAIJ8QZAA4rbffflvvvfeeRowYoe7du6tUqVJKSUnRxo0b9dZbb2no0KF68sknHV0mAAciyABwaps3b9a4ceO0YcMGGWP0+++/KywsTC+//LL69+9v98wYAH8+BBkAlpCSkqKzZ8/K19fXdr0LABBkAACAZXFMFgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWNb/Bz03gEv0s3TvAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# ax.hist(sas98s)\n",
    "# ax.set(\n",
    "#     xlabel=\"S@98\",\n",
    "#     ylabel=\"Counts\",\n",
    "#     title=\"Histogram of S@98 for 4000 Trees\"\n",
    "# )\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "849cf066-5b46-4aa9-be9d-0cc004c2b673",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f23e3460386495c91463961ca1c35c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Reps Histogram S@98:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# sas98s = []\n",
    "# for seed in trange(100, desc=\"Reps Histogram S@98\"):\n",
    "#     est = HonestForestClassifier(\n",
    "#         n_estimators=10000,\n",
    "#         random_state=seed,\n",
    "#         honest_fraction=0.5,\n",
    "#         n_jobs=-1,\n",
    "#         bootstrap=True,\n",
    "#         stratify=True,\n",
    "#         max_samples=1.6,\n",
    "#         # permute_per_tree=True,\n",
    "#     )\n",
    "#     _, posterior_arr = build_hyppo_oob_forest(\n",
    "#         est,\n",
    "#         X,\n",
    "#         y,\n",
    "#         verbose=False\n",
    "#     )\n",
    "#     sas98 = sensitivity_at_specificity(\n",
    "#         y, posterior_arr, target_specificity=0.98\n",
    "#     )\n",
    "#     sas98s.append(sas98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ca519858-cd3a-4cbb-8496-9ab2e153de3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/WklEQVR4nO3dd3hU1d7+/zu9FwIhIdJBuhI6iBJ6QA5FQAU9HuDQVMBHET0gCgTRIChwHkAQC/BFEUVpiiIdQYoSRKQYiiAtCVKSEEoIyfr9wS/zMKYQQpLJxvfruuaCWXvtNZ819c4uM07GGCMAAAALcnZ0AQAAAPlFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkIFlVKxYUX379nV0GXe9yZMnq3LlynJxcVF4eLijyylyhw4dUvv27RUQECAnJyctW7bM0SUByAVBBg4xb948OTk5aefOndkub9myperUqXPHt/PNN99o3LhxdzzO38Xq1av18ssvq3nz5po7d67efPPNXPt/9dVXioiIUOnSpeXt7a3KlSvrscce06pVq/J0e5cuXVJ6enqe+q5Zs0YPPvigvL29VaJECfXs2VPHjh3L0u/q1auKjo5WrVq15O3trXvuuUePPvqo9u3bl6fb6dOnj3799Ve98cYbWrBggRo2bJin9fJr1qxZevTRR1W+fHk5OTnlGtYTExM1aNAgBQcHy8fHR61atdKuXbuy7btixQrVr19fnp6eKl++vMaOHavr168XyZg3q1ixopycnG55mTdvXq7jADkygAPMnTvXSDI//fRTtssjIiJM7dq17dquXr1qrl27dlu3M2TIEMPTPO/+85//GGdnZ5OamnrLvpMnTzaSTEREhJkyZYqZPXu2GTFihAkPDzd9+vTJcb3Vq1eb7t27m8DAQCPJuLi4mKpVq5qRI0eauLi4bNf56quvjLOzs2nYsKH573//a15//XVTqlQpc88995gzZ87Y9e3evbtxdXU1zzzzjHn//fdNVFSUKV26tPHz8zPHjh3LdU6XL182kszo0aNvOf+CUqFCBRMUFGQ6dOhgXF1dc7zv0tPTzQMPPGB8fHzMuHHjzIwZM0ytWrWMn5+fOXjwoF3fb775xjg5OZlWrVqZOXPmmGHDhhlnZ2fz9NNPF/qYf7V06VKzYMEC26V3795Gkpk6dapd+5EjR27/zgOMMbzDwyHyE2Tyw9FBJiUlxWG3nR/9+vUzPj4+t+yXlpZm/P39Tbt27bJdnpCQkKUtJSXF9OjRwzg5OZmOHTua6dOnm6+//tp8/vnnZsyYMebee+81gYGB5osvvsiybq1atUzVqlXtAtbu3buNs7OzGT58uK3t5MmTRpIZMWKE3frr1683ksyUKVNyndcff/xhJJnJkyfn2u923Oo5cOzYMZORkWGMMcbHxyfHIPPZZ58ZSWbx4sW2tjNnzpjAwEDTu3dvu761atUydevWNWlpaba20aNHGycnJ3PgwIFCHfNWMgPw0aNHc+1ntdcOHIcgA4fIT5CpUKGC3Zv8tWvXzLhx40zVqlWNh4eHCQoKMs2bNzerV682xhjTp08fIynLJVNKSooZPny4KVu2rHF3dzfVqlUzkydPtn2oZLp8+bIZNmyYKVmypPH19TWdO3e2fWCOHTvW1m/s2LFGktm3b5/p3bu3CQwMNOHh4cYYY3755RfTp08fU6lSJePh4WFCQkJMv379zNmzZ+1uK3OM2NhY8+STTxp/f39TqlQp8+qrr5qMjAxz/Phx06VLF+Pn52dCQkLM22+/naf7Oy0tzYwfP95UrlzZuLu7mwoVKphRo0aZq1ev2vpkd1/NnTs32/Hi4uKMJDNu3Lg8337Lli1N+fLlzY8//phjn7feesu4u7ubr7/+2tZ+7tw5I8m89NJLWdapXbu2CQsLs10/cOBAtkEks33WrFk51ph53998qVChgm35rl27TIcOHYyfn5/x8fExrVu3Ntu2bbMbI/N5vXHjRvPMM8+Y4OBgExgYmOt9c7Pcgsyjjz5qQkJCTHp6ul37oEGDjLe3t+2x3Ldvn5FkZs6cadfv1KlTRpJ5/fXXC3XMW8kuyPTp08f4+PiYw4cPm44dOxpfX1/TtWtXY8yNrUZTp041tWrVMh4eHqZ06dJm0KBB5vz581nG/uabb8yDDz5ovL29ja+vr3n44YfN3r177frExcWZvn37mnvuuce4u7ub0NBQ06VLl1sGKxRfroW1ywrIi6SkJJ09ezZLe1pa2i3XHTdunKKjozVgwAA1btxYycnJ2rlzp3bt2qV27dpp8ODBOn36tNasWaMFCxbYrWuMUZcuXbRhwwb1799f4eHh+u677/TSSy/p1KlTmjp1qq1v37599fnnn+upp55S06ZNtWnTJnXq1CnHuh599FHde++9evPNN2WMkXTj+I7ff/9d/fr1U2hoqPbt26c5c+Zo37592r59u5ycnOzGePzxx1WzZk1NnDhRK1eu1IQJExQUFKT33ntPrVu31ltvvaVPPvlEI0aMUKNGjdSiRYtc76sBAwZo/vz56tmzp1588UXt2LFD0dHROnDggJYuXSpJWrBggebMmaMff/xRH3zwgSTpgQceyHa80qVLy8vLS1999ZWGDRumoKCgXG8/OjpasbGxiomJUZkyZSRJGRkZunLlinx8fJSRkaHExES9/PLL8vPz07///W8dPnxYfn5+Sk1NlSR5eXllGdfb21v79u1TfHy8QkNDVaVKFZUtW1bvvPOOqlevrnr16un06dN6+eWXValSJfXq1SvHGrt3767AwEC98MIL6t27tx5++GH5+vpKkvbt26eHHnpI/v7+evnll+Xm5qb33ntPLVu21KZNm9SkSRO7sZ599lkFBwdrzJgxunTpUq73TV79/PPPql+/vpyd7Q9tbNy4sebMmaODBw/qvvvu088//yxJWY7tCQsLU9myZW3LC2vM/Lp+/boiIyP14IMP6u2335a3t7ckafDgwZo3b5769eun5557TkePHtWMGTP0888/64cffpCbm5ukG8/fPn36KDIyUm+99ZYuX76sWbNm6cEHH9TPP/+sihUrSpJ69Oihffv2adiwYapYsaLOnDmjNWvW6Pjx47Y+sBhHJyn8PWX+5Zrb5VZbZOrWrWs6deqU6+3ktGtp2bJlRpKZMGGCXXvPnj2Nk5OTOXz4sDHGmJiYGCPJPP/883b9+vbtm+MWmb9ukjfmxladv/r000+NJPP9999nGWPQoEG2tuvXr5uyZcsaJycnM3HiRFv7hQsXjJeXV67HoxhzYxeMJDNgwAC79hEjRhhJZv369ba2zL+M82LMmDFGkvHx8TEdO3Y0b7zxhomJicnSLykpyfj7+5tly5bZ2ubMmWNKlChhe5y//PJLu8epfv36Zs6cOcaYG3+RBwYGmjZt2tiNe/bsWePj42MkmZ07d9rad+zYYapUqWL3XGrQoEGOx9/c7OjRo9lu0enWrZtxd3e3O47j9OnTxs/Pz7Ro0cLWlvm8fvDBB83169dveXt/ldsWGR8fH/Pvf/87S/vKlSuNJLNq1SpjzP9t8Th+/HiWvo0aNTJNmzYt1DFvJactMpLMyJEj7fpu3rzZSDKffPKJXfuqVavs2i9evGgCAwPNwIED7frFx8ebgIAAW/uFCxcKfNchHI+zluBQM2fO1Jo1a7Jc7r///luuGxgYqH379unQoUO3fbvffPONXFxc9Nxzz9m1v/jiizLG6Ntvv5Uk29k3zz77rF2/YcOG5Tj2008/naXt5q0JV69e1dmzZ9W0aVNJyvYMkQEDBtj+7+LiooYNG8oYo/79+9vaAwMDVb16df3+++851iLdmKskDR8+3K79xRdflCStXLky1/VzEhUVpYULF6pevXr67rvvNHr0aDVo0ED169fXgQMHbP1Wr16toKAgdenSRdKN+Q4ePFg9evTQ0qVL9fjjj2vgwIF2Y3ft2lUbN26UJDk7O2vw4MFat26dRo0apUOHDikmJkaPPfaYrl27Jkm6cuWKbd0SJUooPDxcI0eO1LJly/T222/r2LFjevTRR3X16tXbnmd6erpWr16tbt26qXLlyrb2MmXK6IknntCWLVuUnJxst87AgQPl4uJy27eVmytXrsjDwyNLu6enp235zf/m1Pfm+6owxrwTzzzzjN31xYsXKyAgQO3atdPZs2dtlwYNGsjX11cbNmyQdGOLZ2Jionr37m3Xz8XFRU2aNLH18/Lykru7uzZu3KgLFy4USM1wPHYtwaEaN26c7emtJUqUyHaX083Gjx+vrl27qlq1aqpTp446dOigp556Kk8h6I8//lBYWJj8/Pzs2mvWrGlbnvmvs7OzKlWqZNevatWqOY79176SdP78eUVFRWnRokU6c+aM3bKkpKQs/cuXL293PSAgQJ6enipVqlSW9nPnzuVYy81z+GvNoaGhCgwMtM01P3r37q3evXsrOTlZO3bs0Lx587Rw4UJ17txZe/fulaenp2JiYhQREWHbffbBBx+oZcuWev/99yVJ3bp1U3p6uqKiomzjhoSEaMuWLbbr48eP19mzZzVp0iRNnDhRktS+fXv1799fs2fPtu0CSkpK0kMPPaSXXnrJFtSkG7tEWrZsqblz52b5sLyVP//8U5cvX1b16tWzLKtZs6YyMjJ04sQJ1a5d29ae3XPgTnl5edl2s90sM5xlhuXMf3Pqe3OoLowx88vV1VVly5a1azt06JCSkpJUunTpbNfJfC1l/jHTunXrbPv5+/tLuhHE3nrrLb344osKCQlR06ZN9Y9//EP/+te/FBoaesdzgGMQZGBZLVq00JEjR7R8+XKtXr1aH3zwgaZOnarZs2fbbdEoatm9qT/22GPaunWrXnrpJYWHh8vX11cZGRnq0KGDMjIysvTP7q/5nP7CN///cTi38tfjcAqSv7+/2rVrp3bt2snNzU3z58/Xjh07FBERoXPnziksLMzW99ixY2rUqJHd+o0bN7a7fuLECZUsWdJ23d3dXR988IHeeOMNHTx4UCEhIapWrZqeeOIJu5D25ZdfKiEhwbb1J1NERIT8/f31ww8/3HaQyY+C+GD/qzJlyiguLi5Le2Zb5n2ceQxSXFycypUrl6Xvzfd1YYyZXx4eHlmO1cnIyFDp0qX1ySefZLtOcHCwrZ904ziZ7AKJq+v/fdQ9//zz6ty5s5YtW6bvvvtOr732mqKjo7V+/XrVq1fvjueBokeQgaUFBQWpX79+6tevn1JSUtSiRQuNGzfOFmRy+vCuUKGC1q5dq4sXL9ptlfntt99syzP/zcjI0NGjR3Xvvffa+h0+fDjPNV64cEHr1q1TVFSUxowZY2vPzy6x/Micw6FDh2xbnCQpISFBiYmJtrkWlIYNG2r+/Pm2D0N/f3+7rU6hoaE6cuSI3To37x67evWqFixYYHdfZQoJCVFISIikG7t8Nm7cqCZNmti2yCQkJNiW3cwYo/T09Ft+eVt2goOD5e3trdjY2CzLfvvtNzk7O2f5cC8M4eHh2rx5szIyMuw+8Hfs2CFvb29Vq1bN1k+Sdu7caRcwTp8+rZMnT2rQoEGFOmZBqlKlitauXavmzZvnGg6rVKki6cZB6G3bts3TuC+++KJefPFFHTp0SOHh4XrnnXf08ccfF1jtKDocIwPL+usuFV9fX1WtWtVu87ePj4+kG99eerOHH35Y6enpmjFjhl371KlT5eTkpI4dO0qSIiMjJUnvvvuuXb/p06fnuc7MLSl/3XIybdq0PI9xJx5++OFsb2/KlCmSlOsZWDm5fPmytm3blu2yzOOLMnfF1KxZUzt27LAtf+SRR7R06VLNnDlTf/zxh7755hvbNwhv3rxZ7du3V4kSJfTPf/4z1xrefvttxcXF2e1CyvzgXbRokV3fFStW6NKlS/n6i9vFxUXt27fX8uXL7b5JOCEhQQsXLtSDDz5o23VRmHr27KmEhAQtWbLE1nb27FktXrxYnTt3th2/Urt2bdWoUUNz5syxC3SzZs2Sk5OTevbsWahjFqTHHntM6enpev3117Msu379uu11HRkZKX9/f7355pvZnvH4559/SrrxvP3rcVJVqlSxOzsO1sMWGVhWrVq11LJlSzVo0EBBQUHauXOnvvjiCw0dOtTWp0GDBpKk5557TpGRkXJxcVGvXr3UuXNntWrVSqNHj9axY8dUt25drV69WsuXL9fzzz9v+wuvQYMG6tGjh6ZNm6Zz587ZTr8+ePCgpLztrvH391eLFi00adIkpaWl6Z577tHq1at19OjRQrhXsqpbt6769OmjOXPmKDExUREREfrxxx81f/58devWTa1atbrtMS9fvqwHHnhATZs2VYcOHVSuXDklJiZq2bJl2rx5s7p162YLDR06dNDTTz+tn3/+WfXq1VPnzp01ePBgDR06VEOHDpW3t7eioqL00ksvqWXLlurZs6eWLFlid2Dpxx9/rC+//FItWrSQr6+v1q5dq88//1wDBgxQjx49bP06d+6s2rVra/z48frjjz/UtGlTHT58WDNmzFCZMmXsDpa+HRMmTLD9RMKzzz4rV1dXvffee0pNTdWkSZPyNWamr776Sr/88oukG187sGfPHk2YMEGS1KVLF9sxXz179lTTpk3Vr18/7d+/X6VKldK7776b5fgi6cbvZXXp0kXt27dXr169tHfvXs2YMUMDBgyw2ypXGGMWpIiICA0ePFjR0dHavXu32rdvLzc3Nx06dEiLFy/Wf//7X/Xs2VP+/v6aNWuWnnrqKdWvX1+9evVScHCwjh8/rpUrV6p58+aaMWOGDh48qDZt2uixxx5TrVq15OrqqqVLlyohISHXU/NRzDn0nCn8bRXEF+JNmDDBNG7c2AQGBhovLy9To0YN88Ybb9j9jMH169fNsGHDTHBwsHFycrI7xffixYvmhRdeMGFhYcbNzc3ce++92X4h3qVLl8yQIUNMUFCQ8fX1Nd26dTOxsbFGkt3p0JmnTv/5559Z5nPy5EnzyCOPmMDAQBMQEGAeffRRc/r06RxP4f7rGDmdFp3Xb0BOS0szUVFRplKlSsbNzc2UK1cuyxfi5XY72Y33/vvvm27dupkKFSoYDw8P4+3tberVq2cmT56c5ScO+vTpY5o0aWLXfuTIEbN582Zz4cIFc+XKFbNt2zaTmJiY7e3t2LHDtGjRwpQoUcJ4enqaunXrmtmzZ2d5rIwx5vz58+aFF14w1apVMx4eHqZUqVKmV69e5vfff7/lvHI6/dqYG1+IFxkZaXx9fY23t7dp1aqV2bp1q12fWz2vs5PTFzcqmy8kPH/+vOnfv78pWbKk8fb2NhERETne1tKlS014eLjx8PAwZcuWNa+++mq2P/FRGGPmJrcvxMvJnDlzTIMGDYyXl5fx8/Mz9913n3n55ZfN6dOn7fpt2LDBREZGmoCAAOPp6WmqVKli+vbtazs9/+zZs2bIkCGmRo0axsfHxwQEBJgmTZqYzz///LbmgOLFyZg8HikIwGb37t2qV6+ePv74Yz355JOOLqfYyzxltk6dOvr000+z3RWTnp6upUuXFtpuCgB3J4IMcAtXrlzJcqBh3759tWDBAh07dqxIDvS8Gxw8eFCdOnVScnKyhg4dqnbt2iksLEzJycnasmWLZsyYofj4eO3atSvL6ecAkBOCDHALUVFRiomJUatWreTq6qpvv/1W3377rQYNGqT33nvP0eVZysWLFzV58mR98MEHdqf9+vn56cknn9SYMWNsp/oCQF4QZIBbWLNmjaKiorR//36lpKSofPnyeuqppzR69Gi776dA3hljdPjwYcXHx8vf3181a9aUu7u7o8sCYEEEGQAAYFl8jwwAALAsggwAALCsu34Hf0ZGhk6fPi0/P79C/a0ZAABQcIwxunjxosLCwrL8DtfN7vogc/r0aU6PBQDAok6cOJHll9FvdtcHmcwfBDxx4kSR/B4KAAC4c8nJySpXrpzdD/tm564PMpm7k/z9/QkyAABYzK0OC+FgXwAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkODTLR0dFq1KiR/Pz8VLp0aXXr1k2xsbF2fVq2bCknJye7y9NPP+2gigEAQHHi0CCzadMmDRkyRNu3b9eaNWuUlpam9u3b69KlS3b9Bg4cqLi4ONtl0qRJDqoYAAAUJw790chVq1bZXZ83b55Kly6tmJgYtWjRwtbu7e2t0NDQoi4PAAAUc8XqGJmkpCRJUlBQkF37J598olKlSqlOnToaNWqULl++7IjyAABAMePQLTI3y8jI0PPPP6/mzZurTp06tvYnnnhCFSpUUFhYmPbs2aP//Oc/io2N1ZIlS7IdJzU1VampqbbrycnJhV47AABwjGITZIYMGaK9e/dqy5Ytdu2DBg2y/f++++5TmTJl1KZNGx05ckRVqlTJMk50dLSioqIKvV7gblRx5EpHl3Dbjk3s5OgSADhQsdi1NHToUH399dfasGGDypYtm2vfJk2aSJIOHz6c7fJRo0YpKSnJdjlx4kSB1wsAAIoHh26RMcZo2LBhWrp0qTZu3KhKlSrdcp3du3dLksqUKZPtcg8PD3l4eBRkmQAAoJhyaJAZMmSIFi5cqOXLl8vPz0/x8fGSpICAAHl5eenIkSNauHChHn74YZUsWVJ79uzRCy+8oBYtWuj+++93ZOkAAKAYcGiQmTVrlqQbX3p3s7lz56pv375yd3fX2rVrNW3aNF26dEnlypVTjx499OqrrzqgWgAAUNw4fNdSbsqVK6dNmzYVUTUAAMBqisXBvgAAAPlBkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJZFkAEAAJbl6ugCAOBOVBy50tEl3LZjEzs5ugTgrsEWGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkEGQAAYFkODTLR0dFq1KiR/Pz8VLp0aXXr1k2xsbF2fa5evaohQ4aoZMmS8vX1VY8ePZSQkOCgigEAQHHi0CCzadMmDRkyRNu3b9eaNWuUlpam9u3b69KlS7Y+L7zwgr766istXrxYmzZt0unTp9W9e3cHVg0AAIoLh/7W0qpVq+yuz5s3T6VLl1ZMTIxatGihpKQkffjhh1q4cKFat24tSZo7d65q1qyp7du3q2nTpo4oGwAAFBPF6hiZpKQkSVJQUJAkKSYmRmlpaWrbtq2tT40aNVS+fHlt27Yt2zFSU1OVnJxsdwEAAHenYhNkMjIy9Pzzz6t58+aqU6eOJCk+Pl7u7u4KDAy06xsSEqL4+Phsx4mOjlZAQIDtUq5cucIuHQAAOEixCTJDhgzR3r17tWjRojsaZ9SoUUpKSrJdTpw4UUAVAgCA4sahx8hkGjp0qL7++mt9//33Klu2rK09NDRU165dU2Jiot1WmYSEBIWGhmY7loeHhzw8PAq7ZAAAUAw4dIuMMUZDhw7V0qVLtX79elWqVMlueYMGDeTm5qZ169bZ2mJjY3X8+HE1a9asqMsFAADFjEO3yAwZMkQLFy7U8uXL5efnZzvuJSAgQF5eXgoICFD//v01fPhwBQUFyd/fX8OGDVOzZs04YwkAADg2yMyaNUuS1LJlS7v2uXPnqm/fvpKkqVOnytnZWT169FBqaqoiIyP17rvvFnGlAACgOHJokDHG3LKPp6enZs6cqZkzZxZBRQAAwEqKzVlLAAAAt4sgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALIsgAwAALMuhQeb7779X586dFRYWJicnJy1btsxued++feXk5GR36dChg2OKBQAAxY5Dg8ylS5dUt25dzZw5M8c+HTp0UFxcnO3y6aefFmGFAACgOHN15I137NhRHTt2zLWPh4eHQkNDi6giAABgJcX+GJmNGzeqdOnSql69up555hmdO3fO0SUBAIBiwqFbZG6lQ4cO6t69uypVqqQjR47olVdeUceOHbVt2za5uLhku05qaqpSU1Nt15OTk4uqXAAAUMSKdZDp1auX7f/33Xef7r//flWpUkUbN25UmzZtsl0nOjpaUVFRRVUikKOKI1c6ugQAuOsV+11LN6tcubJKlSqlw4cP59hn1KhRSkpKsl1OnDhRhBUCAICiVKy3yPzVyZMnde7cOZUpUybHPh4eHvLw8CjCqgAAgKM4NMikpKTYbV05evSodu/eraCgIAUFBSkqKko9evRQaGiojhw5opdffllVq1ZVZGSkA6sGAADFRb52Le3atUu//vqr7fry5cvVrVs3vfLKK7p27Vqex9m5c6fq1aunevXqSZKGDx+uevXqacyYMXJxcdGePXvUpUsXVatWTf3791eDBg20efNmtrgAAABJ+dwiM3jwYI0cOVL33Xeffv/9d/Xq1UuPPPKIFi9erMuXL2vatGl5Gqdly5YyxuS4/LvvvstPeQAA4G8iX1tkDh48qPDwcEnS4sWL1aJFCy1cuFDz5s3Tl19+WZD1AQAA5ChfQcYYo4yMDEnS2rVr9fDDD0uSypUrp7NnzxZcdQAAALnIV5Bp2LChJkyYoAULFmjTpk3q1KmTpBsH64aEhBRogQAAADnJV5CZOnWqdu3apaFDh2r06NGqWrWqJOmLL77QAw88UKAFAgAA5CRfB/vWrVvX7qylTJMnT5arq6W+mgYAAFhYvrbIVK5cOdsfb7x69aqqVat2x0UBAADkRb6CzLFjx5Senp6lPTU1VSdPnrzjogAAAPLitvYDrVixwvb/7777TgEBAbbr6enpWrdunSpVqlRw1QEAAOTitoJMt27dJElOTk7q06eP3TI3NzdVrFhR77zzToEVBwAAkJvbCjKZ3x1TqVIl/fTTTypVqlShFAUAAJAX+TrF6OjRowVdBwAAwG3L97nS69at07p163TmzBnblppMH3300R0XBgAAcCv5CjJRUVEaP368GjZsqDJlysjJyamg6wIAALilfAWZ2bNna968eXrqqacKuh4AAIA8y9f3yFy7do2fIgAAAA6XryAzYMAALVy4sKBrAQAAuC352rV09epVzZkzR2vXrtX9998vNzc3u+VTpkwpkOIAAAByk68gs2fPHoWHh0uS9u7da7eMA38BAEBRyVeQ2bBhQ0HXAQAAcNvydYwMAABAcZCvLTKtWrXKdRfS+vXr810QAABAXuUryGQeH5MpLS1Nu3fv1t69e7P8mCQAAEBhyVeQmTp1arbt48aNU0pKyh0VBAAAkFcFeozMP//5T35nCQAAFJkCDTLbtm2Tp6dnQQ4JAACQo3ztWurevbvddWOM4uLitHPnTr322msFUhgAAMCt5CvIBAQE2F13dnZW9erVNX78eLVv375ACgMAALiVfAWZuXPnFnQdAAAAty1fQSZTTEyMDhw4IEmqXbu26tWrVyBFAQAA5EW+gsyZM2fUq1cvbdy4UYGBgZKkxMREtWrVSosWLVJwcHBB1ggAAJCtfJ21NGzYMF28eFH79u3T+fPndf78ee3du1fJycl67rnnCrpGAACAbOVri8yqVau0du1a1axZ09ZWq1YtzZw5k4N9AQBAkcnXFpmMjAy5ubllaXdzc1NGRsYdFwUAAJAX+QoyrVu31v/8z//o9OnTtrZTp07phRdeUJs2bQqsOAAAgNzkK8jMmDFDycnJqlixoqpUqaIqVaqoUqVKSk5O1vTp0wu6RgAAgGzl6xiZcuXKadeuXVq7dq1+++03SVLNmjXVtm3bAi0OAAAgN7cVZNavX6+hQ4dq+/bt8vf3V7t27dSuXTtJUlJSkmrXrq3Zs2froYceKpRi8fdVceRKR5cAACiGbmvX0rRp0zRw4ED5+/tnWRYQEKDBgwdrypQpBVYcAABAbm4ryPzyyy/q0KFDjsvbt2+vmJiYOy4KAAAgL24ryCQkJGR72nUmV1dX/fnnn3dcFAAAQF7cVpC55557tHfv3hyX79mzR2XKlLnjogAAAPLitoLMww8/rNdee01Xr17NsuzKlSsaO3as/vGPfxRYcQAAALm5rbOWXn31VS1ZskTVqlXT0KFDVb16dUnSb7/9ppkzZyo9PV2jR48ulEIBAAD+6raCTEhIiLZu3apnnnlGo0aNkjFGkuTk5KTIyEjNnDlTISEhhVIoAADAX932F+JVqFBB33zzjS5cuKDDhw/LGKN7771XJUqUKIz6AAAAcpSvb/aVpBIlSqhRo0YFWQsAAMBtyddvLQEAABQHBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZDg0y33//vTp37qywsDA5OTlp2bJldsuNMRozZozKlCkjLy8vtW3bVocOHXJMsQAAoNhxaJC5dOmS6tatq5kzZ2a7fNKkSfrf//1fzZ49Wzt27JCPj48iIyN19erVIq4UAAAUR66OvPGOHTuqY8eO2S4zxmjatGl69dVX1bVrV0nS//t//08hISFatmyZevXqVZSlAgCAYqjYHiNz9OhRxcfHq23btra2gIAANWnSRNu2bctxvdTUVCUnJ9tdAADA3anYBpn4+HhJUkhIiF17SEiIbVl2oqOjFRAQYLuUK1euUOsEAACOU2yDTH6NGjVKSUlJtsuJEyccXRIAACgkxTbIhIaGSpISEhLs2hMSEmzLsuPh4SF/f3+7CwAAuDsV2yBTqVIlhYaGat26dba25ORk7dixQ82aNXNgZQAAoLhw6FlLKSkpOnz4sO360aNHtXv3bgUFBal8+fJ6/vnnNWHCBN17772qVKmSXnvtNYWFhalbt26OKxoAABQbDg0yO3fuVKtWrWzXhw8fLknq06eP5s2bp5dfflmXLl3SoEGDlJiYqAcffFCrVq2Sp6eno0oGAADFiJMxxji6iMKUnJysgIAAJSUlcbyMhVUcudLRJQAF5tjETo4uASj28vr5XWyPkQEAALgVggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsggwAALAsV0cXAAAo/iqOXOnoEm7bsYmdHF0CigBbZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGXx69cAUMSs+EvSQHHFFhkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZBBkAAGBZxTrIjBs3Tk5OTnaXGjVqOLosAABQTLg6uoBbqV27ttauXWu77upa7EsGAABFpNinAldXV4WGhjq6DAAAUAwV611LknTo0CGFhYWpcuXKevLJJ3X8+PFc+6empio5OdnuAgAA7k5Oxhjj6CJy8u233yolJUXVq1dXXFycoqKidOrUKe3du1d+fn7ZrjNu3DhFRUVlaU9KSpK/v39hl4xCUnHkSkeXAACF7tjETo4uodhITk5WQEDALT+/i3WQ+avExERVqFBBU6ZMUf/+/bPtk5qaqtTUVNv15ORklStXjiBjcQQZAH8HBJn/k9cgU+yPkblZYGCgqlWrpsOHD+fYx8PDQx4eHkVYFQAAcJRif4zMzVJSUnTkyBGVKVPG0aUAAIBioFgHmREjRmjTpk06duyYtm7dqkceeUQuLi7q3bu3o0sDAADFQLHetXTy5En17t1b586dU3BwsB588EFt375dwcHBji4NAAAUA8U6yCxatMjRJQAAgGKsWO9aAgAAyA1BBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWJarowuwsoojVzq6hHw5NrGTo0sAAKBAsEUGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYFkEGAABYlqujC0DRqzhypaNLAABkw4rvz8cmdnLo7bNFBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWBZBBgAAWJYlgszMmTNVsWJFeXp6qkmTJvrxxx8dXRIAACgGin2Q+eyzzzR8+HCNHTtWu3btUt26dRUZGakzZ844ujQAAOBgxT7ITJkyRQMHDlS/fv1Uq1YtzZ49W97e3vroo48cXRoAAHCwYh1krl27ppiYGLVt29bW5uzsrLZt22rbtm0OrAwAABQHro4uIDdnz55Venq6QkJC7NpDQkL022+/ZbtOamqqUlNTbdeTkpIkScnJyQVeX0bq5QIfEwAAKymMz9ebxzXG5NqvWAeZ/IiOjlZUVFSW9nLlyjmgGgAA7m4B0wp3/IsXLyogICDH5cU6yJQqVUouLi5KSEiwa09ISFBoaGi264waNUrDhw+3Xc/IyND58+dVsmRJOTk5FWq9BSE5OVnlypXTiRMn5O/v7+hyChVzvXv9nebLXO9OzNXxjDG6ePGiwsLCcu1XrIOMu7u7GjRooHXr1qlbt26SbgSTdevWaejQodmu4+HhIQ8PD7u2wMDAQq604Pn7+xerJ1RhYq53r7/TfJnr3Ym5OlZuW2IyFesgI0nDhw9Xnz591LBhQzVu3FjTpk3TpUuX1K9fP0eXBgAAHKzYB5nHH39cf/75p8aMGaP4+HiFh4dr1apVWQ4ABgAAfz/FPshI0tChQ3PclXS38fDw0NixY7PsHrsbMde7199pvsz17sRcrcPJ3Oq8JgAAgGKqWH8hHgAAQG4IMgAAwLIIMgAAwLIIMgAAwLIIMgVs5syZqlixojw9PdWkSRP9+OOPOfZ9//339dBDD6lEiRIqUaKE2rZtm23/AwcOqEuXLgoICJCPj48aNWqk48eP25a3bNlSTk5Odpenn366UOb3VwU937/OI/MyefJkW5/z58/rySeflL+/vwIDA9W/f3+lpKQU2hwzOWKuFStWzLJ84sSJhTbHTAU915SUFA0dOlRly5aVl5eX7Zfsb3b16lUNGTJEJUuWlK+vr3r06JHlW70LgyPm6qjXbEHPNSEhQX379lVYWJi8vb3VoUMHHTp0yK7P3fK45mWuVnhclyxZooYNGyowMFA+Pj4KDw/XggUL7PoYYzRmzBiVKVNGXl5eatu2bZa5Oup9OFsGBWbRokXG3d3dfPTRR2bfvn1m4MCBJjAw0CQkJGTb/4knnjAzZ840P//8szlw4IDp27evCQgIMCdPnrT1OXz4sAkKCjIvvfSS2bVrlzl8+LBZvny53ZgRERFm4MCBJi4uznZJSkqy5HxvnkNcXJz56KOPjJOTkzly5IitT4cOHUzdunXN9u3bzebNm03VqlVN796978q5VqhQwYwfP96uX0pKiuXmOnDgQFOlShWzYcMGc/ToUfPee+8ZFxcXs3z5clufp59+2pQrV86sW7fO7Ny50zRt2tQ88MADd+VcHfGaLei5ZmRkmKZNm5qHHnrI/Pjjj+a3334zgwYNMuXLl7d7jt4Nj2te52qFx3XDhg1myZIlZv/+/ebw4cNm2rRpxsXFxaxatcrWZ+LEiSYgIMAsW7bM/PLLL6ZLly6mUqVK5sqVK7Y+jngfzglBpgA1btzYDBkyxHY9PT3dhIWFmejo6Dytf/36dePn52fmz59va3v88cfNP//5z1zXi4iIMP/zP/+Tr5rvRGHM96+6du1qWrdubbu+f/9+I8n89NNPtrZvv/3WODk5mVOnTuVjFnnjiLkacyPITJ06NV8151dhzLV27dpm/Pjxdv3q169vRo8ebYwxJjEx0bi5uZnFixfblh84cMBIMtu2bbuT6eTKEXM1xjGv2YKea2xsrJFk9u7dazdmcHCwef/9940xd8/jmpe5GmPNx9UYY+rVq2deffVVY8yN0BYaGmomT55sW56YmGg8PDzMp59+aoxx3PtwTti1VECuXbummJgYtW3b1tbm7Oystm3batu2bXka4/Lly0pLS1NQUJCkG78rtXLlSlWrVk2RkZEqXbq0mjRpomXLlmVZ95NPPlGpUqVUp04djRo1SpcvXy6QeeWkMOb7VwkJCVq5cqX69+9va9u2bZsCAwPVsGFDW1vbtm3l7OysHTt25HM2uXPUXDNNnDhRJUuWVL169TR58mRdv349fxPJg8Ka6wMPPKAVK1bo1KlTMsZow4YNOnjwoNq3by9JiomJUVpamt3t1qhRQ+XLl8/z7d4uR801U1G+ZgtjrqmpqZIkT09PuzE9PDy0ZcsWSXfP45qXuWay0uNqjNG6desUGxurFi1aSJKOHj2q+Ph4uzEDAgLUpEkT25iOeB/OjSW+2dcKzp49q/T09Cw/nRASEqLffvstT2P85z//UVhYmO0JdObMGaWkpGjixImaMGGC3nrrLa1atUrdu3fXhg0bFBERIUl64oknVKFCBYWFhWnPnj36z3/+o9jYWC1ZsqRgJ3mTwpjvX82fP19+fn7q3r27rS0+Pl6lS5e26+fq6qqgoCDFx8ff5izyxlFzlaTnnntO9evXV1BQkLZu3apRo0YpLi5OU6ZMyd9kbqGw5jp9+nQNGjRIZcuWlaurq5ydnfX+++/b3jzj4+Pl7u6e5QdeQ0JCLPe43mquUtG/ZgtjrpmBZNSoUXrvvffk4+OjqVOn6uTJk4qLi5N09zyueZmrZJ3HNSkpSffcc49SU1Pl4uKid999V+3atZMk2+OS3ZiZyxzxPpwbgkwxMXHiRC1atEgbN260pf6MjAxJUteuXfXCCy9IksLDw7V161bNnj3bFmQGDRpkG+e+++5TmTJl1KZNGx05ckRVqlQp4pnkTXbz/auPPvpITz75ZI7LreJO5jp8+HDb/++//365u7tr8ODBio6OLpZfJ57TXKdPn67t27drxYoVqlChgr7//nsNGTIk13BX3N3JXK32ms1urm5ublqyZIn69++voKAgubi4qG3bturYsaOMhb8w/k7mapXH1c/PT7t371ZKSorWrVun4cOHq3LlymrZsqWjS8sXgkwBKVWqlFxcXLIcjZ+QkKDQ0NBc13377bc1ceJErV27Vvfff7/dmK6urqpVq5Zd/5o1a2bZnHmzJk2aSJIOHz5caC+ewpjvzTZv3qzY2Fh99tlndu2hoaE6c+aMXdv169d1/vz5W95ufjlqrtlp0qSJrl+/rmPHjql69ep5n0QeFcZcr1y5oldeeUVLly5Vp06dJN0IZbt379bbb7+ttm3bKjQ0VNeuXVNiYqLdX+95ud38ctRcs1PYr9nCeg43aNBAu3fvVlJSkq5du6bg4GA1adLEtsvhbnlcpVvPNTvF9XF1dnZW1apVJd344/jAgQOKjo5Wy5YtbeslJCSoTJkydmOGh4dLcsz7cG44RqaAuLu7q0GDBlq3bp2tLSMjQ+vWrVOzZs1yXG/SpEl6/fXXtWrVqiwvCHd3dzVq1EixsbF27QcPHlSFChVyHHP37t2SZPckLGiFMd+bffjhh2rQoIHq1q1r196sWTMlJiYqJibG1rZ+/XplZGTY3jQKmqPmmp3du3fL2dk5y2bdglIYc01LS1NaWpqcne3fblxcXGxbHRs0aCA3Nze7242NjdXx48dzvd074ai5ZqewX7OF/RwOCAhQcHCwDh06pJ07d6pr166S7p7H9WY5zTU7xfVx/auMjAzbcUCVKlVSaGio3ZjJycnasWOHbUxHvA/nqsgPL76LLVq0yHh4eJh58+aZ/fv3m0GDBpnAwEATHx9vjDHmqaeeMiNHjrT1nzhxonF3dzdffPGF3el6Fy9etPVZsmSJcXNzM3PmzDGHDh0y06dPNy4uLmbz5s3GmBunZ48fP97s3LnTHD161CxfvtxUrlzZtGjRwpLzNcaYpKQk4+3tbWbNmpXt7Xbo0MHUq1fP7Nixw2zZssXce++9RXL6dVHPdevWrWbq1Klm9+7d5siRI+bjjz82wcHB5l//+pfl5hoREWFq165tNmzYYH7//Xczd+5c4+npad59911bn6efftqUL1/erF+/3uzcudM0a9bMNGvW7K6bq6Nes4Ux188//9xs2LDBHDlyxCxbtsxUqFDBdO/e3e5275bH9VZztcrj+uabb5rVq1ebI0eOmP3795u3337buLq62p19NXHiRBMYGGiWL19u9uzZY7p27Zrt6ddF/T6cE4JMAZs+fbopX768cXd3N40bNzbbt2+3LYuIiDB9+vSxXa9QoYKRlOUyduxYuzE//PBDU7VqVePp6Wnq1q1rli1bZlt2/Phx06JFCxMUFGQ8PDxM1apVzUsvvVQk3yNjTOHM97333jNeXl4mMTEx29s8d+6c6d27t/H19TX+/v6mX79+WQJCYSjqucbExJgmTZqYgIAA4+npaWrWrGnefPNNc/Xq1cKaok1BzzUuLs707dvXhIWFGU9PT1O9enXzzjvvmIyMDFufK1eumGeffdaUKFHCeHt7m0ceecTExcXddXN15Gu2oOf63//+15QtW9a4ubmZ8uXLm1dffdWkpqba3ebd8rjeaq5WeVxHjx5t+zwpUaKEadasmVm0aJHdeBkZGea1114zISEhxsPDw7Rp08bExsba9XHU+3B2nIyx8FFZAADgb41jZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAAAgGURZAA4xJ9//qlnnnlG5cuXl4eHh0JDQxUZGakffvjBrt+5c+c0btw4NWrUSMHBwSpfvrw6deqkRYsWZfsry7t27VK7du0UGBiokiVLatCgQUpJSbHr89NPP6lNmzYKDAxUiRIlFBkZqV9++aVQ5wugcBBkADhEjx499PPPP2v+/Pk6ePCgVqxYoZYtW+rcuXO2PqtXr1a1atX0008/acSIEVq9erWWLFmif/zjH3r99dcVGRmpS5cu2fqfPn1abdu2VdWqVbVjxw6tWrVK+/btU9++fW19UlJS1KFDB5UvX147duzQli1b5Ofnp8jISKWlpRXlXQCgAPATBQCKXGJiokqUKKGNGzcqIiIi2z47d+5UZGSk5s2bp86dO2dZfv36dQ0aNEhnz57VihUrJElz5szRa6+9pri4ONsvUP/666+6//77dejQIVWtWlU7d+5Uo0aNdPz4cZUrVy7bPgCsgy0yAIqcr6+vfH19tWzZMqWmpmbbZ9iwYXrjjTfUuXNn7d+/XxEREQoODtZjjz2m4cOHa9KkSZo9e7b279+vDRs2SJJSU1Pl7u5uCzGS5OXlJUnasmWLJKl69eoqWbKkPvzwQ127dk1XrlzRhx9+qJo1a6pixYqFO3EABY4gA6DIubq6at68eZo/f74CAwPVvHlzvfLKK9qzZ48k6dChQzp27JgGDBig9PR0PfLII6pRo4bWrFmjhx56SNOnT9e1a9fk7u6u3r1767vvvpMktW7dWvHx8Zo8ebKuXbumCxcuaOTIkZKkuLg4SZKfn582btyojz/+WF5eXvL19dWqVav07bffytXV1TF3CIB8I8gAcIgePXro9OnTWrFihTp06KCNGzeqfv36mjdvnn799Vc1atRIrq6uio2N1alTpzRjxgyFh4dr2LBhatmypW2cMmXK6MKFC5Kk2rVra/78+XrnnXfk7e2t0NBQVapUSSEhIbatNFeuXFH//v3VvHlzbd++XT/88IPq1KmjTp066cqVK464KwDcAYIMAIfx9PRUu3bt9Nprr2nr1q3q27evxo4dq+vXr9t2CV27dk1ubm5yc3Ozrefr62v7/65du+yOa3niiScUHx+vU6dO2c54+vPPP1W5cmVJ0sKFC3Xs2DHNnTtXjRo1UtOmTbVw4UIdPXpUy5cvL6KZAygoBBkAxUatWrV06dIlVa1aVb/++qukG8e0uLm5acaMGUpPT9f27dv13XffKS0tTfPnz9e3335rd1ZSppCQEPn6+uqzzz6zBSZJunz5spydneXk5GTrm3k9IyOjSOYJoOAQZAAUuXPnzql169b6+OOPtWfPHh09elSLFy/WpEmT1LVrV9WrV09XrlzRhg0b5OXlpXnz5mnMmDHy8PBQv3791K1bN7311luaO3euVq9ereDgYNvYM2bM0K5du3Tw4EHNnDlTQ4cOVXR0tAIDAyVJ7dq104ULFzRkyBAdOHBA+/btU79+/eTq6qpWrVo56B4BkF+cfg2gyKWmpmrcuHFavXq1jhw5orS0NJUrV06PPvqoXnnlFXl5eenzzz/XiBEj9P3336tixYpKT09XfHy8wsLClJSUJFdXV7tdTJn+9a9/aeXKlUpJSVGNGjU0YsQIPfXUU3Z91qxZo6ioKO3du1fOzs6qV6+e3njjDTVt2rSo7gIABYQgA6DYevPNNzVlyhSNGjVKjz/+uMqWLavU1FRt2rRJr7/+uoYPH65HHnnE0WUCcCCCDIBibfPmzZowYYI2btwoY4yuX7+uWrVq6bnnntOAAQPsvjMGwN8PQQaAJaSmpurMmTPy8/OzHe8CAAQZAABgWWyTBQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlvX/AXm+P9PjQsu8AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.hist(sas98s)\n",
    "ax.set(\n",
    "    xlabel=\"S@98\",\n",
    "    ylabel=\"Counts\",\n",
    "    title=\"Histogram of S@98 for 10000 Trees\"\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "034152c3-b914-4594-9352-a654ae98ca93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0a2c146bde84fa2b585a32df10d7006",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Reps Histogram S@98:   0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X, y = generate_data()\n",
    "sas98s = []\n",
    "for seed in trange(50, desc=\"Reps Histogram S@98\"):\n",
    "    est = HonestForestClassifier(\n",
    "        n_estimators=100000,\n",
    "        random_state=seed,\n",
    "        honest_fraction=0.5,\n",
    "        n_jobs=-1,\n",
    "        bootstrap=True,\n",
    "        stratify=True,\n",
    "        max_samples=1.6,\n",
    "        # permute_per_tree=True,\n",
    "    )\n",
    "    _, posterior_arr = build_hyppo_oob_forest(\n",
    "        est,\n",
    "        X,\n",
    "        y,\n",
    "        verbose=False\n",
    "    )\n",
    "    sas98 = sensitivity_at_specificity(\n",
    "        y, posterior_arr, target_specificity=0.98\n",
    "    )\n",
    "    sas98s.append(sas98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5f285a4f-7723-4052-b06a-034e00724353",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAHHCAYAAACvJxw8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/z0lEQVR4nO3deVhV5f7+8RtkcmBwQBDFWXNMnIcGHFBMsyjNIY+pR9NKraOpaeaUmqWldtIy6xz1VGZpaWbmkGZZkgNoKc7zCIoKOALC8/vDH/vbDlgCMrjt/bqufSnP+qy1P+sB2bdrrb22kzHGCAAAABlyLugGAAAA7maEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEJfwtVaxYUX369CnoNu5506dPV+XKlVWoUCEFBQUVdDv57uDBg2rXrp28vb3l5OSk5cuXF3RLAHKAsASHt2DBAjk5OWn79u0ZLm/ZsqXq1Klzx8+zatUqTZgw4Y6383exdu1ajRw5Ug888IDmz5+vN954w7L+22+/VXBwsEqXLq0iRYqocuXK6tq1q1avXp2l57t69apSUlKyVLtu3To9+OCDKlKkiIoXL64uXbro2LFj6epu3LihqVOnqlatWipSpIjKli2rp556SlFRUVl6nt69e2vXrl2aMmWKPvnkEzVq1ChL6+XUBx98oKeeekrly5eXk5OT5X8I4uLiNGDAAPn6+qpo0aJq1aqVIiMjM6xdsWKFGjRoIA8PD5UvX17jx4/XzZs377ptpkn7nXC7R8WKFS23A9gYwMHNnz/fSDLbtm3LcHlwcLCpXbu23diNGzdMUlJStp5n0KBBhn8yWffKK68YZ2dnk5iYeNva6dOnG0kmODjYzJgxw8ydO9cMHz7cBAUFmd69e2e63tq1a82TTz5pfHx8jCRTqFAhU7VqVTNq1Chz9uzZDNf59ttvjbOzs2nUqJF59913zaRJk0ypUqVM2bJlzblz5+xqn3zySePi4mKef/5589FHH5mJEyea0qVLG09PT3Ps2DHLfbp27ZqRZMaMGXPb/c8tFSpUMCVKlDDt27c3Li4umc5dSkqKadGihSlatKiZMGGCmT17tqlVq5bx9PQ0Bw4csKtdtWqVcXJyMq1atTLz5s0zQ4YMMc7Ozua55567q7b5Z4cPHzaffPKJ3cPd3d089NBDdmPLli3L+uTib43f/HB4OQlLOVHQYenKlSsF9tw50bdvX1O0aNHb1iUnJxsvLy/Ttm3bDJfHxMSkG7ty5Yrp3LmzcXJyMo888oh57733zMqVK82XX35pxo0bZ6pVq2Z8fHzM0qVL061bq1YtU7VqVbsQt3PnTuPs7GyGDRtmGzt16pSRZIYPH263/oYNG4wkM2PGDMv9On78uJFkpk+fblmXHbf7GTh27JhJTU01xhhTtGjRTMPSF198YSSZJUuW2MbOnTtnfHx8TI8ePexqa9WqZerVq2eSk5NtY2PGjDFOTk5m7969d802b8dqPtIkJydnKdzj74ewBIeXk7BUoUIFu1+cSUlJZsKECaZq1arG3d3dlChRwjzwwANm7dq1xhhjevfubSSle6S5cuWKGTZsmClXrpxxc3Mz1atXN9OnT7e9cKW5du2aGTJkiClZsqQpVqyY6dSpk+1Fefz48ba68ePHG0kmKirK9OjRw/j4+JigoCBjjDG///676d27t6lUqZJxd3c3fn5+pm/fviY2NtbuudK2sX//ftOzZ0/j5eVlSpUqZV577TWTmppqTpw4YR577DHj6elp/Pz8zNtvv52l+U5OTjavv/66qVy5snFzczMVKlQwo0ePNjdu3LDVZDRX8+fPz3B7Z8+eNZLMhAkTsvz8LVu2NOXLlzdbt27NtOatt94ybm5uZuXKlbbxCxcuGElmxIgR6dapXbu2CQgIsH29d+/eDMNO2vgHH3yQaY9pc//nR4UKFWzLIyMjTfv27Y2np6cpWrSoad26tQkPD7fbRtrP9caNG83zzz9vfH19jY+Pj+Xc/JlVOHjqqaeMn5+fSUlJsRsfMGCAKVKkiO17GRUVZSSZOXPm2NWdPn3aSDKTJk26K7aZFX+dj6NHj9q+vzNnzjSVK1c2zs7OZseOHcaYW9/nzp07m+LFixt3d3fTsGFD880336Tb7qVLl8xLL71k+7dfpUoV8+abb6br+fPPPzcNGjQwxYoVM56enqZOnTpm1qxZWe4fBcslz87vAfksPj5esbGx6caTk5Nvu+6ECRM0depU9e/fX02aNFFCQoK2b9+uyMhItW3bVgMHDtSZM2e0bt06ffLJJ3brGmP02GOP6ccff1S/fv0UFBSkNWvWaMSIETp9+rRmzpxpq+3Tp4++/PJL9erVS82aNdNPP/2kjh07ZtrXU089pWrVqumNN96QMUbSrettjhw5or59+8rf319RUVGaN2+eoqKi9Ntvv8nJycluG926dVPNmjX15ptv6rvvvtPkyZNVokQJffjhh2rdurXeeustffbZZxo+fLgaN26shx9+2HKu+vfvr4ULF6pLly56+eWXtWXLFk2dOlV79+7VsmXLJEmffPKJ5s2bp61bt+rjjz+WJLVo0SLD7ZUuXVqFCxfWt99+qyFDhqhEiRKWzz916lTt379fERERKlOmjCQpNTVV169fV9GiRZWamqq4uDiNHDlSnp6e+uc//6lDhw7J09NTiYmJkqTChQun226RIkUUFRWl6Oho+fv7q0qVKipXrpzeeecd3Xfffapfv77OnDmjkSNHqlKlSurevXumPT755JPy8fHR0KFD1aNHD3Xo0EHFihWTJEVFRemhhx6Sl5eXRo4cKVdXV3344Ydq2bKlfvrpJzVt2tRuWy+88IJ8fX01btw4Xb161XJusmrHjh1q0KCBnJ3tL1tt0qSJ5s2bpwMHDqhu3brasWOHJKW71iogIEDlypWzLS/obd6J+fPn68aNGxowYIDc3d1VokQJRUVF6YEHHlDZsmU1atQoFS1aVF9++aXCwsL01Vdf6YknnpAkXbt2TcHBwTp9+rQGDhyo8uXLa/PmzRo9erTOnj2rWbNmSbr1b7ZHjx5q06aN3nrrLUnS3r179euvv+qll166o/6RTwo6rQF3Ku1/4FaP2x1ZqlevnunYsaPl82R2Gm758uVGkpk8ebLdeJcuXYyTk5M5dOiQMcaYiIgII8n861//sqvr06dPpkeWMjrVcO3atXRjn3/+uZFkfv7553TbGDBggG3s5s2bply5csbJycm8+eabtvFLly6ZwoUL3/Y0xc6dO40k079/f7vx4cOHG0lmw4YNtrHevXtn6TScMcaMGzfOSDJFixY1jzzyiJkyZYqJiIhIVxcfH2+8vLzM8uXLbWPz5s0zxYsXt32fv/rqK7vvU4MGDcy8efOMMbeugfHx8TFt2rSx225sbKwpWrSokWS2b99uG9+yZYupUqWK3c9Sw4YNM70e6s/+fOTiz8LCwoybm5s5fPiwbezMmTPG09PTPPzww7axtJ/rBx980Ny8efO2z/dXVkeWihYtav75z3+mG//uu++MJLN69WpjzP9dS3bixIl0tY0bNzbNmjW7K7aZFZkdWfLy8kp3rVqbNm1M3bp17Y5cpaammhYtWphq1arZxiZNmmSKFi2a7vqpUaNGmUKFCtn28aWXXjJeXl45+j7i7sC74XDPmDNnjtatW5fucf/99992XR8fH0VFRengwYPZft5Vq1apUKFCevHFF+3GX375ZRlj9P3330uS7V1dL7zwgl3dkCFDMt32c889l27sz0dFbty4odjYWDVr1kySMnyXUP/+/W1/L1SokBo1aiRjjPr162cb9/Hx0X333acjR45k2ot0a18ladiwYXbjL7/8siTpu+++s1w/MxMnTtSiRYtUv359rVmzRmPGjFHDhg3VoEED7d2711a3du1alShRQo899pikW/s7cOBAde7cWcuWLVO3bt307LPP2m378ccf18aNGyVJzs7OGjhwoNavX6/Ro0fr4MGDioiIUNeuXZWUlCRJun79um3d4sWLKygoSKNGjdLy5cv19ttv69ixY3rqqad048aNbO9nSkqK1q5dq7CwMFWuXNk2XqZMGT399NP65ZdflJCQYLfOs88+q0KFCmX7uaxcv35d7u7u6cY9PDxsy//8Z2a1f56rgtzmnejcubN8fX1tX1+8eFEbNmxQ165ddfnyZcXGxio2NlYXLlxQaGioDh48qNOnT0uSlixZooceekjFixe31cXGxiokJEQpKSn6+eefJd3693X16lWtW7fujvtFweA0HO4ZTZo0yfCt2Wm/yKy8/vrrevzxx1W9enXVqVNH7du3V69evbIUtI4fP66AgAB5enrajdesWdO2PO1PZ2dnVapUya6uatWqmW77r7XSrV/mEydO1OLFi3Xu3Dm7ZfHx8enqy5cvb/e1t7e3PDw8VKpUqXTjFy5cyLSXP+/DX3v29/eXj4+PbV9zokePHurRo4cSEhK0ZcsWLViwQIsWLVKnTp20e/dueXh4KCIiQsHBwbZTjR9//LFatmypjz76SJIUFhamlJQUTZw40bZdPz8//fLLL7avX3/9dcXGxmratGl68803JUnt2rVTv379NHfuXNvpsvj4eD300EMaMWKELQxKt04ftWzZUvPnz9fzzz+frX08f/68rl27pvvuuy/dspo1ayo1NVUnT55U7dq1beMZ/QzcqcKFC9tOSf5ZWgBMC+Rpf2ZW++fgXpDbvBN/nd9Dhw7JGKOxY8dq7NixGa5z7tw5lS1bVgcPHtQff/xhF7b+Wifd+g/Sl19+qUceeURly5ZVu3bt1LVrV7Vv3/6O+0f+ICwBkh5++GEdPnxY33zzjdauXauPP/5YM2fO1Ny5c+2OzOS3jF4Munbtqs2bN2vEiBEKCgpSsWLFlJqaqvbt2ys1NTVdfUZHJTI7UmH+/3VRt/PX66Jyk5eXl9q2bau2bdvK1dVVCxcu1JYtWxQcHKwLFy4oICDAVnvs2DE1btzYbv0mTZrYfX3y5EmVLFnS9rWbm5s+/vhjTZkyRQcOHJCfn5+qV6+up59+2i4IfvXVV4qJibEdxUoTHBwsLy8v/frrr9kOSzmRG4Hgr8qUKaOzZ8+mG08bS5vjtGvCzp49q8DAwHS1f57rgtzmnfjr/Kb9Gxo+fLhCQ0MzXCftZyQ1NVVt27bVyJEjM6yrXr26pFvX5e3cuVNr1qzR999/r++//17z58/XM888o4ULF97xPiDvEZaA/69EiRLq27ev+vbtqytXrujhhx/WhAkTbGEps4BQoUIF/fDDD7p8+bLd0aV9+/bZlqf9mZqaqqNHj6patWq2ukOHDmW5x0uXLmn9+vWaOHGixo0bZxvPyenDnEjbh4MHD9qOnElSTEyM4uLibPuaWxo1aqSFCxfaXhy9vLzsjp75+/vr8OHDduv8+VTijRs39Mknn9jNVRo/Pz/5+flJunV6bOPGjWratKntyFJMTIxt2Z8ZY5SSkpLhDRRvx9fXV0WKFNH+/fvTLdu3b5+cnZ3TBYi8EBQUpE2bNik1NdXu4uktW7aoSJEithf5tLuub9++3S7EnDlzRqdOndKAAQPuim3mprTTo66urgoJCbGsrVKliq5cuXLbOulWSO/UqZM6deqk1NRUvfDCC/rwww81duxYy6PLuDtwzRIgpTv9VKxYMVWtWtXuFEDRokUl3bqj8J916NBBKSkpmj17tt34zJkz5eTkpEceeUSSbP9Lff/99+3q3nvvvSz3mXZE6K9HgNLedZPXOnTokOHzzZgxQ5Is39mXmWvXrik8PDzDZWnXe6WdtqpZs6a2bNliW/7EE09o2bJlmjNnjo4fP65Vq1bZ7hS+adMmtWvXTsWLF9c//vEPyx7efvttnT171u50W9oL8eLFi+1qV6xYoatXr6p+/frZ3NNb37927drpm2++sbtjeExMjBYtWqQHH3xQXl5e2d5udnXp0kUxMTH6+uuvbWOxsbFasmSJOnXqZLtOqHbt2qpRo4bmzZtnFxo/+OADOTk5qUuXLnfFNnNT6dKl1bJlS3344YcZHtU6f/687e9du3ZVeHi41qxZk64uLi7OFqj/+vvF2dnZdoo/o9OMuPtwZAmQVKtWLbVs2VINGzZUiRIltH37di1dulSDBw+21TRs2FCS9OKLLyo0NFSFChVS9+7d1alTJ7Vq1UpjxozRsWPHVK9ePa1du1bffPON/vWvf6lKlSq29Tt37qxZs2bpwoULtlsHHDhwQFLWTm15eXnp4Ycf1rRp05ScnKyyZctq7dq1Onr0aB7MSnr16tVT7969NW/ePMXFxSk4OFhbt27VwoULFRYWplatWmV7m9euXVOLFi3UrFkztW/fXoGBgYqLi9Py5cu1adMmhYWF2YJJ+/bt9dxzz2nHjh2qX7++OnXqpIEDB2rw4MEaPHiwihQpookTJ2rEiBFq2bKlunTpoq+//truRfXTTz/VV199pYcffljFihXTDz/8oC+//FL9+/dX586dbXWdOnVS7dq19frrr+v48eNq1qyZDh06pNmzZ6tMmTJ2F8hnx+TJk20ft/LCCy/IxcVFH374oRITEzVt2rQcbTPNt99+q99//13SrVtm/PHHH5o8ebIk6bHHHrO9QHfp0kXNmjVT3759tWfPHpUqVUrvv/9+uuu9pFuf7/fYY4+pXbt26t69u3bv3q3Zs2erf//+dkcXC3qbuWnOnDl68MEHVbduXT377LOqXLmyYmJiFB4erlOnTtnmeMSIEVqxYoUeffRR9enTRw0bNtTVq1e1a9cuLV26VMeOHVOpUqXUv39/Xbx4Ua1bt1a5cuV0/PhxvffeewoKCrLbX9zFCvKteEBuyI2bUk6ePNk0adLE+Pj4mMKFC5saNWqYKVOm2H0kys2bN82QIUOMr6+vcXJysnt7+uXLl83QoUNNQECAcXV1NdWqVcvwppRXr141gwYNMiVKlDDFihUzYWFhZv/+/UaS3Vv50972f/78+XT7c+rUKfPEE08YHx8f4+3tbZ566ilz5syZTG8/8NdtZPaW/qze6Tw5OdlMnDjRVKpUybi6uprAwMB0N6W0ep6MtvfRRx+ZsLAwU6FCBePu7m6KFCli6tevb6ZPn57ujsq9e/c2TZs2tRs/fPiw2bRpk7l06ZK5fv26CQ8PN3FxcRk+35YtW8zDDz9sihcvbjw8PEy9evXM3Llz032vjDHm4sWLZujQoaZ69erG3d3dlCpVynTv3t0cOXLktvuV2a0DjLl1U8rQ0FBTrFgxU6RIEdOqVSuzefNmu5rb/VxnJLObpyqDm4JevHjR9OvXz5QsWdIUKVLEBAcHZ/pcy5YtM0FBQcbd3d2UK1fOvPbaaxl+XFBBb9OK1U0pM3L48GHzzDPPGH9/f+Pq6mrKli1rHn300XR3hb98+bIZPXq0qVq1qnFzczOlSpUyLVq0MG+//bZtf5YuXWratWtnSpcubdzc3Ez58uXNwIEDs3QLCtwdnIzJ4hWdAPLEzp07Vb9+fX366afq2bNnQbdz14uNjVXDhg1Vp04dff755xmetkpJSdGyZcvsTukAQE4RloB8dP369XTvvunTp48++eQTHTt2LF8u7r0XHDhwQB07dlRCQoIGDx6stm3bKiAgQAkJCfrll180e/ZsRUdHKzIyMt2tEwAguwhLQD6aOHGiIiIi1KpVK7m4uNjeRjxgwAB9+OGHBd2eQ7l8+bKmT5+ujz/+2O5CXE9PT/Xs2VPjxo2zvU0dAO4EYQnIR+vWrdPEiRO1Z88eXblyReXLl1evXr00ZswYubjwfoucMMbo0KFDio6OlpeXl2rWrCk3N7eCbgvAPYSwBAAAYIH7LAEAAFggLAEAAFjgIolckJqaqjNnzsjT0zNPPzMLAADkHmOMLl++rICAALuP1PkrwlIuOHPmDG/5BgDAQZ08eVLlypXLdDlhKRekfXjqyZMn8+VznQAAwJ1LSEhQYGCg3YegZ4SwlAvSTr15eXkRlgAAcDC3u4SGC7wBAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsOFxYmjNnjipWrCgPDw81bdpUW7dutaxfsmSJatSoIQ8PD9WtW1erVq3KtPa5556Tk5OTZs2alctdAwAAR+VQYemLL77QsGHDNH78eEVGRqpevXoKDQ3VuXPnMqzfvHmzevTooX79+mnHjh0KCwtTWFiYdu/ena522bJl+u233xQQEJDXuwEAAByIQ4WlGTNm6Nlnn1Xfvn1Vq1YtzZ07V0WKFNF///vfDOvfffddtW/fXiNGjFDNmjU1adIkNWjQQLNnz7arO336tIYMGaLPPvtMrq6u+bErAADAQThMWEpKSlJERIRCQkJsY87OzgoJCVF4eHiG64SHh9vVS1JoaKhdfWpqqnr16qURI0aodu3aedM8AABwWC4F3UBWxcbGKiUlRX5+fnbjfn5+2rdvX4brREdHZ1gfHR1t+/qtt96Si4uLXnzxxSz3kpiYqMTERNvXCQkJWV4XAAA4Foc5spQXIiIi9O6772rBggVycnLK8npTp06Vt7e37REYGJiHXQIAgILkMGGpVKlSKlSokGJiYuzGY2Ji5O/vn+E6/v7+lvWbNm3SuXPnVL58ebm4uMjFxUXHjx/Xyy+/rIoVK2bay+jRoxUfH297nDx58s52DgAA3LUcJiy5ubmpYcOGWr9+vW0sNTVV69evV/PmzTNcp3nz5nb1krRu3Tpbfa9evfTHH39o586dtkdAQIBGjBihNWvWZNqLu7u7vLy87B4AAODe5DDXLEnSsGHD1Lt3bzVq1EhNmjTRrFmzdPXqVfXt21eS9Mwzz6hs2bKaOnWqJOmll15ScHCw3nnnHXXs2FGLFy/W9u3bNW/ePElSyZIlVbJkSbvncHV1lb+/v+6777783TkAAHBXcqiw1K1bN50/f17jxo1TdHS0goKCtHr1attF3CdOnJCz8/8dLGvRooUWLVqk1157Ta+++qqqVaum5cuXq06dOgW1CwAAwME4GWNMQTfh6BISEuTt7a34+HhOyQEA4CCy+vrtMNcsAQAAFATCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAXCEgAAgAWHC0tz5sxRxYoV5eHhoaZNm2rr1q2W9UuWLFGNGjXk4eGhunXratWqVbZlycnJeuWVV1S3bl0VLVpUAQEBeuaZZ3TmzJm83g0AAOAgHCosffHFFxo2bJjGjx+vyMhI1atXT6GhoTp37lyG9Zs3b1aPHj3Ur18/7dixQ2FhYQoLC9Pu3bslSdeuXVNkZKTGjh2ryMhIff3119q/f78ee+yx/NwtAABwF3MyxpiCbiKrmjZtqsaNG2v27NmSpNTUVAUGBmrIkCEaNWpUuvpu3brp6tWrWrlypW2sWbNmCgoK0ty5czN8jm3btqlJkyY6fvy4ypcvn6W+EhIS5O3trfj4eHl5eeVgzwAAQH7L6uu3wxxZSkpKUkREhEJCQmxjzs7OCgkJUXh4eIbrhIeH29VLUmhoaKb1khQfHy8nJyf5+PjkSt8AAMCxuRR0A1kVGxurlJQU+fn52Y37+flp3759Ga4THR2dYX10dHSG9Tdu3NArr7yiHj16WCbMxMREJSYm2r5OSEjI6m4AAAAH4zBHlvJacnKyunbtKmOMPvjgA8vaqVOnytvb2/YIDAzMpy4BAEB+c5iwVKpUKRUqVEgxMTF24zExMfL3989wHX9//yzVpwWl48ePa926dbe97mj06NGKj4+3PU6ePJmDPQIAAI7AYcKSm5ubGjZsqPXr19vGUlNTtX79ejVv3jzDdZo3b25XL0nr1q2zq08LSgcPHtQPP/ygkiVL3rYXd3d3eXl52T0AAMC9yWGuWZKkYcOGqXfv3mrUqJGaNGmiWbNm6erVq+rbt68k6ZlnnlHZsmU1depUSdJLL72k4OBgvfPOO+rYsaMWL16s7du3a968eZJuBaUuXbooMjJSK1euVEpKiu16phIlSsjNza1gdhQAANw1HCosdevWTefPn9e4ceMUHR2toKAgrV692nYR94kTJ+Ts/H8Hy1q0aKFFixbptdde06uvvqpq1app+fLlqlOnjiTp9OnTWrFihSQpKCjI7rl+/PFHtWzZMl/2CwAA3L0c6j5LdyvuswQAgOO55+6zBAAAUBAISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABYISwAAABZyFJYiIyO1a9cu29fffPONwsLC9OqrryopKSnXmgMAAChoOQpLAwcO1IEDByRJR44cUffu3VWkSBEtWbJEI0eOzNUGAQAAClKOwtKBAwcUFBQkSVqyZIkefvhhLVq0SAsWLNBXX32Vm/0BAAAUqByFJWOMUlNTJUk//PCDOnToIEkKDAxUbGxs7nUHAABQwHIUlho1aqTJkyfrk08+0U8//aSOHTtKko4ePSo/P79cbRAAAKAg5SgszZw5U5GRkRo8eLDGjBmjqlWrSpKWLl2qFi1a5GqDAAAABcnJGGNya2M3btyQi4uLXFxccmuTDiEhIUHe3t6Kj4+Xl5dXQbcDAACyIKuv3zk6slS5cmVduHAh3fiNGzdUvXr1nGwSAADgrpSjsHTs2DGlpKSkG09MTNSpU6fuuCkAAIC7RbbOl61YscL29zVr1sjb29v2dUpKitavX69KlSrlXncAAAAFLFthKSwsTJLk5OSk3r172y1zdXVVxYoV9c477+RacwAAAAUtW2Ep7d5KlSpV0rZt21SqVKk8aQoAAOBukaO3rR09ejS3+wAAALgr5fg9/uvXr9f69et17tw52xGnNP/973/vuDEAAIC7QY7C0sSJE/X666+rUaNGKlOmjJycnHK7LwAAgLtCjsLS3LlztWDBAvXq1Su3+wEAALir5Og+S0lJSXysCQAA+FvIUVjq37+/Fi1alNu9AAAA3HVyFJZu3LihGTNmKDg4WEOGDNGwYcPsHnlpzpw5qlixojw8PNS0aVNt3brVsn7JkiWqUaOGPDw8VLduXa1atcpuuTFG48aNU5kyZVS4cGGFhITo4MGDebkLAADAgeQoLP3xxx8KCgqSs7Ozdu/erR07dtgeO3fuzOUW/88XX3yhYcOGafz48YqMjFS9evUUGhqqc+fOZVi/efNm9ejRQ/369dOOHTsUFhamsLAw7d6921Yzbdo0/fvf/9bcuXO1ZcsWFS1aVKGhobpx40ae7QcAAHAcTsYYU9BNZFXTpk3VuHFjzZ49W9Ktm2QGBgZqyJAhGjVqVLr6bt266erVq1q5cqVtrFmzZgoKCtLcuXNljFFAQIBefvllDR8+XJIUHx8vPz8/LViwQN27d89SX1n91GIAAHD3yOrrd46OLBWEpKQkRUREKCQkxDbm7OyskJAQhYeHZ7hOeHi4Xb0khYaG2uqPHj2q6Ohouxpvb281bdo0021Ktz4wOCEhwe4BAADuTTm6dUCrVq0s7620YcOGHDeUmdjYWKWkpMjPz89u3M/PT/v27ctwnejo6Azro6OjbcvTxjKrycjUqVM1ceLEbO8DAABwPDkKS0FBQXZfJycna+fOndq9e3e6D9i9F40ePdruQvaEhAQFBgYWYEcAACCv5CgszZw5M8PxCRMm6MqVK3fUUGZKlSqlQoUKKSYmxm48JiZG/v7+Ga7j7+9vWZ/2Z0xMjMqUKWNX89dA+Gfu7u5yd3fPyW4AAAAHk6vXLP3jH//Is8+Fc3NzU8OGDbV+/XrbWGpqqtavX6/mzZtnuE7z5s3t6iVp3bp1tvpKlSrJ39/friYhIUFbtmzJdJsAAODvJccfpJuR8PBweXh45OYm7QwbNky9e/dWo0aN1KRJE82aNUtXr15V3759JUnPPPOMypYtq6lTp0qSXnrpJQUHB+udd95Rx44dtXjxYm3fvl3z5s2TJDk5Oelf//qXJk+erGrVqqlSpUoaO3asAgICFBYWlmf7AQAAHEeOwtKTTz5p97UxRmfPntX27ds1duzYXGksI926ddP58+c1btw4RUdHKygoSKtXr7ZdoH3ixAk5O//fwbIWLVpo0aJFeu211/Tqq6+qWrVqWr58uerUqWOrGTlypK5evaoBAwYoLi5ODz74oFavXp2noQ8AADiOHN1nKe1IThpnZ2f5+vqqdevWateuXa415yi4zxIAAI4nq6/fOTqyNH/+/Bw3BgAA4Eju6JqliIgI7d27V5JUu3Zt1a9fP1eaAgAAuFvkKCydO3dO3bt318aNG+Xj4yNJiouLU6tWrbR48WL5+vrmZo8AAAAFJke3DhgyZIguX76sqKgoXbx4URcvXtTu3buVkJCgF198Mbd7BAAAKDA5usDb29tbP/zwgxo3bmw3vnXrVrVr105xcXG51Z9D4AJvAAAcT55+kG5qaqpcXV3Tjbu6uio1NTUnmwQAALgr5SgstW7dWi+99JLOnDljGzt9+rSGDh2qNm3a5FpzAAAABS1HYWn27NlKSEhQxYoVVaVKFVWpUkWVKlVSQkKC3nvvvdzuEQAAoMDk6N1wgYGBioyM1A8//KB9+/ZJkmrWrKmQkJBcbQ4AAKCgZevI0oYNG1SrVi0lJCTIyclJbdu21ZAhQzRkyBA1btxYtWvX1qZNm/KqVwAAgHyXrbA0a9YsPfvssxleMe7t7a2BAwdqxowZudYcAABAQctWWPr999/Vvn37TJe3a9dOERERd9wUAADA3SJbYSkmJibDWwakcXFx0fnz5++4KQAAgLtFtsJS2bJltXv37kyX//HHHypTpswdNwUAAHC3yFZY6tChg8aOHasbN26kW3b9+nWNHz9ejz76aK41BwAAUNCy9XEnMTExatCggQoVKqTBgwfrvvvukyTt27dPc+bMUUpKiiIjI+Xn55dnDd+N+LgTAAAcT1Zfv7N1nyU/Pz9t3rxZzz//vEaPHq20nOXk5KTQ0FDNmTPnbxeUAADAvS3bN6WsUKGCVq1apUuXLunQoUMyxqhatWoqXrx4XvQHAABQoHJ0B29JKl68uBo3bpybvQAAANx1cvTZcAAAAH8XhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALhCUAAAALDhOWLl68qJ49e8rLy0s+Pj7q16+frly5YrnOjRs3NGjQIJUsWVLFihVT586dFRMTY1v++++/q0ePHgoMDFThwoVVs2ZNvfvuu3m9KwAAwIE4TFjq2bOnoqKitG7dOq1cuVI///yzBgwYYLnO0KFD9e2332rJkiX66aefdObMGT355JO25RERESpdurQ+/fRTRUVFacyYMRo9erRmz56d17sDAAAchJMxxhR0E7ezd+9e1apVS9u2bVOjRo0kSatXr1aHDh106tQpBQQEpFsnPj5evr6+WrRokbp06SJJ2rdvn2rWrKnw8HA1a9Ysw+caNGiQ9u7dqw0bNmS5v4SEBHl7eys+Pl5eXl452EMAAJDfsvr67RBHlsLDw+Xj42MLSpIUEhIiZ2dnbdmyJcN1IiIilJycrJCQENtYjRo1VL58eYWHh2f6XPHx8SpRooRlP4mJiUpISLB7AACAe5NDhKXo6GiVLl3abszFxUUlSpRQdHR0puu4ubnJx8fHbtzPzy/TdTZv3qwvvvjitqf3pk6dKm9vb9sjMDAw6zsDAAAcSoGGpVGjRsnJycnysW/fvnzpZffu3Xr88cc1fvx4tWvXzrJ29OjRio+Ptz1OnjyZLz0CAID851KQT/7yyy+rT58+ljWVK1eWv7+/zp07Zzd+8+ZNXbx4Uf7+/hmu5+/vr6SkJMXFxdkdXYqJiUm3zp49e9SmTRsNGDBAr7322m37dnd3l7u7+23rAACA4yvQsOTr6ytfX9/b1jVv3lxxcXGKiIhQw4YNJUkbNmxQamqqmjZtmuE6DRs2lKurq9avX6/OnTtLkvbv368TJ06oefPmtrqoqCi1bt1avXv31pQpU3JhrwAAwL3EId4NJ0mPPPKIYmJiNHfuXCUnJ6tv375q1KiRFi1aJEk6ffq02rRpo//9739q0qSJJOn555/XqlWrtGDBAnl5eWnIkCGSbl2bJN069da6dWuFhoZq+vTptucqVKhQlkJcGt4NBwCA48nq63eBHlnKjs8++0yDBw9WmzZt5OzsrM6dO+vf//63bXlycrL279+va9eu2cZmzpxpq01MTFRoaKjef/992/KlS5fq/Pnz+vTTT/Xpp5/axitUqKBjx47ly34BAIC7m8McWbqbcWQJAADHc0/dZwkAAKCgEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsEJYAAAAsOExYunjxonr27CkvLy/5+PioX79+unLliuU6N27c0KBBg1SyZEkVK1ZMnTt3VkxMTIa1Fy5cULly5eTk5KS4uLg82AMAAOCIHCYs9ezZU1FRUVq3bp1Wrlypn3/+WQMGDLBcZ+jQofr222+1ZMkS/fTTTzpz5oyefPLJDGv79eun+++/Py9aBwAADszJGGMKuonb2bt3r2rVqqVt27apUaNGkqTVq1erQ4cOOnXqlAICAtKtEx8fL19fXy1atEhdunSRJO3bt081a9ZUeHi4mjVrZqv94IMP9MUXX2jcuHFq06aNLl26JB8fnyz3l5CQIG9vb8XHx8vLy+vOdhYAAOSLrL5+O8SRpfDwcPn4+NiCkiSFhITI2dlZW7ZsyXCdiIgIJScnKyQkxDZWo0YNlS9fXuHh4baxPXv26PXXX9f//vc/OTtnbToSExOVkJBg9wAAAPcmhwhL0dHRKl26tN2Yi4uLSpQooejo6EzXcXNzS3eEyM/Pz7ZOYmKievTooenTp6t8+fJZ7mfq1Kny9va2PQIDA7O3QwAAwGEUaFgaNWqUnJycLB/79u3Ls+cfPXq0atasqX/84x/ZXi8+Pt72OHnyZB51CAAACppLQT75yy+/rD59+ljWVK5cWf7+/jp37pzd+M2bN3Xx4kX5+/tnuJ6/v7+SkpIUFxdnd3QpJibGts6GDRu0a9cuLV26VJKUdvlWqVKlNGbMGE2cODHDbbu7u8vd3T0ruwgAABxcgYYlX19f+fr63rauefPmiouLU0REhBo2bCjpVtBJTU1V06ZNM1ynYcOGcnV11fr169W5c2dJ0v79+3XixAk1b95ckvTVV1/p+vXrtnW2bdumf/7zn9q0aZOqVKlyp7sHAADuAQUalrKqZs2aat++vZ599lnNnTtXycnJGjx4sLp37257J9zp06fVpk0b/e9//1OTJk3k7e2tfv36adiwYSpRooS8vLw0ZMgQNW/e3PZOuL8GotjYWNvzZefdcAAA4N7lEGFJkj777DMNHjxYbdq0kbOzszp37qx///vftuXJycnav3+/rl27ZhubOXOmrTYxMVGhoaF6//33C6J9AADgoBziPkt3O+6zBACA47mn7rMEAABQUAhLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFghLAAAAFlwKuoF7gTFGkpSQkFDAnQAAgKxKe91Oex3PDGEpF1y+fFmSFBgYWMCdAACA7Lp8+bK8vb0zXe5kbhencFupqak6c+aMPD095eTklGvbTUhIUGBgoE6ePCkvL69c2y7sMc/5g3nOP8x1/mCe80dezrMxRpcvX1ZAQICcnTO/MokjS7nA2dlZ5cqVy7Pte3l58Q8xHzDP+YN5zj/Mdf5gnvNHXs2z1RGlNFzgDQAAYIGwBAAAYIGwdBdzd3fX+PHj5e7uXtCt3NOY5/zBPOcf5jp/MM/5426YZy7wBgAAsMCRJQAAAAuEJQAAAAuEJQAAAAuEJQAAAAuEpQI2Z84cVaxYUR4eHmratKm2bt1qWb9kyRLVqFFDHh4eqlu3rlatWpVPnTq27MzzRx99pIceekjFixdX8eLFFRISctvvC27J7s9zmsWLF8vJyUlhYWF52+A9IrvzHBcXp0GDBqlMmTJyd3dX9erV+d2RRdmd61mzZum+++5T4cKFFRgYqKFDh+rGjRv51K1j+vnnn9WpUycFBATIyclJy5cvv+06GzduVIMGDeTu7q6qVatqwYIFedukQYFZvHixcXNzM//9739NVFSUefbZZ42Pj4+JiYnJsP7XX381hQoVMtOmTTN79uwxr732mnF1dTW7du3K584dS3bn+emnnzZz5swxO3bsMHv37jV9+vQx3t7e5tSpU/ncuWPJ7jynOXr0qClbtqx56KGHzOOPP54/zTqw7M5zYmKiadSokenQoYP55ZdfzNGjR83GjRvNzp0787lzx5Pduf7ss8+Mu7u7+eyzz8zRo0fNmjVrTJkyZczQoUPzuXPHsmrVKjNmzBjz9ddfG0lm2bJllvVHjhwxRYoUMcOGDTN79uwx7733nilUqJBZvXp1nvVIWCpATZo0MYMGDbJ9nZKSYgICAszUqVMzrO/atavp2LGj3VjTpk3NwIED87RPR5fdef6rmzdvGk9PT7Nw4cK8avGekJN5vnnzpmnRooX5+OOPTe/evQlLWZDdef7ggw9M5cqVTVJSUn61eM/I7lwPGjTItG7d2m5s2LBh5oEHHsjTPu8lWQlLI0eONLVr17Yb69atmwkNDc2zvjgNV0CSkpIUERGhkJAQ25izs7NCQkIUHh6e4Trh4eF29ZIUGhqaaT1yNs9/de3aNSUnJ6tEiRJ51abDy+k8v/766ypdurT69euXH206vJzM84oVK9S8eXMNGjRIfn5+qlOnjt544w2lpKTkV9sOKSdz3aJFC0VERNhO1R05ckSrVq1Shw4d8qXnv4uCeC3kg3QLSGxsrFJSUuTn52c37ufnp3379mW4TnR0dIb10dHRedano8vJPP/VK6+8ooCAgHT/OPF/cjLPv/zyi/7zn/9o586d+dDhvSEn83zkyBFt2LBBPXv21KpVq3To0CG98MILSk5O1vjx4/OjbYeUk7l++umnFRsbqwcffFDGGN28eVPPPfecXn311fxo+W8js9fChIQEXb9+XYULF8715+TIEmDhzTff1OLFi7Vs2TJ5eHgUdDv3jMuXL6tXr1766KOPVKpUqYJu556Wmpqq0qVLa968eWrYsKG6deumMWPGaO7cuQXd2j1n48aNeuONN/T+++8rMjJSX3/9tb777jtNmjSpoFvDHeLIUgEpVaqUChUqpJiYGLvxmJgY+fv7Z7iOv79/tuqRs3lO8/bbb+vNN9/UDz/8oPvvvz8v23R42Z3nw4cP69ixY+rUqZNtLDU1VZLk4uKi/fv3q0qVKnnbtAPKyc9zmTJl5OrqqkKFCtnGatasqejoaCUlJcnNzS1Pe3ZUOZnrsWPHqlevXurfv78kqW7durp69aoGDBigMWPGyNmZ4xO5IbPXQi8vrzw5qiRxZKnAuLm5qWHDhlq/fr1tLDU1VevXr1fz5s0zXKd58+Z29ZK0bt26TOuRs3mWpGnTpmnSpElavXq1GjVqlB+tOrTsznONGjW0a9cu7dy50/Z47LHH1KpVK+3cuVOBgYH52b7DyMnP8wMPPKBDhw7ZwqgkHThwQGXKlCEoWcjJXF+7di1dIEoLqYaPYc01BfJamGeXjuO2Fi9ebNzd3c2CBQvMnj17zIABA4yPj4+Jjo42xhjTq1cvM2rUKFv9r7/+alxcXMzbb79t9u7da8aPH8+tA7Igu/P85ptvGjc3N7N06VJz9uxZ2+Py5csFtQsOIbvz/Fe8Gy5rsjvPJ06cMJ6enmbw4MFm//79ZuXKlaZ06dJm8uTJBbULDiO7cz1+/Hjj6elpPv/8c3PkyBGzdu1aU6VKFdO1a9eC2gWHcPnyZbNjxw6zY8cOI8nMmDHD7Nixwxw/ftwYY8yoUaNMr169bPVptw4YMWKE2bt3r5kzZw63DrjXvffee6Z8+fLGzc3NNGnSxPz222+2ZcHBwaZ379529V9++aWpXr26cXNzM7Vr1zbfffddPnfsmLIzzxUqVDCS0j3Gjx+f/407mOz+PP8ZYSnrsjvPmzdvNk2bNjXu7u6mcuXKZsqUKebmzZv53LVjys5cJycnmwkTJpgqVaoYDw8PExgYaF544QVz6dKl/G/cgfz4448Z/s5Nm9vevXub4ODgdOsEBQUZNzc3U7lyZTN//vw87dHJGI4NAgAAZIZrlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgDck86fP6/nn39e5cuXl7u7u/z9/RUaGqpff/3Vru7ChQuaMGGCGjduLF9fX5UvX14dO3bU4sWLM/w8r8jISLVt21Y+Pj4qWbKkBgwYoCtXrtjVbNu2TW3atJGPj4+KFy+u0NBQ/f7773m6vwDyDmEJwD2pc+fO2rFjhxYuXKgDBw5oxYoVatmypS5cuGCrWbt2rapXr65t27Zp+PDhWrt2rb7++ms9+uijmjRpkkJDQ3X16lVb/ZkzZxQSEqKqVatqy5YtWr16taKiotSnTx9bzZUrV9S+fXuVL19eW7Zs0S+//CJPT0+FhoYqOTk5P6cAQC7h404A3HPi4uJUvHhxbdy4UcHBwRnWbN++XaGhoVqwYIE6deqUbvnNmzc1YMAAxcbGasWKFZKkefPmaezYsTp79qzt0+V37dql+++/XwcPHlTVqlW1fft2NW7cWCdOnFBgYGCGNQAcC0eWANxzihUrpmLFimn58uVKTEzMsGbIkCGaMmWKOnXqpD179ig4OFi+vr7q2rWrhg0bpmnTpmnu3Lnas2ePfvzxR0lSYmKi3NzcbEFJkgoXLixJ+uWXXyRJ9913n0qWLKn//Oc/SkpK0vXr1/Wf//xHNWvWVMWKFfN2xwHkCcISgHuOi4uLFixYoIULF8rHx0cPPPCAXn31Vf3xxx+SpIMHD+rYsWPq37+/UlJS9MQTT6hGjRpat26dHnroIb333ntKSkqSm5ubevTooTVr1kiSWrdurejoaE2fPl1JSUm6dOmSRo0aJUk6e/asJMnT01MbN27Up59+qsKFC6tYsWJavXq1vv/+e7m4uBTMhAC4I4QlAPekzp0768yZM1qxYoXat2+vjRs3qkGDBlqwYIF27dqlxo0by8XFRfv379fp06c1e/ZsBQUFaciQIWrZsqVtO2XKlNGlS5ckSbVr19bChQv1zjvvqEiRIvL391elSpXk5+dnO9p0/fp19evXTw888IB+++03/frrr6pTp446duyo69evF8RUALhDhCUA9ywPDw+1bdtWY8eO1ebNm9WnTx+NHz9eN2/etJ0+S0pKkqurq1xdXW3rFStWzPb3yMhIu+uMnn76aUVHR+v06dO2d9KdP39elStXliQtWrRIx44d0/z589W4cWM1a9ZMixYt0tGjR/XNN9/k054DyE2EJQB/G7Vq1dLVq1dVtWpV7dq1S9Kta4xcXV01e/ZspaSk6LffftOaNWuUnJyshQsX6vvvv7d7t1saPz8/FStWTF988YUtlEnStWvX5OzsLCcnJ1tt2tepqan5sp8AchdhCcA958KFC2rdurU+/fRT/fHHHzp69KiWLFmiadOm6fHHH1f9+vV1/fp1/fjjjypcuLAWLFigcePGyd3dXX379lVYWJjeeustzZ8/X2vXrpWvr69t27Nnz1ZkZKQOHDigOXPmaPDgwZo6dap8fHwkSW3bttWlS5c0aNAg7d27V1FRUerbt69cXFzUqlWrApoRAHeCWwcAuOckJiZqwoQJWrt2rQ4fPqzk5GQFBgbqqaee0quvvqrChQvryy+/1PDhw/Xzzz+rYsWKSklJUXR0tAICAhQfHy8XFxe703FpnnnmGX333Xe6cuWKatSooeHDh6tXr152NevWrdPEiRO1e/duOTs7q379+poyZYqaNWuWX1MAIBcRlgD8bb3xxhuaMWOGRo8erW7duqlcuXJKTEzUTz/9pEmTJmnYsGF64oknCrpNAAWMsATgb23Tpk2aPHmyNm7cKGOMbt68qVq1aunFF19U//797e6pBODvibAEALp16u7cuXPy9PS0XX8EABJhCQAAwBLHlwEAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACwQlgAAACz8P9YeNNFg9nRDAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.hist(sas98s)\n",
    "ax.set(\n",
    "    xlabel=\"S@98\",\n",
    "    ylabel=\"Counts\",\n",
    "    title=\"Histogram of S@98 for 100000 Trees\"\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "97dcf30d-5ce6-4782-a3bb-330813d3d7fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6af1de765843481c8bd3720bc135b0c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Reps Histogram S@98:   0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[23], line 14\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m seed \u001b[38;5;129;01min\u001b[39;00m trange(\u001b[38;5;241m50\u001b[39m, desc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReps Histogram S@98\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m      4\u001b[0m     est \u001b[38;5;241m=\u001b[39m HonestForestClassifier(\n\u001b[1;32m      5\u001b[0m         n_estimators\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50000\u001b[39m,\n\u001b[1;32m      6\u001b[0m         random_state\u001b[38;5;241m=\u001b[39mseed,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[38;5;66;03m# permute_per_tree=True,\u001b[39;00m\n\u001b[1;32m     13\u001b[0m     )\n\u001b[0;32m---> 14\u001b[0m     _, posterior_arr \u001b[38;5;241m=\u001b[39m \u001b[43mbuild_hyppo_oob_forest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     15\u001b[0m \u001b[43m        \u001b[49m\u001b[43mest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     17\u001b[0m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     18\u001b[0m \u001b[43m        \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m     19\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     20\u001b[0m     sas98 \u001b[38;5;241m=\u001b[39m sensitivity_at_specificity(\n\u001b[1;32m     21\u001b[0m         y, posterior_arr, target_specificity\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.98\u001b[39m\n\u001b[1;32m     22\u001b[0m     )\n\u001b[1;32m     23\u001b[0m     sas98s\u001b[38;5;241m.\u001b[39mappend(sas98)\n",
      "File \u001b[0;32m~/workspace/scikit-tree/sktree/stats/forestht.py:1358\u001b[0m, in \u001b[0;36mbuild_hyppo_oob_forest\u001b[0;34m(est, X, y, verbose, **est_kwargs)\u001b[0m\n\u001b[1;32m   1355\u001b[0m est \u001b[38;5;241m=\u001b[39m clone(est)\n\u001b[1;32m   1357\u001b[0m \u001b[38;5;66;03m# build forest\u001b[39;00m\n\u001b[0;32m-> 1358\u001b[0m \u001b[43mest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mravel\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mest_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1360\u001b[0m \u001b[38;5;66;03m# now evaluate\u001b[39;00m\n\u001b[1;32m   1361\u001b[0m X \u001b[38;5;241m=\u001b[39m est\u001b[38;5;241m.\u001b[39m_validate_X_predict(X)\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/sklearn/base.py:1351\u001b[0m, in \u001b[0;36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[0;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1344\u001b[0m     estimator\u001b[38;5;241m.\u001b[39m_validate_params()\n\u001b[1;32m   1346\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[1;32m   1347\u001b[0m     skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[1;32m   1348\u001b[0m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[1;32m   1349\u001b[0m     )\n\u001b[1;32m   1350\u001b[0m ):\n\u001b[0;32m-> 1351\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfit_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mestimator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/workspace/scikit-tree/sktree/ensemble/_honest_forest.py:643\u001b[0m, in \u001b[0;36mHonestForestClassifier.fit\u001b[0;34m(self, X, y, sample_weight, classes)\u001b[0m\n\u001b[1;32m    638\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwarm_start \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mestimators_) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m    639\u001b[0m     \u001b[38;5;66;03m# We draw from the random state to get the random state we\u001b[39;00m\n\u001b[1;32m    640\u001b[0m     \u001b[38;5;66;03m# would have got if we hadn't used a warm_start.\u001b[39;00m\n\u001b[1;32m    641\u001b[0m     random_state\u001b[38;5;241m.\u001b[39mrandint(MAX_INT, size\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mestimators_))\n\u001b[0;32m--> 643\u001b[0m trees \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_construct_trees\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    644\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    645\u001b[0m \u001b[43m    \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    646\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    647\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    648\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_samples_bootstrap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    649\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmissing_values_in_feature_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    650\u001b[0m \u001b[43m    \u001b[49m\u001b[43mclasses\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    651\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_more_estimators\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    652\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    654\u001b[0m \u001b[38;5;66;03m# Collect newly grown trees\u001b[39;00m\n\u001b[1;32m    655\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mestimators_\u001b[38;5;241m.\u001b[39mextend(trees)\n",
      "File \u001b[0;32m~/workspace/scikit-tree/sktree/ensemble/_honest_forest.py:712\u001b[0m, in \u001b[0;36mHonestForestClassifier._construct_trees\u001b[0;34m(self, X, y, sample_weight, random_state, n_samples_bootstrap, missing_values_in_feature_mask, classes, n_more_estimators)\u001b[0m\n\u001b[1;32m    701\u001b[0m trees \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m    702\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_estimator(append\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, random_state\u001b[38;5;241m=\u001b[39mrandom_state)\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(n_more_estimators)\n\u001b[1;32m    704\u001b[0m ]\n\u001b[1;32m    706\u001b[0m \u001b[38;5;66;03m# Parallel loop: we prefer the threading backend as the Cython code\u001b[39;00m\n\u001b[1;32m    707\u001b[0m \u001b[38;5;66;03m# for fitting the trees is internally releasing the Python GIL\u001b[39;00m\n\u001b[1;32m    708\u001b[0m \u001b[38;5;66;03m# making threading more efficient than multiprocessing in\u001b[39;00m\n\u001b[1;32m    709\u001b[0m \u001b[38;5;66;03m# that case. However, for joblib 0.12+ we respect any\u001b[39;00m\n\u001b[1;32m    710\u001b[0m \u001b[38;5;66;03m# parallel_backend contexts set at a higher level,\u001b[39;00m\n\u001b[1;32m    711\u001b[0m \u001b[38;5;66;03m# since correctness does not rely on using threads.\u001b[39;00m\n\u001b[0;32m--> 712\u001b[0m trees \u001b[38;5;241m=\u001b[39m \u001b[43mParallel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    713\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_jobs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_jobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    714\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    715\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprefer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mthreads\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    716\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    717\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdelayed\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_parallel_build_trees\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    718\u001b[0m \u001b[43m        \u001b[49m\u001b[43mt\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    719\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbootstrap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    720\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    721\u001b[0m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    722\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    723\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    724\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtrees\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    725\u001b[0m \u001b[43m        \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    726\u001b[0m \u001b[43m        \u001b[49m\u001b[43mclass_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclass_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    727\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_samples_bootstrap\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_samples_bootstrap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    728\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmissing_values_in_feature_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmissing_values_in_feature_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    729\u001b[0m \u001b[43m        \u001b[49m\u001b[43mclasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclasses\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    730\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    731\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43menumerate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtrees\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    732\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    733\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trees\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1952\u001b[0m, in \u001b[0;36mParallel.__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1946\u001b[0m \u001b[38;5;66;03m# The first item from the output is blank, but it makes the interpreter\u001b[39;00m\n\u001b[1;32m   1947\u001b[0m \u001b[38;5;66;03m# progress until it enters the Try/Except block of the generator and\u001b[39;00m\n\u001b[1;32m   1948\u001b[0m \u001b[38;5;66;03m# reach the first `yield` statement. This starts the aynchronous\u001b[39;00m\n\u001b[1;32m   1949\u001b[0m \u001b[38;5;66;03m# dispatch of the tasks to the workers.\u001b[39;00m\n\u001b[1;32m   1950\u001b[0m \u001b[38;5;28mnext\u001b[39m(output)\n\u001b[0;32m-> 1952\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m output \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_generator \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(output)\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1595\u001b[0m, in \u001b[0;36mParallel._get_outputs\u001b[0;34m(self, iterator, pre_dispatch)\u001b[0m\n\u001b[1;32m   1592\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m\n\u001b[1;32m   1594\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backend\u001b[38;5;241m.\u001b[39mretrieval_context():\n\u001b[0;32m-> 1595\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_retrieve()\n\u001b[1;32m   1597\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mGeneratorExit\u001b[39;00m:\n\u001b[1;32m   1598\u001b[0m     \u001b[38;5;66;03m# The generator has been garbage collected before being fully\u001b[39;00m\n\u001b[1;32m   1599\u001b[0m     \u001b[38;5;66;03m# consumed. This aborts the remaining tasks if possible and warn\u001b[39;00m\n\u001b[1;32m   1600\u001b[0m     \u001b[38;5;66;03m# the user if necessary.\u001b[39;00m\n\u001b[1;32m   1601\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/cancer/lib/python3.11/site-packages/joblib/parallel.py:1707\u001b[0m, in \u001b[0;36mParallel._retrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1702\u001b[0m \u001b[38;5;66;03m# If the next job is not ready for retrieval yet, we just wait for\u001b[39;00m\n\u001b[1;32m   1703\u001b[0m \u001b[38;5;66;03m# async callbacks to progress.\u001b[39;00m\n\u001b[1;32m   1704\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ((\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jobs) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m\n\u001b[1;32m   1705\u001b[0m     (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jobs[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mget_status(\n\u001b[1;32m   1706\u001b[0m         timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimeout) \u001b[38;5;241m==\u001b[39m TASK_PENDING)):\n\u001b[0;32m-> 1707\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m0.01\u001b[39m)\n\u001b[1;32m   1708\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m   1710\u001b[0m \u001b[38;5;66;03m# We need to be careful: the job list can be filling up as\u001b[39;00m\n\u001b[1;32m   1711\u001b[0m \u001b[38;5;66;03m# we empty it and Python list are not thread-safe by\u001b[39;00m\n\u001b[1;32m   1712\u001b[0m \u001b[38;5;66;03m# default hence the use of the lock\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "X, y = generate_data()\n",
    "sas98s = []\n",
    "for seed in trange(50, desc=\"Reps Histogram S@98\"):\n",
    "    est = HonestForestClassifier(\n",
    "        n_estimators=50000,\n",
    "        random_state=seed,\n",
    "        honest_fraction=0.5,\n",
    "        n_jobs=-1,\n",
    "        bootstrap=True,\n",
    "        stratify=True,\n",
    "        max_samples=1.6,\n",
    "        # permute_per_tree=True,\n",
    "    )\n",
    "    _, posterior_arr = build_hyppo_oob_forest(\n",
    "        est,\n",
    "        X,\n",
    "        y,\n",
    "        verbose=False\n",
    "    )\n",
    "    sas98 = sensitivity_at_specificity(\n",
    "        y, posterior_arr, target_specificity=0.98\n",
    "    )\n",
    "    sas98s.append(sas98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96120bc1-e45d-40cf-a64d-633b2785e8be",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.hist(sas98s)\n",
    "ax.set(\n",
    "    xlabel=\"S@98\",\n",
    "    ylabel=\"Counts\",\n",
    "    title=\"Histogram of S@98 for 50000 Trees\"\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730962c4-d200-472c-a31a-59f4a9b10690",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}