examples/XGBOD_accuracy_comparison_GB1_2016.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7ef19594",
   "metadata": {},
   "source": [
    "## XGBOD accuracy comparison--GB1_2016\n",
    "GB1_2016 has combintorially measured all the possible mutations and been studied extensively in the previous literature. Here, we use GB1_2016 as an example to check the XGBOD outlier detection accuracy with different fitness threshold.\n",
    "To use this notebook, please change the global control varibles and read the comments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5b992ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "import odbo\n",
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b77cf7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experiment settings \n",
    "dataset_name ='GB1_2016'\n",
    "random_seed = 0 #Random seed for the trial\n",
    "search_iter = 50 #Number of new observations, GB1_2014=100, BRCA1=50, avGFP_2016=50\n",
    "# Initialization method protocol\n",
    "update_method='independent'#find round 0 experiments to initiate BO. For the datasets with few changes in the sequences, 'correlate' mode is recommended. \n",
    "allow_abundance=True #If we allow the top scoring experiments to take abundance of a mutation in different sites into account.\n",
    "# Featurization settings\n",
    "method=['Avg','Max','Avg','Max'] #switching order for feature spaces to overcome local maxima in one certain representation\n",
    "mode='independent' #Feature computing mode. \n",
    "threshold = 0.05 #thresholds to be tested\n",
    "cMat_plot = True #Plot the confusion matrix to check the accuracy of search space prescreening or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6f28cce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset\n",
    "np.random.seed(random_seed)\n",
    "data_test = pd.read_csv('../datasets/GB1_2016_149361.csv', sep=',')\n",
    "name_pre, Y_test = np.array(data_test['AACombo']), np.array(data_test['Fitness'])\n",
    "shuffle_order = np.arange(len(Y_test))\n",
    "np.random.shuffle(shuffle_order[1:])\n",
    "name_pre[1:], Y_test[1:] = name_pre[shuffle_order[1:]], Y_test[shuffle_order[1:]]\n",
    "name = odbo.utils.code_to_array(name_pre)\n",
    "#Load the preselected indices using a certain shuffling order. Control Round 0 experiments to be the same for different trials\n",
    "if os.path.isfile('sele_experiment_GB1_2016.npy') == True:\n",
    "    name_sele = np.load('sele_experiment_GB1_2016.npy')\n",
    "    Y_train = np.load('sele_fitness_GB1_2016.npy')\n",
    "else:\n",
    "    # Let each site has 20 AA codes at least show up twice \n",
    "    sele_indices = odbo.initialization.initial_design(name, least_occurance=2*np.ones(name.shape[1]),allow_abundance=allow_abundance, update_method=update_method,verbose = True)\n",
    "    # Initial experiments are selected to be name_sele with fitness of Y_sele\n",
    "    name_sele, Y_train = name[sele_indices, :], Y_test[sele_indices]\n",
    "print('Selected initial experiments no. is ', len(Y_train))\n",
    "print('Select max Y: ', Y_train.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82c2ca12",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Using MassiveFeatureTransform method to transform features. (Since GB1 2016 mutates all the sites)\n",
    "feature_model = odbo.featurization.MassiveFeatureTransform(raw_vars=name_sele, Y=Y_train, method = method[0], mode=mode)\n",
    "X_train = feature_model.transform(name_sele)\n",
    "X_test = feature_model.transform(name)\n",
    "# Get outliers or inliers using the threshold\n",
    "labels_train = odbo.prescreening.sp_label(X_train, Y_train, thres=threshold)\n",
    "# Find the XGBOD adaptive search space model\n",
    "pre_model = odbo.prescreening.XGBOD(eval_metric = 'error', random_state = random_seed)\n",
    "pre_model.fit(X_train, labels_train)\n",
    "# Predict the entire search space to get the adapt search space\n",
    "labels_test = odbo.prescreening.sp_label(X_test, Y_test, thres=threshold)\n",
    "pred_test_labels = pre_model.predict(X_test)\n",
    "sele_id_test = list(np.where(pred_test_labels == 0)[0])\n",
    "# Plot the confusion matrix to check the accuracy of search space prescreening\n",
    "if cMat_plot:\n",
    "    out_outlier, in_outlier, out_inlier, in_inlier = odbo.plot.plot_cm(labels_test, pred_test_labels, Y_test)\n",
    "    print(\"Correct ratio: {0:.3%}\".format((len(out_outlier)+len(in_inlier))/len(labels_test)))\n",
    "    print(\"FP ratio: {0:.3%}\".format(len(in_outlier)/(len(in_outlier)+len(out_outlier))))\n",
    "    print(\"FN ratio: {0:.3%}\".format(len(out_inlier)/(len(out_inlier)+len(in_inlier))))\n",
    "    print(\"Adapt space size, Entire space size: \", len(sele_id_test), name.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22f0a87f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}