Adult.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPh6HAMlwOUzJPF8Ztz9AKy",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aayushhyadav/MIA/blob/feature%2Fadult/Adult.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install seaborn"
      ],
      "metadata": {
        "id": "ypDCnk6DvLJ0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow-privacy"
      ],
      "metadata": {
        "id": "1Il1G-Jvmd3r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U matplotlib"
      ],
      "metadata": {
        "id": "FiWQxzy_mfiO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U tensorflow-datasets"
      ],
      "metadata": {
        "id": "lX4OiPi3mmIA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U pandas"
      ],
      "metadata": {
        "id": "0o5k3MmFm3U3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oTi6XLSjBniK"
      },
      "outputs": [],
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip adult.zip"
      ],
      "metadata": {
        "id": "V65KOWGKDDgy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import math\n",
        "import statistics\n",
        "import random"
      ],
      "metadata": {
        "id": "6u1oOZhIDaYa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_model(optimizer, loss):\n",
        "  model = tf.keras.Sequential()\n",
        "  model.add(tf.keras.layers.Input(shape=(104,)))\n",
        "  model.add(tf.keras.layers.Dense(128, activation='relu'))\n",
        "  model.add(tf.keras.layers.Dense(128, activation='relu'))\n",
        "  model.add(tf.keras.layers.Dense(2, activation='softmax'))\n",
        "\n",
        "  model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])\n",
        "\n",
        "  return model"
      ],
      "metadata": {
        "id": "qXc6cLRB2d25"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_unequal_class_distribution(data_x, data_y, neg_class):\n",
        "  class_dist = [0] * 2\n",
        "\n",
        "  examples = list()\n",
        "  train_X = list()\n",
        "  train_y = list()\n",
        "\n",
        "  for i in range(len(data_y)):\n",
        "    if(data_y[i][0] == 1 and class_dist[0] <= neg_class):\n",
        "      class_dist[0] += 1\n",
        "      examples.append(i)\n",
        "    elif(data_y[i][0] == 0):\n",
        "      class_dist[1] += 1\n",
        "      examples.append(i)\n",
        "\n",
        "  for i in range(len(examples)):\n",
        "    train_X.append(data_x[examples[i]])\n",
        "    train_y.append(data_y[examples[i]])\n",
        "\n",
        "  train_X = np.array(train_X)\n",
        "  train_y = np.array(train_y)\n",
        "\n",
        "  print(f\"{data_x.shape, train_X.shape}\")\n",
        "  print(f\"{data_y.shape, train_y.shape}\")\n",
        "\n",
        "  return train_X, train_y"
      ],
      "metadata": {
        "id": "tt5_4IyTciM4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # plots the loss and accuracy for training and validation sets\n",
        "'''\n",
        "def plot_learning_curves(history):\n",
        "  fig, (ax1, ax2) = plt.subplots(1, 2)\n",
        "  ax1.set_title('Cross Entropy Loss')\n",
        "  ax1.plot(history.history['loss'], color = 'blue', label = 'train')\n",
        "  ax1.plot(history.history['val_loss'], color = 'red', label = 'validation')\n",
        "\n",
        "  ax2.set_title('Accuracy')\n",
        "  ax2.plot(history.history['accuracy'], color = 'blue', label = 'train')\n",
        "  ax2.plot(history.history['val_accuracy'], color = 'red', label = 'validation')\n",
        "\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "ZSN1870_C7Kj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # computes entropy for probability vectors\n",
        "  # prob_vec - probability vector\n",
        "'''\n",
        "def cal_entropy(prob_vec):\n",
        "  entropy = 0\n",
        "\n",
        "  for prob in prob_vec:\n",
        "    if(prob != 0):\n",
        "      entropy += prob * math.log(prob, 2)\n",
        "\n",
        "  return -1 * entropy"
      ],
      "metadata": {
        "id": "XwRCo_jfFrKo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # computes standard deviation for probability vector\n",
        "'''\n",
        "def cal_std(prob_vec):\n",
        "  mean = statistics.mean(prob_vec)\n",
        "  variance = 0\n",
        "\n",
        "  for prob in prob_vec:\n",
        "    variance += (prob - mean) ** 2\n",
        "  variance /= len(prob_vec)\n",
        "\n",
        "  return math.sqrt(variance)"
      ],
      "metadata": {
        "id": "QLZinK1DGXqD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # calculates AUC score for attack model using trapezoidal method\n",
        "  # train_vec - entropy for probability vectors of training examples\n",
        "  # test_vec - entropy for probability vectors of testing examples\n",
        "  # pos_count - number of examples which are members of training set\n",
        "  # neg_count - number of examples which are non-members of training set\n",
        "  # inc_threshold - controls the number of thresholds for which AUC is computed\n",
        "'''\n",
        "def cal_auc(train_vec, test_vec, pos_count, neg_count, inc_threshold = 0.1):\n",
        "  threshold = 0\n",
        "  tpr_list, fpr_list = list(), list()\n",
        "  tuple_tpr, tuple_fpr = list(), list()\n",
        "  points = list()\n",
        "  auc = 0\n",
        "\n",
        "  while(threshold <= 1):\n",
        "    tp_count = 0\n",
        "    fp_count = 0\n",
        "\n",
        "    for val in train_vec:\n",
        "      if(val <= threshold):\n",
        "        tp_count += 1\n",
        "\n",
        "    for val in test_vec:\n",
        "      if(val <= threshold):\n",
        "        fp_count += 1\n",
        "\n",
        "    tpr = tp_count / pos_count\n",
        "    fpr = fp_count / neg_count\n",
        "\n",
        "    tpr_list.append(tpr)\n",
        "    fpr_list.append(fpr)\n",
        "\n",
        "    threshold += inc_threshold\n",
        "\n",
        "  for i in range(len(tpr_list)):\n",
        "    points.append([fpr_list[i], tpr_list[i]])\n",
        "\n",
        "  points.sort()\n",
        "\n",
        "  for i in range(len(points) - 1):\n",
        "    tuple_tpr.append([points[i][1], points[i + 1][1]])\n",
        "    tuple_fpr.append([points[i][0], points[i + 1][0]])\n",
        "\n",
        "  auc = sum(np.trapz(tuple_tpr, tuple_fpr))\n",
        "\n",
        "  plt.scatter(fpr_list, tpr_list)\n",
        "  plt.show()\n",
        "\n",
        "  return auc"
      ],
      "metadata": {
        "id": "KBoqhzUZHGKF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_threshold(vec):\n",
        "  return np.percentile(vec, 90)"
      ],
      "metadata": {
        "id": "d77hssIyiy8T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # computes the precision and recall for attack model\n",
        "'''\n",
        "def cal_pre_recall(train_vec, test_vec, threshold):\n",
        "  tp = 0\n",
        "  fp = 0\n",
        "  fn = 0\n",
        "\n",
        "  for val in train_vec:\n",
        "    if(val >= threshold):\n",
        "      tp += 1\n",
        "    else:\n",
        "      fn += 1\n",
        "\n",
        "  for val in test_vec:\n",
        "    if(val >= threshold):\n",
        "      fp += 1\n",
        "\n",
        "  precision = tp / (tp + fp)\n",
        "  recall = tp / (tp + fn)\n",
        "\n",
        "  return precision, recall"
      ],
      "metadata": {
        "id": "SDO6JDsli4mO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "  # makes the distribution of metrics (entropy, std, max probability) of test set similar to training set\n",
        "  # num_class is the number of classes in the classification task\n",
        "  # max_prob_train is the list of maximum probabilities of training examples\n",
        "  # max_prob_class is the class predicted by the model for a specific test example\n",
        "  # returns a list of probabilities for the specific test example\n",
        "'''\n",
        "def make_metric_dist_same(num_class, max_prob_train, max_prob_class):\n",
        "  p = [0] * num_class\n",
        "  p[max_prob_class]  = random.choice(max_prob_train)\n",
        "  rem_p = 1 - p[max_prob_class]\n",
        "  i = 0\n",
        "\n",
        "  while(rem_p > 0):\n",
        "    if(i != max_prob_class):\n",
        "      update_p_i = random.uniform(0, min(rem_p, p[max_prob_class]))\n",
        "\n",
        "      if(p[i] + update_p_i < p[max_prob_class]):\n",
        "        p[i] += update_p_i\n",
        "        rem_p -= p[i]\n",
        "\n",
        "    i = (i + 1) % num_class\n",
        "\n",
        "  return p"
      ],
      "metadata": {
        "id": "3OUa9QzvfYzQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# computes average entropy, standard deviation, and maximum posterior probability\n",
        "# for prediction vector of training set examples\n",
        "\n",
        "def compute_train_metrics(prob_vec_train):\n",
        "  entropy_train = list()\n",
        "  std_train = list()\n",
        "  max_prob_train = list()\n",
        "\n",
        "  sum_entropy = 0\n",
        "  sum_std = 0\n",
        "  sum_max_prob = 0\n",
        "\n",
        "  for prob_vec in prob_vec_train:\n",
        "    cur_entropy = cal_entropy(prob_vec)\n",
        "    cur_std = cal_std(prob_vec)\n",
        "\n",
        "    cur_max_prob = max(prob_vec)\n",
        "\n",
        "    entropy_train.append(cur_entropy)\n",
        "    std_train.append(cur_std)\n",
        "    max_prob_train.append(cur_max_prob)\n",
        "\n",
        "    sum_entropy += cur_entropy\n",
        "    sum_std += cur_std\n",
        "    sum_max_prob += cur_max_prob\n",
        "\n",
        "  avg_train_entropy = sum_entropy / len(prob_vec_train)\n",
        "  avg_train_std = sum_std / len(prob_vec_train)\n",
        "  avg_train_max_prob = sum_max_prob / len(prob_vec_train)\n",
        "\n",
        "  print(f\"Avg train entropy - {avg_train_entropy}, Avg train std - {avg_train_std}, Avg train max probability - {avg_train_max_prob}\")\n",
        "\n",
        "  return entropy_train, std_train, max_prob_train, avg_train_entropy, avg_train_std, avg_train_max_prob"
      ],
      "metadata": {
        "id": "jWxspbNkGlnn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# computes average entropy, standard deviation, and maximum posterior probability\n",
        "# for prediction vector of test set examples\n",
        "\n",
        "def compute_test_metrics(prob_vec_test):\n",
        "  entropy_test = list()\n",
        "  std_test = list()\n",
        "  max_prob_test = list()\n",
        "\n",
        "  sum_entropy = 0\n",
        "  sum_std = 0\n",
        "  sum_max_prob = 0\n",
        "\n",
        "  for prob_vec in prob_vec_test:\n",
        "    cur_entropy = cal_entropy(prob_vec)\n",
        "    cur_std = cal_std(prob_vec)\n",
        "    cur_max_prob = max(prob_vec)\n",
        "\n",
        "    entropy_test.append(cur_entropy)\n",
        "    std_test.append(cur_std)\n",
        "    max_prob_test.append(cur_max_prob)\n",
        "\n",
        "    sum_entropy = sum_entropy + cur_entropy\n",
        "    sum_std = sum_std + cur_std\n",
        "    sum_max_prob += cur_max_prob\n",
        "\n",
        "  avg_test_entropy = sum_entropy / len(prob_vec_test)\n",
        "  avg_test_std = sum_std / len(prob_vec_test)\n",
        "  avg_test_max_prob = sum_max_prob / len(prob_vec_test)\n",
        "\n",
        "  print(f\"Avg test entropy - {avg_test_entropy}, Avg test std - {avg_test_std}, Avg test max probability - {avg_test_max_prob}\")\n",
        "\n",
        "  return entropy_test, std_test, max_prob_test, avg_test_entropy, avg_test_std, avg_test_max_prob"
      ],
      "metadata": {
        "id": "BFU65pfHHBPl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_entropy_dist(entropy_train, entropy_test, entropy_shrinked):\n",
        "  figure, axis = plt.subplots(1, 3, figsize=(18, 5), sharex=True)\n",
        "  sns.distplot(ax=axis[0], a=entropy_train, bins=10)\n",
        "  sns.distplot(ax=axis[1], a=entropy_test, bins=10)\n",
        "  # sns.distplot(ax=axis[2], a=entropy_shrinked, bins=10)\n",
        "\n",
        "  axis[0].set_title(\"Entropy Distribution for Training Set\")\n",
        "  axis[1].set_title(\"Entropy Distribution for Test Set\")\n",
        "  axis[2].set_title(\"Metric Mapping\")\n",
        "\n",
        "  axis[0].set_xlabel(\"Entropy\")\n",
        "  axis[1].set_xlabel(\"Entropy\")\n",
        "  axis[2].set_xlabel(\"Entropy\")"
      ],
      "metadata": {
        "id": "5DSjnRrj52jC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_std_dist(std_train, std_test, std_shrinked):\n",
        "  figure, axis = plt.subplots(1, 3, figsize=(18, 5), sharex=True)\n",
        "  sns.distplot(ax=axis[0], a=std_train, bins=10)\n",
        "  sns.distplot(ax=axis[1], a=std_test, bins=10)\n",
        "  # sns.distplot(ax=axis[2], a=entropy_shrinked, bins=10)\n",
        "\n",
        "  axis[0].set_title(\"Std Distribution for Training Set\")\n",
        "  axis[1].set_title(\"Std Distribution for Test Set\")\n",
        "  axis[2].set_title(\"Metric Mapping\")\n",
        "\n",
        "  axis[0].set_xlabel(\"Std\")\n",
        "  axis[1].set_xlabel(\"Std\")\n",
        "  axis[2].set_xlabel(\"Std\")"
      ],
      "metadata": {
        "id": "7smJngyq72kw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_max_prob_dist(max_prob_train, max_prob_test, max_prob_shrinked):\n",
        "  figure, axis = plt.subplots(1, 3, figsize=(18, 5), sharex=True)\n",
        "  sns.distplot(ax=axis[0], a=max_prob_train, bins=10)\n",
        "  sns.distplot(ax=axis[1], a=max_prob_test, bins=10)\n",
        "  # sns.distplot(ax=axis[2], a=entropy_shrinked, bins=10)\n",
        "\n",
        "  axis[0].set_title(\"Max Prob Distribution for Training Set\")\n",
        "  axis[1].set_title(\"Max Prob Distribution for Test Set\")\n",
        "  axis[2].set_title(\"Metric Mapping\")\n",
        "\n",
        "  axis[0].set_xlabel(\"Max Probability\")\n",
        "  axis[1].set_xlabel(\"Max Probability\")\n",
        "  axis[2].set_xlabel(\"Max Probability\")"
      ],
      "metadata": {
        "id": "XT2zifEIdsOw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"adult.csv\")\n",
        "data.head()\n",
        "\n",
        "# deleting records with missing values\n",
        "\n",
        "missing_workclass = data.index[data['workclass'] == '?'].tolist()\n",
        "data = data.drop(index = missing_workclass)\n",
        "\n",
        "missing_occupation = data.index[data['occupation'] == '?'].tolist()\n",
        "data = data.drop(index = missing_occupation)\n",
        "\n",
        "missing_country = data.index[data['native-country'] == '?'].tolist()\n",
        "data = data.drop(index = missing_country)\n",
        "\n",
        "print(data.info())\n",
        "\n",
        "# extracting numerical features\n",
        "num_features = data.select_dtypes(include=['int64', 'float64']).columns\n",
        "num_cols = data[num_features]\n",
        "print(num_cols.head())\n",
        "\n",
        "# applying one-hot encoding on categorical features\n",
        "ohe_data = pd.get_dummies(data)\n",
        "\n",
        "ohe_data = ohe_data.drop(columns=['age', 'fnlwgt', 'educational-num', 'capital-gain', 'capital-loss', 'hours-per-week'])\n",
        "ohe_cols = list(ohe_data)\n",
        "ohe_data = np.array(ohe_data)\n",
        "ohe_data = pd.DataFrame(ohe_data, columns=ohe_cols)\n",
        "\n",
        "print(ohe_data.head())\n",
        "\n",
        "# feature scaling for numerical features\n",
        "sc = StandardScaler()\n",
        "scaled_num_cols = sc.fit_transform(num_cols)\n",
        "scaled_data = pd.DataFrame(scaled_num_cols, columns=['age', 'fnlwgt', 'educational-num', 'capital-gain', 'capital-loss', 'hours-per-week'])\n",
        "print(scaled_data.head())\n",
        "\n",
        "# aggregating pre-processed data\n",
        "pre_pro_data = pd.concat([scaled_data, ohe_data], axis=1)\n",
        "pre_pro_data = pre_pro_data.dropna()\n",
        "print(pre_pro_data.head())\n",
        "\n",
        "# extracting target labels\n",
        "target_labels = pre_pro_data[['income_<=50K', 'income_>50K']]\n",
        "features = pre_pro_data.drop(columns=['income_<=50K', 'income_>50K'])\n",
        "target_labels = pd.DataFrame(target_labels, columns=['income_<=50K', 'income_>50K'])\n",
        "print(target_labels.head())\n",
        "\n",
        "# features\n",
        "print(features.head())\n",
        "\n",
        "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\n",
        "\n",
        "loss = tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=False, reduction=tf.losses.Reduction.NONE)\n",
        "\n",
        "train_X, test_X, train_y, test_y = train_test_split(features, target_labels, test_size = 0.25, random_state = 42)\n",
        "\n",
        "# for class imbalance\n",
        "# train_X, train_y = get_unequal_class_distribution(np.array(train_X), np.array(train_y), 20000)\n",
        "\n",
        "test_X = np.array(test_X)\n",
        "test_y = np.array(test_y)\n",
        "print(f'training examples - {train_X.shape}')\n",
        "\n",
        "model = create_model(optimizer, loss)\n",
        "history = model.fit(train_X, train_y, epochs = 100, batch_size = 64, validation_data = (test_X, test_y))\n",
        "\n",
        "plot_learning_curves(history)\n",
        "\n",
        "prob_vec_train = model.predict(train_X)\n",
        "prob_vec_test = model.predict(test_X)\n",
        "\n",
        "entropy_train, std_train, max_prob_train, avg_train_entropy, avg_train_std, avg_train_max_prob = compute_train_metrics(prob_vec_train)\n",
        "entropy_test, std_test, max_prob_test, avg_test_entropy, avg_test_std, avg_test_max_prob = compute_test_metrics(prob_vec_test)\n",
        "\n",
        "solutions = []\n",
        "\n",
        "# applying metric mapping\n",
        "for p in prob_vec_test:\n",
        "  p = p.tolist()\n",
        "  cur_max_prob = p.index(max(p))\n",
        "  solutions.append(make_metric_dist_same(2, max_prob_train, cur_max_prob))\n",
        "\n",
        "entropy_shrinked, std_shrinked, max_prob_shrinked, avg_shrinked_entropy, avg_shrinked_std, avg_shrinked_max_prob = compute_test_metrics(np.array(solutions))\n",
        "\n",
        "# calculate auc score using entropy as the parameter\n",
        "cal_auc(entropy_train, entropy_test, len(entropy_train), len(entropy_test), 0.00001)\n",
        "\n",
        "# calculate auc score using standard deviation as the parameter\n",
        "cal_auc(std_train, std_test, len(std_train), len(std_test), 0.00001)\n",
        "\n",
        "# calculate auc score using max probability as the parameter\n",
        "cal_auc(max_prob_train, max_prob_test, len(max_prob_train), len(max_prob_test), 0.0001)\n",
        "\n",
        "# get class distribution for training set\n",
        "train_y = pd.DataFrame(train_y, columns=['income_<=50K', 'income_>50K'])\n",
        "class_examples_train = [0] * 2\n",
        "\n",
        "for example in train_y['income_<=50K']:\n",
        "  if(example == 1):\n",
        "    class_examples_train[0] += 1\n",
        "  else:\n",
        "    class_examples_train[1] += 1\n",
        "\n",
        "print(f'class distribution for training dataset - {class_examples_train}')\n",
        "\n",
        "# get class distribution for testing set\n",
        "test_y = pd.DataFrame(test_y, columns=['income_<=50K', 'income_>50K'])\n",
        "class_examples_test = [0] * 2\n",
        "\n",
        "for example in test_y['income_<=50K']:\n",
        "  if(example == 1):\n",
        "    class_examples_test[0] += 1\n",
        "  else:\n",
        "    class_examples_test[1] += 1\n",
        "\n",
        "print(f'class distribution for training dataset - {class_examples_test}')\n",
        "\n",
        "threshold = get_threshold(entropy_shrinked)\n",
        "precision, recall = cal_pre_recall(entropy_train, entropy_shrinked, threshold)\n",
        "print(f\"precision - {precision}, recall - {recall}\")\n",
        "\n",
        "threshold = get_threshold(std_shrinked)\n",
        "precision, recall = cal_pre_recall(std_train, std_shrinked, threshold)\n",
        "print(f\"precision - {precision}, recall - {recall}\")\n",
        "\n",
        "threshold = get_threshold(max_prob_shrinked)\n",
        "precision, recall = cal_pre_recall(max_prob_train, max_prob_shrinked, threshold)\n",
        "print(f\"precision - {precision}, recall - {recall}\")"
      ],
      "metadata": {
        "id": "LVAK6dx1ESY6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}