Fair_Knn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random,time,csv\n",
    "import numpy as np\n",
    "import math,copy,os\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import sklearn.metrics as metrics\n",
    "import matplotlib.pyplot as plt\n",
    "from statistics import mode\n",
    "\n",
    "k_odd = 11\n",
    "k_even = 6\n",
    "algorithm = 'kd_tree'\n",
    "metric = 'euclidean'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Load dataset\n",
    "from sklearn import preprocessing\n",
    "df_orig = pd.read_csv('dataset/adult.data.csv',na_values = ' ?')\n",
    "\n",
    "## Drop NULL values\n",
    "df_orig = df_orig.dropna(how='any',axis=0)\n",
    "\n",
    "## For testing\n",
    "df = copy.deepcopy(df_orig)\n",
    "\n",
    "df = pd.concat([df,pd.get_dummies(df['workclass'], prefix='workclass')],axis=1)\n",
    "df = pd.concat([df,pd.get_dummies(df['education'], prefix='education')],axis=1)\n",
    "df = pd.concat([df,pd.get_dummies(df['marital-status'], prefix='marital-status')],axis=1)\n",
    "df = pd.concat([df,pd.get_dummies(df['occupation'], prefix='occupation')],axis=1)\n",
    "df = pd.concat([df,pd.get_dummies(df['relationship'], prefix='relationship')],axis=1)\n",
    "\n",
    "\n",
    "\n",
    "df.drop(['workclass','education','marital-status','occupation','relationship','native-country'],axis=1, inplace=True)\n",
    "\n",
    "\n",
    "## Change symbolics to numerics\n",
    "df['sex'] = np.where(df['sex'] == ' Male', 1, 0)\n",
    "df['race'] = np.where(df['race'] != ' White', 0, 1)\n",
    "df['Probability'] = np.where(df['Probability'] == ' <=50K', 0, 1)\n",
    "\n",
    "\n",
    "## Discretize age\n",
    "df['age'] = np.where(df['age'] >= 70, 70, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 60 ) & (df['age'] < 70), 60, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 50 ) & (df['age'] < 60), 50, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 40 ) & (df['age'] < 50), 40, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 30 ) & (df['age'] < 40), 30, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 20 ) & (df['age'] < 30), 20, df['age'])\n",
    "df['age'] = np.where((df['age'] >= 10 ) & (df['age'] < 10), 10, df['age'])\n",
    "df['age'] = np.where(df['age'] < 10, 0, df['age'])\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "df = pd.DataFrame(scaler.fit_transform(df),columns = df.columns)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(df, test_size=0.2, shuffle = True)\n",
    "\n",
    "X_train, y_train = df_train.loc[:, df_train.columns != 'Probability'], df_train['Probability']\n",
    "\n",
    "\n",
    "X_test , y_test = df_test.loc[:, df_test.columns != 'Probability'], df_test['Probability']\n",
    "\n",
    "clf = LogisticRegression()\n",
    "\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Feature importance\n",
    "# import matplotlib.pyplot as plt\n",
    "# y = np.arange(len(df.columns)-1)\n",
    "# plt.barh(y,clf.coef_[0])\n",
    "# plt.yticks(y,df.columns)\n",
    "# plt.show()\n",
    "\n",
    "# print(clf.coef_[0])\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "print(cnf_matrix)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train unsupervised KDTree on training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KDTree\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "nbrs = NearestNeighbors(n_neighbors=k_even, algorithm=algorithm, metric=metric).fit(X_train)\n",
    "\n",
    "\n",
    "# ['auto', 'ball_tree', 'kd_tree', 'brute']\n",
    "#[“euclidean”,“manhattan”,“chebyshev”,“minkowski”,“wminkowski”,“seuclidean”,“mahalanobis”]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.drop(['Probability'],axis=1, inplace=True)\n",
    "\n",
    "def knnPred(df_orig,index_list,y_pred):\n",
    "    df_explain = df_orig.iloc[index_list,:]\n",
    "    df_explain['Probability']  = np.where(df_explain['Probability'] == ' >50K', 1, 0)\n",
    "    prabibilty_list = df_explain['Probability']    \n",
    "    knn_pred = mode(prabibilty_list) ## Majority Voting         \n",
    "    if (y_pred[0] != knn_pred):\n",
    "        return df_explain\n",
    "    else:\n",
    "        return \"OK\"\n",
    "        \n",
    "           \n",
    "count = 0\n",
    "\n",
    "for index,row in df_test.iterrows():    \n",
    "    row_ = [row.values]    \n",
    "    y_pred = clf.predict(row_)        \n",
    "    if y_pred[0] == 0:\n",
    "        distances, knn_list = nbrs.kneighbors(row_)\n",
    "        index_list = np.append(knn_list[0],index)            \n",
    "        result = knnPred(df_orig,index_list,y_pred)\n",
    "        if result is not \"OK\":\n",
    "            csv_name = \"Adult\\euclidean_Adult\"+str(count)+\".csv\"\n",
    "            result.to_csv(csv_name)\n",
    "            count += 1\n",
    "\n",
    "print(count)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}