preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import pickle\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Read source file(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "sd = pd.read_csv(\"data\\DTI-AAL-SD.csv\")\n",
    "mean = pd.read_csv(\"data\\DTI-AAL-Mean.csv\")\n",
    "volume = pd.read_csv(\"data\\DTI-AAL-Volume.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Remove empty rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "sd.dropna(inplace=True)\n",
    "mean.dropna(inplace=True)\n",
    "volume.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Eliminate replicated columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "columns=[\"age\", \"gender\", \"handedness\", \"affected hemisphere\", \"post-stroke time\", \"group\"]\n",
    "sd.drop(columns=columns, inplace=True)\n",
    "mean.drop(columns=columns, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Fix the name of the columns\n",
    "- Remove the number that have been added by pandas\n",
    "- Attach the related parameter's name to first of the columns\n",
    "- Attach the data set name to the fist of each column due to the merge action that will be done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "DTI_PARAMETERS=[\"FA\", \"MD\", \"AxD\", \"RD\", \"RA\", \"TEMP\"]\n",
    "datasets ={\n",
    "    'mean':mean,\n",
    "    'sd':sd\n",
    "}\n",
    "for dataset_key in datasets.keys():\n",
    "    dataset=datasets.get(dataset_key)\n",
    "    parameter_index=0\n",
    "    columns=dataset.columns\n",
    "    columns=[re.sub('.\\d{1}$', '', index) for index in columns]\n",
    "    for index in range(1, len(columns)):\n",
    "        if columns[index].startswith(DTI_PARAMETERS[parameter_index+1]):\n",
    "            parameter_index+=1\n",
    "        elif not columns[index].startswith(DTI_PARAMETERS[parameter_index]):\n",
    "            columns[index]=\"{0}_{1}\".format(DTI_PARAMETERS[parameter_index], columns[index])\n",
    "        columns[index]=\"{0}_{1}\".format(dataset_key, columns[index])\n",
    "    globals()[dataset_key]=pd.DataFrame(data=dataset.to_numpy(), columns=columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Fix data type of some columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "columns = [\"file code\", \"post-stroke time\", \"age\"]\n",
    "volume[columns] = volume[columns].applymap(np.int64)\n",
    "columns = mean.columns\n",
    "mean[columns] = mean[columns].applymap(np.float64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Merge three dataframe into a comprehensive dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.merge(volume, mean, on=\"file code\", how=\"outer\")\n",
    "df = pd.merge(df, sd, on=\"file code\", how=\"outer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Encode some columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "df[\"group\"] = encoder.fit_transform(df[\"group\"])\n",
    "df[\"gender\"] = encoder.fit_transform(df[\"gender\"])\n",
    "df[\"handedness\"] = encoder.fit_transform(df[\"handedness\"])\n",
    "df[\"affected hemisphere\"] = encoder.fit_transform(df[\"affected hemisphere\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Split the target and source columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"config/train_params.json\") as file:\n",
    "    columns=json.load(file)\n",
    "x = df[columns]\n",
    "y = df[\"group\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Compute the combined columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "x[\"mean_AxD_Frontal_Sup_L * mean_RD_Frontal_Inf_Oper_R\"] = df[\"mean_AxD_Frontal_Sup_L\"] * df[\"mean_RD_Frontal_Inf_Oper_R\"]\n",
    "x[\"Occipital_Sup_R * mean_AxD_Heschl_L\"] = df[\"Occipital_Sup_R\"]*df[\"mean_AxD_Heschl_L\"]\n",
    "x[\"Volume Occipital_Sup_R * mean_FA_Frontal_Inf_Tri_L\"] = df[\"Occipital_Sup_R\"]*df[\"mean_FA_Frontal_Inf_Tri_L\"]\n",
    "x[\"mean_MD_Caudate_L * Paracentral_Lobule_L\"] = df[\"mean_MD_Caudate_L\"]*df[\"Paracentral_Lobule_L\"]\n",
    "x[\"mean_AxD_Rolandic_Oper_R * mean_MD_Frontal_Mid_L\"] = df[\"mean_AxD_Rolandic_Oper_R\"]*df[\"mean_MD_Frontal_Mid_L\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Save the data frame into a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"data\\DataFrame.pickle\", \"wb\") as file:\n",
    "    pickle.dump([x, y, df], file, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}