mushroom-classification-rfc-1-0-score.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/romanvelichkin/mushroom-classification-rfc-1-0-score?scriptVersionId=142816653\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","source":"## Mushroom classification\nСlassify mushrooms into poisonous and edible according to their appearance.\n\n### Result:\nMushrooms are well studied, and people have long known how to distinguish poisonous mushrooms from edible ones.\n\nAs initial data, each fungus is described by a detailed set of external features.\n\nThese features were enough to create a model that predicts with 100% accuracy whether this mushroom can be eaten.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-25T14:43:02.67446Z","iopub.execute_input":"2022-08-25T14:43:02.674857Z","iopub.status.idle":"2022-08-25T14:43:03.632939Z","shell.execute_reply.started":"2022-08-25T14:43:02.674825Z","shell.execute_reply":"2022-08-25T14:43:03.631925Z"}}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# enable drawing plots in jupyter\n%matplotlib inline\n\n# import models from scikit-learn\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier\n\n# import model evaluations\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import precision_score, recall_score, f1_score\nfrom sklearn.metrics import plot_roc_curve\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:47.892369Z","iopub.execute_input":"2022-10-01T07:14:47.892864Z","iopub.status.idle":"2022-10-01T07:14:49.802247Z","shell.execute_reply.started":"2022-10-01T07:14:47.892752Z","shell.execute_reply":"2022-10-01T07:14:49.800716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import data","metadata":{}},{"cell_type":"code","source":"# Read data\ndf = pd.read_csv(\"/kaggle/input/mushroom-classification/mushrooms.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:49.804926Z","iopub.execute_input":"2022-10-01T07:14:49.806409Z","iopub.status.idle":"2022-10-01T07:14:49.914934Z","shell.execute_reply.started":"2022-10-01T07:14:49.806332Z","shell.execute_reply":"2022-10-01T07:14:49.913809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data exploration","metadata":{"execution":{"iopub.status.busy":"2022-08-22T12:40:10.174768Z","iopub.execute_input":"2022-08-22T12:40:10.17521Z","iopub.status.idle":"2022-08-22T12:40:10.180619Z","shell.execute_reply.started":"2022-08-22T12:40:10.175174Z","shell.execute_reply":"2022-08-22T12:40:10.179305Z"}}},{"cell_type":"code","source":"# Check how data is distrubuted between two mushroom classes\ndf[\"class\"].value_counts().plot(kind=\"bar\");","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:49.916275Z","iopub.execute_input":"2022-10-01T07:14:49.916995Z","iopub.status.idle":"2022-10-01T07:14:50.192587Z","shell.execute_reply.started":"2022-10-01T07:14:49.916953Z","shell.execute_reply":"2022-10-01T07:14:50.19094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look what type of data we have\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.196616Z","iopub.execute_input":"2022-10-01T07:14:50.198671Z","iopub.status.idle":"2022-10-01T07:14:50.234373Z","shell.execute_reply.started":"2022-10-01T07:14:50.198616Z","shell.execute_reply":"2022-10-01T07:14:50.232885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Is there any missing data\ndf.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.236369Z","iopub.execute_input":"2022-10-01T07:14:50.236884Z","iopub.status.idle":"2022-10-01T07:14:50.257075Z","shell.execute_reply.started":"2022-10-01T07:14:50.236835Z","shell.execute_reply":"2022-10-01T07:14:50.255815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\nAll data is non-numerical. Transform it into numerical categories.","metadata":{}},{"cell_type":"code","source":"# Transform data into numerical categories\n\nfrom sklearn.preprocessing import LabelEncoder\n\nencoder = LabelEncoder()\nfor i in df.columns:\n    df[i] = encoder.fit_transform(df[i])\n\ndf","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.258926Z","iopub.execute_input":"2022-10-01T07:14:50.259684Z","iopub.status.idle":"2022-10-01T07:14:50.373699Z","shell.execute_reply.started":"2022-10-01T07:14:50.259634Z","shell.execute_reply":"2022-10-01T07:14:50.372562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cross-validation showed large score difference between batch samples.\nShuffling data could help to solve this problem.","metadata":{}},{"cell_type":"code","source":"# Shuffle data\nnp.random.seed(99)\n\ndf = df.sample(frac=1).reset_index()\ndf = df.drop(\"index\", axis=1)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.375013Z","iopub.execute_input":"2022-10-01T07:14:50.375422Z","iopub.status.idle":"2022-10-01T07:14:50.411138Z","shell.execute_reply.started":"2022-10-01T07:14:50.375387Z","shell.execute_reply":"2022-10-01T07:14:50.40973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"code","source":"# Split data into X (features) and y (labels)\nX = df.drop(\"class\", axis=1)\ny = df[\"class\"]\ny","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.416615Z","iopub.execute_input":"2022-10-01T07:14:50.417269Z","iopub.status.idle":"2022-10-01T07:14:50.436223Z","shell.execute_reply.started":"2022-10-01T07:14:50.417211Z","shell.execute_reply":"2022-10-01T07:14:50.434697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data into train and test sets\nnp.random.seed(99)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.438389Z","iopub.execute_input":"2022-10-01T07:14:50.439383Z","iopub.status.idle":"2022-10-01T07:14:50.45375Z","shell.execute_reply.started":"2022-10-01T07:14:50.439183Z","shell.execute_reply":"2022-10-01T07:14:50.448911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to fit and score models\ndef fit_and_score(models, X_train, X_test, y_train, y_test):\n    \"\"\"\n    Fits and evaluates given machine learning models.\n    model: a dict of different models\n    X_train: training data (no labels)\n    X_test: test data (no labels)\n    y_train: training labels\n    y_train: training labels\n    \"\"\"\n    np.random.seed(99)\n    \n    #dict to keep model scores\n    model_scores = {}\n    \n    # loop through models\n    for name, model in models.items():\n        # fit model to the data\n        model.fit(X_train, y_train)\n        # evaluate model\n        model_scores[name] = model.score(X_test, y_test)\n    \n    return model_scores","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.459663Z","iopub.execute_input":"2022-10-01T07:14:50.460598Z","iopub.status.idle":"2022-10-01T07:14:50.476107Z","shell.execute_reply.started":"2022-10-01T07:14:50.460539Z","shell.execute_reply":"2022-10-01T07:14:50.47379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look how different models solve this classification problem.","metadata":{}},{"cell_type":"code","source":"# LinearSVC\n# LogisticRegression\n# KNeighborsClassifier\n# RandomForestClassifier\n\n# Create dictionary with models\nmodels = {\"LinearSVC\": LinearSVC(),\n          \"KNeighborsClassifier\": KNeighborsClassifier(),\n          \"RandomForestClassifier\": RandomForestClassifier(),\n          \"LogisticRegression\": LogisticRegression()}\n\nmodel_scores = fit_and_score(models, X_train, X_test, y_train, y_test)\n\nmodel_scores","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:50.47809Z","iopub.execute_input":"2022-10-01T07:14:50.478979Z","iopub.status.idle":"2022-10-01T07:14:52.205402Z","shell.execute_reply.started":"2022-10-01T07:14:50.478925Z","shell.execute_reply":"2022-10-01T07:14:52.203283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluating model \nI've decided to use RandomForestClassifier because it has 100% accuracy and doesn't require any tuning.","metadata":{}},{"cell_type":"code","source":"# Train RFC model\nnp.random.seed(99)\n\nrfc_model = RandomForestClassifier()\nrfc_model.fit(X_train, y_train)\nprint(\"RandomForestClassifier score:\", rfc_model.score(X_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:52.207358Z","iopub.execute_input":"2022-10-01T07:14:52.208193Z","iopub.status.idle":"2022-10-01T07:14:52.739608Z","shell.execute_reply.started":"2022-10-01T07:14:52.208143Z","shell.execute_reply":"2022-10-01T07:14:52.73786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get predictions on test dataset using trained model\ny_preds = rfc_model.predict(X_test)\ny_preds","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:52.741015Z","iopub.execute_input":"2022-10-01T07:14:52.741386Z","iopub.status.idle":"2022-10-01T07:14:52.786875Z","shell.execute_reply.started":"2022-10-01T07:14:52.741344Z","shell.execute_reply":"2022-10-01T07:14:52.785511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot ROC curve, calculate AUC metric and confusion matrix","metadata":{}},{"cell_type":"code","source":"# ROC curve\nplot_roc_curve(rfc_model, X_test, y_test);","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:52.788572Z","iopub.execute_input":"2022-10-01T07:14:52.788992Z","iopub.status.idle":"2022-10-01T07:14:53.034028Z","shell.execute_reply.started":"2022-10-01T07:14:52.788956Z","shell.execute_reply":"2022-10-01T07:14:53.032728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Confusion matrix\n\nprint(confusion_matrix(y_preds, y_test))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:53.035588Z","iopub.execute_input":"2022-10-01T07:14:53.036526Z","iopub.status.idle":"2022-10-01T07:14:53.048233Z","shell.execute_reply.started":"2022-10-01T07:14:53.036472Z","shell.execute_reply":"2022-10-01T07:14:53.046535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classification report as precision, recall and f1-score\n\nprint(classification_report(y_test, y_preds))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:53.051689Z","iopub.execute_input":"2022-10-01T07:14:53.052726Z","iopub.status.idle":"2022-10-01T07:14:53.069898Z","shell.execute_reply.started":"2022-10-01T07:14:53.052669Z","shell.execute_reply":"2022-10-01T07:14:53.068752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cross-validated score","metadata":{}},{"cell_type":"code","source":"# Cross-validated accuracy\nnp.random.seed(99)\n\ncv_accuracy = cross_val_score(rfc_model, X, y, scoring=\"accuracy\")\nprint(cv_accuracy)\n\ncv_accuracy_mean = np.mean(cv_accuracy)\nprint(\"mean accuracy:\", cv_accuracy_mean)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:53.071565Z","iopub.execute_input":"2022-10-01T07:14:53.072411Z","iopub.status.idle":"2022-10-01T07:14:55.277867Z","shell.execute_reply.started":"2022-10-01T07:14:53.072369Z","shell.execute_reply":"2022-10-01T07:14:55.276553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cross-validated precision\nnp.random.seed(99)\n\ncv_precision = cross_val_score(rfc_model, X, y, scoring=\"precision\")\nprint(cv_precision)\n\ncv_precision_mean = np.mean(cv_precision)\nprint(\"mean precision:\", cv_precision_mean)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:55.279569Z","iopub.execute_input":"2022-10-01T07:14:55.280382Z","iopub.status.idle":"2022-10-01T07:14:57.436606Z","shell.execute_reply.started":"2022-10-01T07:14:55.280337Z","shell.execute_reply":"2022-10-01T07:14:57.435315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cross-validated recall\nnp.random.seed(99)\n\ncv_recall = cross_val_score(rfc_model, X, y, scoring=\"recall\")\nprint(cv_recall)\n\ncv_recall_mean = np.mean(cv_recall)\nprint(\"recall mean:\", cv_recall_mean)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:57.438713Z","iopub.execute_input":"2022-10-01T07:14:57.43966Z","iopub.status.idle":"2022-10-01T07:14:59.650008Z","shell.execute_reply.started":"2022-10-01T07:14:57.439606Z","shell.execute_reply":"2022-10-01T07:14:59.648151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cross-validated f1-score\nnp.random.seed(99)\n\ncv_f1 = cross_val_score(rfc_model, X, y, scoring=\"f1\")\nprint(cv_f1)\n\ncv_f1_mean = np.mean(cv_f1)\nprint(\"f1-score mean:\", cv_f1_mean)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:14:59.652493Z","iopub.execute_input":"2022-10-01T07:14:59.653345Z","iopub.status.idle":"2022-10-01T07:15:01.853391Z","shell.execute_reply.started":"2022-10-01T07:14:59.653301Z","shell.execute_reply":"2022-10-01T07:15:01.852144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visialisaion of cross-validated metrics\n\ncv_metrics = pd.DataFrame({\"Accuracy\": cv_accuracy_mean,\n                           \"Precision\": cv_precision_mean,\n                           \"Recall\": cv_recall_mean,\n                           \"F1-score\": cv_f1_mean},\n                          index=[0])\ncv_metrics.T.plot.bar(title=\"Cross-validated classification metrics\",\n                      legend=False);","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:15:01.855233Z","iopub.execute_input":"2022-10-01T07:15:01.855745Z","iopub.status.idle":"2022-10-01T07:15:02.049598Z","shell.execute_reply.started":"2022-10-01T07:15:01.855698Z","shell.execute_reply":"2022-10-01T07:15:02.048118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature importance","metadata":{}},{"cell_type":"code","source":"# Get coefficients\nrfc_model.feature_importances_","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:15:02.05146Z","iopub.execute_input":"2022-10-01T07:15:02.052566Z","iopub.status.idle":"2022-10-01T07:15:02.078142Z","shell.execute_reply.started":"2022-10-01T07:15:02.052507Z","shell.execute_reply":"2022-10-01T07:15:02.076472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Match coef of features to columns\n# Exclude labels from 'columns' (0 column)\nfeature_dict = dict(zip(df.columns[1:], rfc_model.feature_importances_))\nfeature_dict","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:15:02.079429Z","iopub.execute_input":"2022-10-01T07:15:02.079796Z","iopub.status.idle":"2022-10-01T07:15:02.108762Z","shell.execute_reply.started":"2022-10-01T07:15:02.079747Z","shell.execute_reply":"2022-10-01T07:15:02.10741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualisation of feature importance\nfeature_df = pd.DataFrame(feature_dict, index=[0])\nfeature_df.T.plot.bar(title=\"Feature importance\",\n                      legend=False,\n                      figsize=(10,10));","metadata":{"execution":{"iopub.status.busy":"2022-10-01T07:15:02.110586Z","iopub.execute_input":"2022-10-01T07:15:02.113679Z","iopub.status.idle":"2022-10-01T07:15:02.47276Z","shell.execute_reply.started":"2022-10-01T07:15:02.113578Z","shell.execute_reply":"2022-10-01T07:15:02.471501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some features are more important than others: odor, gill-size, gill-color and spore print color.","metadata":{}},{"cell_type":"markdown","source":"## Result\nMushrooms are well studied, and people have long known how to distinguish poisonous mushrooms from edible ones.\n\nGiven features were enough to create a model that predicts with 100% accuracy whether this mushroom can be eaten.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}