DatapreProcessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMdv1DfHdo7JCrSgBdezoiV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abhinandan6123/AI-Projects/blob/main/DatapreProcessing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h3y4xTUayZte"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "IMPORTING LIBRARIES"
      ],
      "metadata": {
        "id": "No3wW-dmy0gw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import IPython.display as display\n",
        "from matplotlib import pyplot as plt\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "_eYW45Egy4YF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "IMPORTING DATASET"
      ],
      "metadata": {
        "id": "qYB8I8OzzVEj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pd.read_csv('delhivery.csv')\n",
        "x = dataset.iloc[:, :-1].values\n",
        "y = dataset.iloc[:, -1].values"
      ],
      "metadata": {
        "id": "qq8Ats8KzZq0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BaYjbtKY2Nyw",
        "outputId": "46c728a8-083a-4ebc-9cdb-bac6818a6051"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 14.0 11.0\n",
            "  11.9653]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 10.0 9.0\n",
            "  9.759]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 16.0 7.0\n",
            "  10.8152]\n",
            " ...\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 20.0 34.0\n",
            "  20.7053]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 17.0 27.0\n",
            "  18.8885]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 268.0 9.0\n",
            "  8.8088]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bQ7lylgh2RSN",
        "outputId": "2477e247-de22-4dd3-834f-ae278f25bf9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[ 1.27272727  1.11111111  2.28571429 ...  0.58823529  0.62962963\n",
            " 29.77777778]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "HANDLING MISSING DATA"
      ],
      "metadata": {
        "id": "MB7UwW3o2sLY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "numeric_columns = dataset.select_dtypes(include=[np.number]).columns\n",
        "non_numeric_columns = dataset.select_dtypes(exclude=[np.number]).columns"
      ],
      "metadata": {
        "id": "y1zyQF_X__1Y"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "numeric_imputer = SimpleImputer(strategy='mean')\n",
        "dataset[numeric_columns] = numeric_imputer.fit_transform(dataset[numeric_columns])\n"
      ],
      "metadata": {
        "id": "c6XhgWoSFnjy"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Forward fill for non-numeric columns (suitable for timestamps and categories)\n",
        "dataset[non_numeric_columns] = dataset[non_numeric_columns].fillna(method='ffill')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mywzQhTSF1Zq",
        "outputId": "81fe69e4-91d2-4d36-fe4a-0707aa835d7c"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-15-3e859ce3049b>:2: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n",
            "  dataset[non_numeric_columns] = dataset[non_numeric_columns].fillna(method='ffill')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zd_QAWvGDlF",
        "outputId": "ede0d2c4-6b46-432d-c67b-792c765f5f6c"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 14.0 11.0\n",
            "  11.9653]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 10.0 9.0\n",
            "  9.759]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 16.0 7.0\n",
            "  10.8152]\n",
            " ...\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 20.0 34.0\n",
            "  20.7053]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 17.0 27.0\n",
            "  18.8885]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 268.0 9.0\n",
            "  8.8088]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "date_columns = ['trip_creation_time', 'od_start_time', 'od_end_time', 'cutoff_timestamp']  # list your datetime columns\n",
        "for col in date_columns:\n",
        "    dataset[col] = pd.to_datetime(dataset[col], errors='coerce')\n"
      ],
      "metadata": {
        "id": "jfXtxH7gGe4a"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for col in date_columns:\n",
        "    dataset[f'{col}_year'] = dataset[col].dt.year\n",
        "    dataset[f'{col}_month'] = dataset[col].dt.month\n",
        "    dataset[f'{col}_day'] = dataset[col].dt.day\n",
        "    dataset[f'{col}_hour'] = dataset[col].dt.hour\n",
        "    dataset[f'{col}_minute'] = dataset[col].dt.minute\n"
      ],
      "metadata": {
        "id": "CjkaY_-qGnnX"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Example with one-hot encoding\n",
        "dataset = pd.get_dummies(dataset, columns=['route_type', 'source_center', 'destination_center'], drop_first=True)\n"
      ],
      "metadata": {
        "id": "GrmVB-ERHAcG"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "scaler = StandardScaler()\n",
        "scaled_columns = ['actual_distance_to_destination', 'actual_time', 'osrm_time', 'osrm_distance']  # add relevant columns\n",
        "dataset[scaled_columns] = scaler.fit_transform(dataset[scaled_columns])\n"
      ],
      "metadata": {
        "id": "hi-_Fb5EHIel"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Example using IQR for 'actual_distance_to_destination'\n",
        "Q1 = dataset['actual_distance_to_destination'].quantile(0.25)\n",
        "Q3 = dataset['actual_distance_to_destination'].quantile(0.75)\n",
        "IQR = Q3 - Q1\n",
        "dataset = dataset[(dataset['actual_distance_to_destination'] >= (Q1 - 1.5 * IQR)) &\n",
        "        (dataset['actual_distance_to_destination'] <= (Q3 + 1.5 * IQR))]\n"
      ],
      "metadata": {
        "id": "UPK-udVvHQbt"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = dataset  # replace 'target_column' with your target variable\n",
        "y = dataset\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n"
      ],
      "metadata": {
        "id": "95BkP7pAHtu2"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset.to_csv('preprocessed_data.csv', index=False)\n"
      ],
      "metadata": {
        "id": "1xmTqyzCI23x"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n"
      ],
      "metadata": {
        "id": "XwY7eYGeKCIo"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(dataset.head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "STtPf5R8KLSq",
        "outputId": "0dc7274c-a652-4af5-9f1c-5044a11c88a0"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       data         trip_creation_time  \\\n",
            "0  training 2018-09-20 02:35:36.476840   \n",
            "1  training 2018-09-20 02:35:36.476840   \n",
            "2  training 2018-09-20 02:35:36.476840   \n",
            "3  training 2018-09-20 02:35:36.476840   \n",
            "4  training 2018-09-20 02:35:36.476840   \n",
            "\n",
            "                                 route_schedule_uuid                trip_uuid  \\\n",
            "0  thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3...  trip-153741093647649320   \n",
            "1  thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3...  trip-153741093647649320   \n",
            "2  thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3...  trip-153741093647649320   \n",
            "3  thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3...  trip-153741093647649320   \n",
            "4  thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3...  trip-153741093647649320   \n",
            "\n",
            "                  source_name               destination_name  \\\n",
            "0  Anand_VUNagar_DC (Gujarat)  Khambhat_MotvdDPP_D (Gujarat)   \n",
            "1  Anand_VUNagar_DC (Gujarat)  Khambhat_MotvdDPP_D (Gujarat)   \n",
            "2  Anand_VUNagar_DC (Gujarat)  Khambhat_MotvdDPP_D (Gujarat)   \n",
            "3  Anand_VUNagar_DC (Gujarat)  Khambhat_MotvdDPP_D (Gujarat)   \n",
            "4  Anand_VUNagar_DC (Gujarat)  Khambhat_MotvdDPP_D (Gujarat)   \n",
            "\n",
            "               od_start_time                od_end_time  \\\n",
            "0 2018-09-20 03:21:32.418600 2018-09-20 04:47:45.236797   \n",
            "1 2018-09-20 03:21:32.418600 2018-09-20 04:47:45.236797   \n",
            "2 2018-09-20 03:21:32.418600 2018-09-20 04:47:45.236797   \n",
            "3 2018-09-20 03:21:32.418600 2018-09-20 04:47:45.236797   \n",
            "4 2018-09-20 03:21:32.418600 2018-09-20 04:47:45.236797   \n",
            "\n",
            "   start_scan_to_end_scan  is_cutoff  ...  destination_center_IND852131AAA  \\\n",
            "0                    86.0       True  ...                            False   \n",
            "1                    86.0       True  ...                            False   \n",
            "2                    86.0       True  ...                            False   \n",
            "3                    86.0       True  ...                            False   \n",
            "4                    86.0      False  ...                            False   \n",
            "\n",
            "  destination_center_IND852139AAB  destination_center_IND852201AAA  \\\n",
            "0                           False                            False   \n",
            "1                           False                            False   \n",
            "2                           False                            False   \n",
            "3                           False                            False   \n",
            "4                           False                            False   \n",
            "\n",
            "   destination_center_IND853204AAA  destination_center_IND854105AAA  \\\n",
            "0                            False                            False   \n",
            "1                            False                            False   \n",
            "2                            False                            False   \n",
            "3                            False                            False   \n",
            "4                            False                            False   \n",
            "\n",
            "   destination_center_IND854105AAB  destination_center_IND854311AAA  \\\n",
            "0                            False                            False   \n",
            "1                            False                            False   \n",
            "2                            False                            False   \n",
            "3                            False                            False   \n",
            "4                            False                            False   \n",
            "\n",
            "   destination_center_IND854326AAB  destination_center_IND854334AAA  \\\n",
            "0                            False                            False   \n",
            "1                            False                            False   \n",
            "2                            False                            False   \n",
            "3                            False                            False   \n",
            "4                            False                            False   \n",
            "\n",
            "   destination_center_IND854335AAA  \n",
            "0                            False  \n",
            "1                            False  \n",
            "2                            False  \n",
            "3                            False  \n",
            "4                            False  \n",
            "\n",
            "[5 rows x 3029 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VGVpZzhWKVzB",
        "outputId": "f4346297-6cbc-4f56-f319-5da38002183d"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 14.0 11.0\n",
            "  11.9653]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 10.0 9.0\n",
            "  9.759]\n",
            " ['training' '2018-09-20 02:35:36.476840'\n",
            "  'thanos::sroute:eb7bfc78-b351-4c0e-a951-fa3d5c3297ef' ... 16.0 7.0\n",
            "  10.8152]\n",
            " ...\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 20.0 34.0\n",
            "  20.7053]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 17.0 27.0\n",
            "  18.8885]\n",
            " ['training' '2018-09-20 16:24:28.436231'\n",
            "  'thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d55f058' ... 268.0 9.0\n",
            "  8.8088]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(dataset.tail())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_KUx_VynKfJB",
        "outputId": "d1a33f29-772a-4e00-bbb1-564f8c127602"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "            data         trip_creation_time  \\\n",
            "144862  training 2018-09-20 16:24:28.436231   \n",
            "144863  training 2018-09-20 16:24:28.436231   \n",
            "144864  training 2018-09-20 16:24:28.436231   \n",
            "144865  training 2018-09-20 16:24:28.436231   \n",
            "144866  training 2018-09-20 16:24:28.436231   \n",
            "\n",
            "                                      route_schedule_uuid  \\\n",
            "144862  thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d5...   \n",
            "144863  thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d5...   \n",
            "144864  thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d5...   \n",
            "144865  thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d5...   \n",
            "144866  thanos::sroute:f0569d2f-4e20-4c31-8542-67b86d5...   \n",
            "\n",
            "                      trip_uuid                 source_name  \\\n",
            "144862  trip-153746066843555182  Sonipat_Kundli_H (Haryana)   \n",
            "144863  trip-153746066843555182  Sonipat_Kundli_H (Haryana)   \n",
            "144864  trip-153746066843555182  Sonipat_Kundli_H (Haryana)   \n",
            "144865  trip-153746066843555182  Sonipat_Kundli_H (Haryana)   \n",
            "144866  trip-153746066843555182  Sonipat_Kundli_H (Haryana)   \n",
            "\n",
            "                     destination_name              od_start_time  \\\n",
            "144862  Gurgaon_Bilaspur_HB (Haryana) 2018-09-20 16:24:28.436231   \n",
            "144863  Gurgaon_Bilaspur_HB (Haryana) 2018-09-20 16:24:28.436231   \n",
            "144864  Gurgaon_Bilaspur_HB (Haryana) 2018-09-20 16:24:28.436231   \n",
            "144865  Gurgaon_Bilaspur_HB (Haryana) 2018-09-20 16:24:28.436231   \n",
            "144866  Gurgaon_Bilaspur_HB (Haryana) 2018-09-20 16:24:28.436231   \n",
            "\n",
            "                      od_end_time  start_scan_to_end_scan  is_cutoff  ...  \\\n",
            "144862 2018-09-20 23:32:09.618069                   427.0       True  ...   \n",
            "144863 2018-09-20 23:32:09.618069                   427.0       True  ...   \n",
            "144864 2018-09-20 23:32:09.618069                   427.0       True  ...   \n",
            "144865 2018-09-20 23:32:09.618069                   427.0       True  ...   \n",
            "144866 2018-09-20 23:32:09.618069                   427.0      False  ...   \n",
            "\n",
            "        destination_center_IND852131AAA destination_center_IND852139AAB  \\\n",
            "144862                            False                           False   \n",
            "144863                            False                           False   \n",
            "144864                            False                           False   \n",
            "144865                            False                           False   \n",
            "144866                            False                           False   \n",
            "\n",
            "        destination_center_IND852201AAA  destination_center_IND853204AAA  \\\n",
            "144862                            False                            False   \n",
            "144863                            False                            False   \n",
            "144864                            False                            False   \n",
            "144865                            False                            False   \n",
            "144866                            False                            False   \n",
            "\n",
            "        destination_center_IND854105AAA  destination_center_IND854105AAB  \\\n",
            "144862                            False                            False   \n",
            "144863                            False                            False   \n",
            "144864                            False                            False   \n",
            "144865                            False                            False   \n",
            "144866                            False                            False   \n",
            "\n",
            "        destination_center_IND854311AAA  destination_center_IND854326AAB  \\\n",
            "144862                            False                            False   \n",
            "144863                            False                            False   \n",
            "144864                            False                            False   \n",
            "144865                            False                            False   \n",
            "144866                            False                            False   \n",
            "\n",
            "        destination_center_IND854334AAA  destination_center_IND854335AAA  \n",
            "144862                            False                            False  \n",
            "144863                            False                            False  \n",
            "144864                            False                            False  \n",
            "144865                            False                            False  \n",
            "144866                            False                            False  \n",
            "\n",
            "[5 rows x 3029 columns]\n"
          ]
        }
      ]
    }
  ]
}