2.dataPreprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a07406eb",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a37b5a5b",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d267daa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ba164fe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# useful pandas settings\n",
    "\n",
    "# Use 2 decimal places in the output display\n",
    "pd.set_option(\"display.precision\", 2)\n",
    "\n",
    "# Don't wrap dataframe across additional lines\n",
    "pd.set_option(\"display.expand_frame_repr\", False)\n",
    "\n",
    "# Set the maximum widths of columns\n",
    "pd.set_option(\"display.max_colwidth\", 60)\n",
    "\n",
    "# Set max rows displayed in output to 20\n",
    "pd.set_option(\"display.max_rows\", 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67b912ab",
   "metadata": {},
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5008aa8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(52039, 9)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import data\n",
    "df_raw = pd.read_csv('data/data_raw/merged.csv')\n",
    "df_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48178ce2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_url</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://towardsdatascience.com/@henriwoodcock</td>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01T13:59:37.090Z</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://towardsdatascience.com/@destingong</td>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01T05:00:14.502Z</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      author_url       user_id                      date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url\n",
       "0  https://towardsdatascience.com/@henriwoodcock  e9ee1187182a  2021-03-01T13:59:37.090Z   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...\n",
       "1     https://towardsdatascience.com/@destingong  fa1913854e95  2021-03-01T05:00:14.502Z  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# take a look at the raw data\n",
    "df_raw.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "be65c22c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_url      object\n",
       "user_id         object\n",
       "date            object\n",
       "reading_time    object\n",
       "title           object\n",
       "subtitle        object\n",
       "claps           object\n",
       "responses       object\n",
       "story_url       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list the columns with their datatypes\n",
    "df_raw.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "de9805cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "author_url      12090\n",
       "user_id         12057\n",
       "date            51985\n",
       "reading_time       61\n",
       "title           51762\n",
       "subtitle        36663\n",
       "claps            1139\n",
       "responses          99\n",
       "story_url       52038\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count the number of distinct entries\n",
    "df_raw.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "029f607b",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-block alert-info\">\n",
    "<b>NOTES:</b>\n",
    "- Both author_url and user_id contain author identifiers, keep only the latter. \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a172200b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01T13:59:37.090Z</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01T05:00:14.502Z</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                      date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url\n",
       "0  e9ee1187182a  2021-03-01T13:59:37.090Z   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...\n",
       "1  fa1913854e95  2021-03-01T05:00:14.502Z  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop author url column \n",
    "df = df_raw.drop(columns=['author_url'])\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d42c058c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1867</th>\n",
       "      <td>6bee41715a0e</td>\n",
       "      <td>2019-08-14T04:23:32.108Z</td>\n",
       "      <td>5 min read</td>\n",
       "      <td>Naive Bayes Explained</td>\n",
       "      <td>-</td>\n",
       "      <td>418</td>\n",
       "      <td>2 responses</td>\n",
       "      <td>https://towardsdatascience.com/naive-bayes-explained-9d2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3305</th>\n",
       "      <td>dd2051e0a449</td>\n",
       "      <td>2019-03-04T13:35:15.151Z</td>\n",
       "      <td>8 min read</td>\n",
       "      <td>LOGISTIC REGRESSION CLASSIFIER</td>\n",
       "      <td>-</td>\n",
       "      <td>75</td>\n",
       "      <td>3 responses</td>\n",
       "      <td>https://towardsdatascience.com/logistic-regression-class...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3490</th>\n",
       "      <td>d88e5b6154ef</td>\n",
       "      <td>2019-03-12T11:43:12.796Z</td>\n",
       "      <td>9 min read</td>\n",
       "      <td>Is Data Science a science?</td>\n",
       "      <td>-</td>\n",
       "      <td>206</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/data-science-and-ai-for-b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4032</th>\n",
       "      <td>c510ccc9027c</td>\n",
       "      <td>2019-01-06T19:52:31.210Z</td>\n",
       "      <td>8 min read</td>\n",
       "      <td>Know your enemy</td>\n",
       "      <td>How you can create and defend against adversarial attacks</td>\n",
       "      <td>450</td>\n",
       "      <td>3 responses</td>\n",
       "      <td>https://towardsdatascience.com/know-your-enemy-7f7c5038b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4268</th>\n",
       "      <td>451599b1142a</td>\n",
       "      <td>2019-01-16T14:02:13.827Z</td>\n",
       "      <td>6 min read</td>\n",
       "      <td>Learn Enough Docker to be Useful</td>\n",
       "      <td>Part 2: A Delicious Dozen Docker Terms You Need to Know</td>\n",
       "      <td>6.6K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/learn-enough-docker-to-be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50943</th>\n",
       "      <td>120b86134681</td>\n",
       "      <td>2022-12-30T00:45:15.776Z</td>\n",
       "      <td>10 min read</td>\n",
       "      <td>-</td>\n",
       "      <td>Python Helper Classes for EDA, Feature…</td>\n",
       "      <td>190</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/mastering-data-science-wo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51189</th>\n",
       "      <td>ff8eba68ccde</td>\n",
       "      <td>2021-08-07T17:56:42.568Z</td>\n",
       "      <td>7 min read</td>\n",
       "      <td>Optimization Algorithms for Machine Learning</td>\n",
       "      <td>Chapter-6: Optimization Problems</td>\n",
       "      <td>25</td>\n",
       "      <td>0 responses</td>\n",
       "      <td>https://towardsdatascience.com/optimization-algorithms-f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51454</th>\n",
       "      <td>d0e73ebdbb0c</td>\n",
       "      <td>2021-08-14T16:48:55.304Z</td>\n",
       "      <td>6 min read</td>\n",
       "      <td>My Google Foobar journey</td>\n",
       "      <td>WALK-THROUGH</td>\n",
       "      <td>3</td>\n",
       "      <td>0 responses</td>\n",
       "      <td>https://towardsdatascience.com/my-google-foobar-journey-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51508</th>\n",
       "      <td>e101cd051ea3</td>\n",
       "      <td>2021-08-16T21:21:39.531Z</td>\n",
       "      <td>6 min read</td>\n",
       "      <td>How to Handle Missing Data</td>\n",
       "      <td>The fastest multiple imputation method using XGBoost</td>\n",
       "      <td>147</td>\n",
       "      <td>3 responses</td>\n",
       "      <td>https://towardsdatascience.com/how-to-handle-missing-dat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51805</th>\n",
       "      <td>df1ad2ced320</td>\n",
       "      <td>2021-08-24T16:26:21.732Z</td>\n",
       "      <td>3 min read</td>\n",
       "      <td>Interactive Data Visualization</td>\n",
       "      <td>Using Altair for Creating Data Visualizations</td>\n",
       "      <td>34</td>\n",
       "      <td>2 responses</td>\n",
       "      <td>https://towardsdatascience.com/interactive-data-visualiz...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>277 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            user_id                      date reading_time                                         title                                                   subtitle claps    responses                                                    story_url\n",
       "1867   6bee41715a0e  2019-08-14T04:23:32.108Z   5 min read                         Naive Bayes Explained                                                          -   418  2 responses  https://towardsdatascience.com/naive-bayes-explained-9d2...\n",
       "3305   dd2051e0a449  2019-03-04T13:35:15.151Z   8 min read                LOGISTIC REGRESSION CLASSIFIER                                                          -    75  3 responses  https://towardsdatascience.com/logistic-regression-class...\n",
       "3490   d88e5b6154ef  2019-03-12T11:43:12.796Z   9 min read                    Is Data Science a science?                                                          -   206   1 response  https://towardsdatascience.com/data-science-and-ai-for-b...\n",
       "4032   c510ccc9027c  2019-01-06T19:52:31.210Z   8 min read                               Know your enemy  How you can create and defend against adversarial attacks   450  3 responses  https://towardsdatascience.com/know-your-enemy-7f7c5038b...\n",
       "4268   451599b1142a  2019-01-16T14:02:13.827Z   6 min read              Learn Enough Docker to be Useful    Part 2: A Delicious Dozen Docker Terms You Need to Know  6.6K  8 responses  https://towardsdatascience.com/learn-enough-docker-to-be...\n",
       "...             ...                       ...          ...                                           ...                                                        ...   ...          ...                                                          ...\n",
       "50943  120b86134681  2022-12-30T00:45:15.776Z  10 min read                                             -                    Python Helper Classes for EDA, Feature…   190   1 response  https://towardsdatascience.com/mastering-data-science-wo...\n",
       "51189  ff8eba68ccde  2021-08-07T17:56:42.568Z   7 min read  Optimization Algorithms for Machine Learning                           Chapter-6: Optimization Problems    25  0 responses  https://towardsdatascience.com/optimization-algorithms-f...\n",
       "51454  d0e73ebdbb0c  2021-08-14T16:48:55.304Z   6 min read                      My Google Foobar journey                                               WALK-THROUGH     3  0 responses  https://towardsdatascience.com/my-google-foobar-journey-...\n",
       "51508  e101cd051ea3  2021-08-16T21:21:39.531Z   6 min read                    How to Handle Missing Data       The fastest multiple imputation method using XGBoost   147  3 responses  https://towardsdatascience.com/how-to-handle-missing-dat...\n",
       "51805  df1ad2ced320  2021-08-24T16:26:21.732Z   3 min read                Interactive Data Visualization              Using Altair for Creating Data Visualizations    34  2 responses  https://towardsdatascience.com/interactive-data-visualiz...\n",
       "\n",
       "[277 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# investigate the duplicated titles\n",
    "df[df.title.duplicated()==True]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "753fc828",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-block alert-info\">\n",
    "<b>NOTES:</b>\n",
    "- Several titles appear more than once. Drop all the rows that have duplicated titles. This might be due to the authors editing an initial version. We will keep the last post among each set of duplicates.\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0bd56136",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(51762, 8)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows with duplicated titles\n",
    "df.drop_duplicates(subset='title', keep='last', inplace=True)\n",
    "\n",
    "# check the output\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "537d7997",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url\n",
       "0  e9ee1187182a  2021-03-01   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...\n",
       "1  fa1913854e95  2021-03-01  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change object type column to datetime and extract the date \n",
    "df.date = pd.to_datetime(df.date).dt.date\n",
    "\n",
    "# check the outcome\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "308d74de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url  year\n",
       "0  e9ee1187182a  2021-03-01   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...  2021\n",
       "1  fa1913854e95  2021-03-01  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator...  2021"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the year as an integer\n",
    "df['year'] = pd.DatetimeIndex(df['date']).year\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e1914d7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url  year  month\n",
       "0  e9ee1187182a  2021-03-01   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3\n",
       "1  fa1913854e95  2021-03-01  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator...  2021      3"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the month as an integer\n",
    "df['month'] = pd.DatetimeIndex(df['date']). month\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c07fbf3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4 min read</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11 min read</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date reading_time                                                     title                                                  subtitle claps    responses                                                    story_url  year  month  day\n",
       "0  e9ee1187182a  2021-03-01   4 min read                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1\n",
       "1  fa1913854e95  2021-03-01  11 min read  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the day as an integer\n",
    "df['day'] = pd. DatetimeIndex(df['date']).day\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "82c984b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1K</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                     title                                                  subtitle claps    responses                                                    story_url  year  month  day\n",
       "0  e9ee1187182a  2021-03-01             4                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python   150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1\n",
       "1  fa1913854e95  2021-03-01            11  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…    1K  8 responses  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract reading time in minutes and change to integer type\n",
    "df.reading_time = df.reading_time.apply(lambda x: int(x[:-9]))\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1f0ee4ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# source: https://dorianlazar.medium.com/scraping-medium-with-python-beautiful-soup-3314f898bbf5\n",
    "\n",
    "def parse_claps(claps_str):\n",
    "    \n",
    "    '''\n",
    "    Function to parse the number of claps.\n",
    "    It splits on character 'K' and then transforms the string into an integer,\n",
    "    also multiplies by 1000 if 'K' is present in the string.\n",
    "    \n",
    "    INPUT:\n",
    "        claps_str (str) - number of claps as a string\n",
    "    OUTPUT:\n",
    "        claps (int) - number of claps as an integer\n",
    "    '''\n",
    "    if (claps_str is None) or (claps_str == '') or (claps_str.split is None) or (len(claps_str) > 5):\n",
    "        return 0\n",
    "    split_claps = claps_str.split('K')\n",
    "    claps = float(split_claps[0])\n",
    "    claps = int(claps*1000) if len(split_claps) == 2 else int(claps)\n",
    "    return claps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "73ae8e7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1 response</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1000</td>\n",
       "      <td>8 responses</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                     title                                                  subtitle  claps    responses                                                    story_url  year  month  day\n",
       "0  e9ee1187182a  2021-03-01             4                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python    150   1 response  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1\n",
       "1  fa1913854e95  2021-03-01            11  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…   1000  8 responses  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parse the number of claps\n",
    "df.claps = df.claps.apply(lambda x: parse_claps(x))\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bc10af0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1000</td>\n",
       "      <td>8</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                     title                                                  subtitle  claps  responses                                                    story_url  year  month  day\n",
       "0  e9ee1187182a  2021-03-01             4                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python    150          1  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1\n",
       "1  fa1913854e95  2021-03-01            11  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…   1000          8  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parse the number of responses\n",
    "df.responses = df.responses.apply(lambda x: int(x.split(' ')[0]))\n",
    "\n",
    "# check the outcome\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6c18a023",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1fc707a6f9e6'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_str = 'https://towardsdatascience.com/customer-segmentation-in-online-retail-1fc707a6f9e6?source=collection_archive---------2-----------------------'\n",
    "# extract the article identifier\n",
    "test_step = test_str[:test_str.index('?')][-12:]\n",
    "test_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7de8623b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>story_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>581a9972805f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1000</td>\n",
       "      <td>8</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7f96042c9809</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                     title                                                  subtitle  claps  responses                                                    story_url  year  month  day      story_id\n",
       "0  e9ee1187182a  2021-03-01             4                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python    150          1  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1  581a9972805f\n",
       "1  fa1913854e95  2021-03-01            11  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…   1000          8  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1  7f96042c9809"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the story id from the story_url information\n",
    "df['story_id'] = df.story_url.apply(lambda x: x[:x.index('?')][-12:])\n",
    "\n",
    "# check the outcome\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db0ff548",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop columns not needed in our analysis\n",
    "#df.drop(columns=['date', 'story_url'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "260b8400",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>story_id</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>581a9972805f</td>\n",
       "      <td>Stop using numpy.random.seed() How to set random seeds f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1000</td>\n",
       "      <td>8</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7f96042c9809</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c4539c5f517b</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>7</td>\n",
       "      <td>Should You Become a Data Engineer in 2021?</td>\n",
       "      <td>Data Engineering is the new Data Science</td>\n",
       "      <td>172</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/should-you-become-a-data-...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4db57b6cce35</td>\n",
       "      <td>Should You Become a Data Engineer in 2021? Data Engineer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>720e3a4ac60c</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>7</td>\n",
       "      <td>Line of Best Fit in Linear Regression</td>\n",
       "      <td>Correlation Coefficient, Coefficient of determination, M...</td>\n",
       "      <td>415</td>\n",
       "      <td>0</td>\n",
       "      <td>https://towardsdatascience.com/line-of-best-fit-in-linea...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13658266fbc8</td>\n",
       "      <td>Line of Best Fit in Linear Regression Correlation Coeffi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>76c21e75463a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>5</td>\n",
       "      <td>What’s the Difference Between Shallow and Deep Copies in...</td>\n",
       "      <td>copy() vs deepcopy() in Python</td>\n",
       "      <td>130</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/whats-the-difference-betw...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>ceee1e061926</td>\n",
       "      <td>What’s the Difference Between Shallow and Deep Copies in...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                        title                                                     subtitle  claps  responses                                                    story_url  year  month  day      story_id                                                       corpus\n",
       "0  e9ee1187182a  2021-03-01             4                               Stop using numpy.random.seed()     How to set random seeds for individual classes in Python    150          1  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1  581a9972805f  Stop using numpy.random.seed() How to set random seeds f...\n",
       "1  fa1913854e95  2021-03-01            11     Semi-Automated Exploratory Data Analysis (EDA) in Python                 Comprehensive Data Exploration Process with…   1000          8  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1  7f96042c9809  Semi-Automated Exploratory Data Analysis (EDA) in Python...\n",
       "2  c4539c5f517b  2021-03-01             7                   Should You Become a Data Engineer in 2021?                     Data Engineering is the new Data Science    172          1  https://towardsdatascience.com/should-you-become-a-data-...  2021      3    1  4db57b6cce35  Should You Become a Data Engineer in 2021? Data Engineer...\n",
       "3  720e3a4ac60c  2021-03-01             7                        Line of Best Fit in Linear Regression  Correlation Coefficient, Coefficient of determination, M...    415          0  https://towardsdatascience.com/line-of-best-fit-in-linea...  2021      3    1  13658266fbc8  Line of Best Fit in Linear Regression Correlation Coeffi...\n",
       "4  76c21e75463a  2021-03-01             5  What’s the Difference Between Shallow and Deep Copies in...                               copy() vs deepcopy() in Python    130          1  https://towardsdatascience.com/whats-the-difference-betw...  2021      3    1  ceee1e061926  What’s the Difference Between Shallow and Deep Copies in..."
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# combine the title and subtitle columns\n",
    "df['corpus'] = df['title'] + ' ' + df['subtitle']\n",
    "\n",
    "# check the output\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b81054a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id         object\n",
       "date            object\n",
       "reading_time     int64\n",
       "title           object\n",
       "subtitle        object\n",
       "claps            int64\n",
       "responses        int64\n",
       "story_url       object\n",
       "year             int64\n",
       "month            int64\n",
       "day              int64\n",
       "story_id        object\n",
       "corpus          object\n",
       "dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the structure of the pre-processed dataframe\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "04501aca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id         12044\n",
       "date             1794\n",
       "reading_time       61\n",
       "title           51762\n",
       "subtitle        36477\n",
       "claps            1136\n",
       "responses          99\n",
       "story_url       51762\n",
       "year                5\n",
       "month              12\n",
       "day                31\n",
       "story_id        51762\n",
       "corpus          51762\n",
       "dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display the number of distinct values in each columns\n",
    "df.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f35805a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id         0\n",
       "date            0\n",
       "reading_time    0\n",
       "title           0\n",
       "subtitle        0\n",
       "claps           0\n",
       "responses       0\n",
       "story_url       0\n",
       "year            0\n",
       "month           0\n",
       "day             0\n",
       "story_id        0\n",
       "corpus          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display the number of missing values in each column\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "677c4da2",
   "metadata": {},
   "source": [
    "## Saved the Preprocessed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e4ee0b77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the preprocessed data to a file\n",
    "df.to_csv(f'data/data_interim/preprocessed_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4a2c4207",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>claps</th>\n",
       "      <th>responses</th>\n",
       "      <th>story_url</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>story_id</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9ee1187182a</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>4</td>\n",
       "      <td>Stop using numpy.random.seed()</td>\n",
       "      <td>How to set random seeds for individual classes in Python</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>https://towardsdatascience.com/stop-using-numpy-random-s...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>581a9972805f</td>\n",
       "      <td>Stop using numpy.random.seed() How to set random seeds f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fa1913854e95</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>11</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python</td>\n",
       "      <td>Comprehensive Data Exploration Process with…</td>\n",
       "      <td>1000</td>\n",
       "      <td>8</td>\n",
       "      <td>https://towardsdatascience.com/semi-automated-explorator...</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7f96042c9809</td>\n",
       "      <td>Semi-Automated Exploratory Data Analysis (EDA) in Python...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id        date  reading_time                                                     title                                                  subtitle  claps  responses                                                    story_url  year  month  day      story_id                                                       corpus\n",
       "0  e9ee1187182a  2021-03-01             4                            Stop using numpy.random.seed()  How to set random seeds for individual classes in Python    150          1  https://towardsdatascience.com/stop-using-numpy-random-s...  2021      3    1  581a9972805f  Stop using numpy.random.seed() How to set random seeds f...\n",
       "1  fa1913854e95  2021-03-01            11  Semi-Automated Exploratory Data Analysis (EDA) in Python              Comprehensive Data Exploration Process with…   1000          8  https://towardsdatascience.com/semi-automated-explorator...  2021      3    1  7f96042c9809  Semi-Automated Exploratory Data Analysis (EDA) in Python..."
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the saved file\n",
    "new_df = pd.read_csv('data/data_interim/preprocessed_data.csv')\n",
    "new_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1e41083",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}