review_dataset_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " As the original Yelp review data is pretty large, here we explore the data and extract the first 10K entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2018-07-07 22:09:11'</td>\n",
       "      <td>b'XQfwVwDr-v0ZS3_CbbE5Xw'</td>\n",
       "      <td>b\"If you decide to eat here, just be aware it ...</td>\n",
       "      <td>b'mh_-eMZ6K5RLWhZyISBhwA'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'KU_O5udG6zpxOg-VcAEodg'</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2012-01-03 15:28:18'</td>\n",
       "      <td>b'7ATYjTIgM3jUlt4UM3IypQ'</td>\n",
       "      <td>b\"I've taken a lot of spin classes over the ye...</td>\n",
       "      <td>b'OyoGAe7OKpv6SyGZT5g77Q'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'BiTunyQ73aT9WBnpR9DZGw'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302909</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2017-04-15 01:01:26'</td>\n",
       "      <td>b'WRMlifrYLjRY8q5YUUbfZA'</td>\n",
       "      <td>b'This is our neighborhood spot! The food is c...</td>\n",
       "      <td>b'xWo5LSE7Tv3gNp-rGE9qZw'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'SIdugeC_1xVChEJc8T2Blg'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302910</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302911</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2017-07-15 01:30:04'</td>\n",
       "      <td>b'wWLic8cT6apGZWK8wWWSQQ'</td>\n",
       "      <td>b'Disappointing.  (No stars deducted for \"just...</td>\n",
       "      <td>b'jy-58SJJqv_U0Mt7XJJ5Sg'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'xv8gwdEiLMePeODrLA92LA'</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302912</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302913</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2012-04-14 03:21:21'</td>\n",
       "      <td>b'HA-X-YkTt9K7c-fkhE-g4Q'</td>\n",
       "      <td>b\"Best Thai in Doylestown. We go here at least...</td>\n",
       "      <td>b'LNEyUWtTpvvMhp9KssJPIg'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'9KjzQVmRCZAhR7uZ-e_8CQ'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>302914 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                    date                business_id  \\\n",
       "0          NaN                     NaN                        NaN   \n",
       "1          0.0  b'2018-07-07 22:09:11'  b'XQfwVwDr-v0ZS3_CbbE5Xw'   \n",
       "2          NaN                     NaN                        NaN   \n",
       "3          1.0  b'2012-01-03 15:28:18'  b'7ATYjTIgM3jUlt4UM3IypQ'   \n",
       "4          NaN                     NaN                        NaN   \n",
       "...        ...                     ...                        ...   \n",
       "302909     0.0  b'2017-04-15 01:01:26'  b'WRMlifrYLjRY8q5YUUbfZA'   \n",
       "302910     NaN                     NaN                        NaN   \n",
       "302911     1.0  b'2017-07-15 01:30:04'  b'wWLic8cT6apGZWK8wWWSQQ'   \n",
       "302912     NaN                     NaN                        NaN   \n",
       "302913     0.0  b'2012-04-14 03:21:21'  b'HA-X-YkTt9K7c-fkhE-g4Q'   \n",
       "\n",
       "                                                     text  \\\n",
       "0                                                     NaN   \n",
       "1       b\"If you decide to eat here, just be aware it ...   \n",
       "2                                                     NaN   \n",
       "3       b\"I've taken a lot of spin classes over the ye...   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "302909  b'This is our neighborhood spot! The food is c...   \n",
       "302910                                                NaN   \n",
       "302911  b'Disappointing.  (No stars deducted for \"just...   \n",
       "302912                                                NaN   \n",
       "302913  b\"Best Thai in Doylestown. We go here at least...   \n",
       "\n",
       "                          user_id  funny  cool                  review_id  \\\n",
       "0                             NaN    NaN   NaN                        NaN   \n",
       "1       b'mh_-eMZ6K5RLWhZyISBhwA'    0.0   0.0  b'KU_O5udG6zpxOg-VcAEodg'   \n",
       "2                             NaN    NaN   NaN                        NaN   \n",
       "3       b'OyoGAe7OKpv6SyGZT5g77Q'    0.0   1.0  b'BiTunyQ73aT9WBnpR9DZGw'   \n",
       "4                             NaN    NaN   NaN                        NaN   \n",
       "...                           ...    ...   ...                        ...   \n",
       "302909  b'xWo5LSE7Tv3gNp-rGE9qZw'    0.0   0.0  b'SIdugeC_1xVChEJc8T2Blg'   \n",
       "302910                        NaN    NaN   NaN                        NaN   \n",
       "302911  b'jy-58SJJqv_U0Mt7XJJ5Sg'    0.0   0.0  b'xv8gwdEiLMePeODrLA92LA'   \n",
       "302912                        NaN    NaN   NaN                        NaN   \n",
       "302913  b'LNEyUWtTpvvMhp9KssJPIg'    0.0   1.0  b'9KjzQVmRCZAhR7uZ-e_8CQ'   \n",
       "\n",
       "        stars  \n",
       "0         NaN  \n",
       "1         3.0  \n",
       "2         NaN  \n",
       "3         5.0  \n",
       "4         NaN  \n",
       "...       ...  \n",
       "302909    5.0  \n",
       "302910    NaN  \n",
       "302911    2.0  \n",
       "302912    NaN  \n",
       "302913    5.0  \n",
       "\n",
       "[302914 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import data from pandas\n",
    "review = pd.read_csv('./yelp_academic_dataset_review_first_100K.csv')\n",
    "review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['useful', 'date', 'business_id', 'text', 'user_id', 'funny', 'cool',\n",
       "       'review_id', 'stars'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(151457, 9)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review.dropna(inplace=True,ignore_index=True)\n",
    "review.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "review.to_csv('./yelp_academic_dataset_review_first_100K.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2018-07-07 22:09:11'</td>\n",
       "      <td>b'XQfwVwDr-v0ZS3_CbbE5Xw'</td>\n",
       "      <td>b\"If you decide to eat here, just be aware it ...</td>\n",
       "      <td>b'mh_-eMZ6K5RLWhZyISBhwA'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'KU_O5udG6zpxOg-VcAEodg'</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2012-01-03 15:28:18'</td>\n",
       "      <td>b'7ATYjTIgM3jUlt4UM3IypQ'</td>\n",
       "      <td>b\"I've taken a lot of spin classes over the ye...</td>\n",
       "      <td>b'OyoGAe7OKpv6SyGZT5g77Q'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'BiTunyQ73aT9WBnpR9DZGw'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2014-02-05 20:30:30'</td>\n",
       "      <td>b'YjUWPpI6HXG530lwP-fb2A'</td>\n",
       "      <td>b'Family diner. Had the buffet. Eclectic assor...</td>\n",
       "      <td>b'8g_iMtfSiwikVnbP2etR0A'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'saUsX_uimxRlCVr67Z4Jig'</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2015-01-04 00:01:03'</td>\n",
       "      <td>b'kxX2SOes4o-D3ZQBkiMRfA'</td>\n",
       "      <td>b\"Wow!  Yummy, different,  delicious.   Our fa...</td>\n",
       "      <td>b'_7bHUi9Uuf5__HHc_Q8guQ'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'AqPFMleE6RsU23_auESxiA'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2017-01-14 20:54:15'</td>\n",
       "      <td>b'e4Vwtrqf-wpJfwesgvdgxQ'</td>\n",
       "      <td>b\"Cute interior and owner (?) gave us tour of ...</td>\n",
       "      <td>b'bcjbaE6dDog4jkNY91ncLQ'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'Sx8TMOWLNuJBWer-0pcmoA'</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151452</th>\n",
       "      <td>2.0</td>\n",
       "      <td>b'2016-07-30 17:00:18'</td>\n",
       "      <td>b'vyKKuYDTofiV1oJDN-g8Lw'</td>\n",
       "      <td>b\"Sharlei is the best!!! Every time I get waxe...</td>\n",
       "      <td>b'5HKAT4-7yHO-kZSXKPSGWQ'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'bLZAPS80ykXHtbseCfnydQ'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151453</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2016-08-20 18:05:44'</td>\n",
       "      <td>b'5xokROlB8Zo5T-_qbHzsrQ'</td>\n",
       "      <td>b\"Just moved to the Tampa area, and had some t...</td>\n",
       "      <td>b'xWYEop8hno4KZqcOXyWnaA'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'HOSQsA5Ont5WaE4Uk1Sgvw'</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151454</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2017-04-15 01:01:26'</td>\n",
       "      <td>b'WRMlifrYLjRY8q5YUUbfZA'</td>\n",
       "      <td>b'This is our neighborhood spot! The food is c...</td>\n",
       "      <td>b'xWo5LSE7Tv3gNp-rGE9qZw'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'SIdugeC_1xVChEJc8T2Blg'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151455</th>\n",
       "      <td>1.0</td>\n",
       "      <td>b'2017-07-15 01:30:04'</td>\n",
       "      <td>b'wWLic8cT6apGZWK8wWWSQQ'</td>\n",
       "      <td>b'Disappointing.  (No stars deducted for \"just...</td>\n",
       "      <td>b'jy-58SJJqv_U0Mt7XJJ5Sg'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>b'xv8gwdEiLMePeODrLA92LA'</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151456</th>\n",
       "      <td>0.0</td>\n",
       "      <td>b'2012-04-14 03:21:21'</td>\n",
       "      <td>b'HA-X-YkTt9K7c-fkhE-g4Q'</td>\n",
       "      <td>b\"Best Thai in Doylestown. We go here at least...</td>\n",
       "      <td>b'LNEyUWtTpvvMhp9KssJPIg'</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>b'9KjzQVmRCZAhR7uZ-e_8CQ'</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151457 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                    date                business_id  \\\n",
       "0          0.0  b'2018-07-07 22:09:11'  b'XQfwVwDr-v0ZS3_CbbE5Xw'   \n",
       "1          1.0  b'2012-01-03 15:28:18'  b'7ATYjTIgM3jUlt4UM3IypQ'   \n",
       "2          0.0  b'2014-02-05 20:30:30'  b'YjUWPpI6HXG530lwP-fb2A'   \n",
       "3          1.0  b'2015-01-04 00:01:03'  b'kxX2SOes4o-D3ZQBkiMRfA'   \n",
       "4          1.0  b'2017-01-14 20:54:15'  b'e4Vwtrqf-wpJfwesgvdgxQ'   \n",
       "...        ...                     ...                        ...   \n",
       "151452     2.0  b'2016-07-30 17:00:18'  b'vyKKuYDTofiV1oJDN-g8Lw'   \n",
       "151453     1.0  b'2016-08-20 18:05:44'  b'5xokROlB8Zo5T-_qbHzsrQ'   \n",
       "151454     0.0  b'2017-04-15 01:01:26'  b'WRMlifrYLjRY8q5YUUbfZA'   \n",
       "151455     1.0  b'2017-07-15 01:30:04'  b'wWLic8cT6apGZWK8wWWSQQ'   \n",
       "151456     0.0  b'2012-04-14 03:21:21'  b'HA-X-YkTt9K7c-fkhE-g4Q'   \n",
       "\n",
       "                                                     text  \\\n",
       "0       b\"If you decide to eat here, just be aware it ...   \n",
       "1       b\"I've taken a lot of spin classes over the ye...   \n",
       "2       b'Family diner. Had the buffet. Eclectic assor...   \n",
       "3       b\"Wow!  Yummy, different,  delicious.   Our fa...   \n",
       "4       b\"Cute interior and owner (?) gave us tour of ...   \n",
       "...                                                   ...   \n",
       "151452  b\"Sharlei is the best!!! Every time I get waxe...   \n",
       "151453  b\"Just moved to the Tampa area, and had some t...   \n",
       "151454  b'This is our neighborhood spot! The food is c...   \n",
       "151455  b'Disappointing.  (No stars deducted for \"just...   \n",
       "151456  b\"Best Thai in Doylestown. We go here at least...   \n",
       "\n",
       "                          user_id  funny  cool                  review_id  \\\n",
       "0       b'mh_-eMZ6K5RLWhZyISBhwA'    0.0   0.0  b'KU_O5udG6zpxOg-VcAEodg'   \n",
       "1       b'OyoGAe7OKpv6SyGZT5g77Q'    0.0   1.0  b'BiTunyQ73aT9WBnpR9DZGw'   \n",
       "2       b'8g_iMtfSiwikVnbP2etR0A'    0.0   0.0  b'saUsX_uimxRlCVr67Z4Jig'   \n",
       "3       b'_7bHUi9Uuf5__HHc_Q8guQ'    0.0   1.0  b'AqPFMleE6RsU23_auESxiA'   \n",
       "4       b'bcjbaE6dDog4jkNY91ncLQ'    0.0   1.0  b'Sx8TMOWLNuJBWer-0pcmoA'   \n",
       "...                           ...    ...   ...                        ...   \n",
       "151452  b'5HKAT4-7yHO-kZSXKPSGWQ'    0.0   0.0  b'bLZAPS80ykXHtbseCfnydQ'   \n",
       "151453  b'xWYEop8hno4KZqcOXyWnaA'    0.0   0.0  b'HOSQsA5Ont5WaE4Uk1Sgvw'   \n",
       "151454  b'xWo5LSE7Tv3gNp-rGE9qZw'    0.0   0.0  b'SIdugeC_1xVChEJc8T2Blg'   \n",
       "151455  b'jy-58SJJqv_U0Mt7XJJ5Sg'    0.0   0.0  b'xv8gwdEiLMePeODrLA92LA'   \n",
       "151456  b'LNEyUWtTpvvMhp9KssJPIg'    0.0   1.0  b'9KjzQVmRCZAhR7uZ-e_8CQ'   \n",
       "\n",
       "        stars  \n",
       "0         3.0  \n",
       "1         5.0  \n",
       "2         3.0  \n",
       "3         5.0  \n",
       "4         4.0  \n",
       "...       ...  \n",
       "151452    5.0  \n",
       "151453    4.0  \n",
       "151454    5.0  \n",
       "151455    2.0  \n",
       "151456    5.0  \n",
       "\n",
       "[151457 rows x 9 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review = pd.read_csv('./yelp_academic_dataset_review_first_100K.csv')\n",
    "review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b\"If you decide to eat here, just be aware it is going to take about 2 hours from beginning to end. We have tried it multiple times, because I want to like it! I have been to it's other locations in NJ and never had a bad experience. \\n\\nThe food is good, but it takes a very long time to come out. The waitstaff is very young, but usually pleasant. We have just had too many experiences where we spent way too long waiting. We usually opt for another diner or restaurant on the weekends, in order to be done quicker.\"\n"
     ]
    }
   ],
   "source": [
    "print(review.loc[0,'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Came in for my 5-6 month prophy and saw Kara - who is a new hygenist and TERRIFIC.  She took text-book perfect bitewings (digital, baby!) and Dr. D is great as always. Plus I love LeeAnn (sp??) - the office manager.  Such a well-run place!\\n\\nPlus I got strawberry prophy paste.\\n\\nWheeeeeee!'\n"
     ]
    }
   ],
   "source": [
    "print(review.loc[99999,'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"b'Came in for my 5-6 month prophy and saw Kara - who is a new hygenist and TERRIFIC.  She took text-book perfect bitewings (digital, baby!) and Dr. D is great as always. Plus I love LeeAnn (sp??) - the office manager.  Such a well-run place!\\\\n\\\\nPlus I got strawberry prophy paste.\\\\n\\\\nWheeeeeee!'\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review.loc[99999,'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean 'text' column\n",
    "review['text'] = review['text'].apply(lambda x: x.removeprefix('b'))\n",
    "review['text'] = review['text'].apply(lambda x: x.removeprefix('\\''))\n",
    "review['text'] = review['text'].apply(lambda x: x.removeprefix('\"'))\n",
    "review['text'] = review['text'].apply(lambda x: x.removesuffix('\\''))\n",
    "review['text'] = review['text'].apply(lambda x: x.removesuffix('\"'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean other str columns\n",
    "for col in ['date','business_id','user_id','review_id']:\n",
    "    review[col] = review[col].apply(lambda x: x.removeprefix('b'))\n",
    "    review[col] = review[col].apply(lambda x: x.removeprefix('\\''))\n",
    "    review[col] = review[col].apply(lambda x: x.removeprefix('\"'))\n",
    "    review[col] = review[col].apply(lambda x: x.removesuffix('\\''))\n",
    "    review[col] = review[col].apply(lambda x: x.removesuffix('\"'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2018-07-07 22:09:11</td>\n",
       "      <td>XQfwVwDr-v0ZS3_CbbE5Xw</td>\n",
       "      <td>If you decide to eat here, just be aware it is...</td>\n",
       "      <td>mh_-eMZ6K5RLWhZyISBhwA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>KU_O5udG6zpxOg-VcAEodg</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-01-03 15:28:18</td>\n",
       "      <td>7ATYjTIgM3jUlt4UM3IypQ</td>\n",
       "      <td>I've taken a lot of spin classes over the year...</td>\n",
       "      <td>OyoGAe7OKpv6SyGZT5g77Q</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>BiTunyQ73aT9WBnpR9DZGw</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-02-05 20:30:30</td>\n",
       "      <td>YjUWPpI6HXG530lwP-fb2A</td>\n",
       "      <td>Family diner. Had the buffet. Eclectic assortm...</td>\n",
       "      <td>8g_iMtfSiwikVnbP2etR0A</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>saUsX_uimxRlCVr67Z4Jig</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-01-04 00:01:03</td>\n",
       "      <td>kxX2SOes4o-D3ZQBkiMRfA</td>\n",
       "      <td>Wow!  Yummy, different,  delicious.   Our favo...</td>\n",
       "      <td>_7bHUi9Uuf5__HHc_Q8guQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AqPFMleE6RsU23_auESxiA</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-01-14 20:54:15</td>\n",
       "      <td>e4Vwtrqf-wpJfwesgvdgxQ</td>\n",
       "      <td>Cute interior and owner (?) gave us tour of up...</td>\n",
       "      <td>bcjbaE6dDog4jkNY91ncLQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Sx8TMOWLNuJBWer-0pcmoA</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151452</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2016-07-30 17:00:18</td>\n",
       "      <td>vyKKuYDTofiV1oJDN-g8Lw</td>\n",
       "      <td>Sharlei is the best!!! Every time I get waxed ...</td>\n",
       "      <td>5HKAT4-7yHO-kZSXKPSGWQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>bLZAPS80ykXHtbseCfnydQ</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151453</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-08-20 18:05:44</td>\n",
       "      <td>5xokROlB8Zo5T-_qbHzsrQ</td>\n",
       "      <td>Just moved to the Tampa area, and had some tim...</td>\n",
       "      <td>xWYEop8hno4KZqcOXyWnaA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HOSQsA5Ont5WaE4Uk1Sgvw</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151454</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-04-15 01:01:26</td>\n",
       "      <td>WRMlifrYLjRY8q5YUUbfZA</td>\n",
       "      <td>This is our neighborhood spot! The food is con...</td>\n",
       "      <td>xWo5LSE7Tv3gNp-rGE9qZw</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>SIdugeC_1xVChEJc8T2Blg</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151455</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-07-15 01:30:04</td>\n",
       "      <td>wWLic8cT6apGZWK8wWWSQQ</td>\n",
       "      <td>Disappointing.  (No stars deducted for \"just o...</td>\n",
       "      <td>jy-58SJJqv_U0Mt7XJJ5Sg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>xv8gwdEiLMePeODrLA92LA</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151456</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-04-14 03:21:21</td>\n",
       "      <td>HA-X-YkTt9K7c-fkhE-g4Q</td>\n",
       "      <td>Best Thai in Doylestown. We go here at least o...</td>\n",
       "      <td>LNEyUWtTpvvMhp9KssJPIg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9KjzQVmRCZAhR7uZ-e_8CQ</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151457 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                 date             business_id  \\\n",
       "0          0.0  2018-07-07 22:09:11  XQfwVwDr-v0ZS3_CbbE5Xw   \n",
       "1          1.0  2012-01-03 15:28:18  7ATYjTIgM3jUlt4UM3IypQ   \n",
       "2          0.0  2014-02-05 20:30:30  YjUWPpI6HXG530lwP-fb2A   \n",
       "3          1.0  2015-01-04 00:01:03  kxX2SOes4o-D3ZQBkiMRfA   \n",
       "4          1.0  2017-01-14 20:54:15  e4Vwtrqf-wpJfwesgvdgxQ   \n",
       "...        ...                  ...                     ...   \n",
       "151452     2.0  2016-07-30 17:00:18  vyKKuYDTofiV1oJDN-g8Lw   \n",
       "151453     1.0  2016-08-20 18:05:44  5xokROlB8Zo5T-_qbHzsrQ   \n",
       "151454     0.0  2017-04-15 01:01:26  WRMlifrYLjRY8q5YUUbfZA   \n",
       "151455     1.0  2017-07-15 01:30:04  wWLic8cT6apGZWK8wWWSQQ   \n",
       "151456     0.0  2012-04-14 03:21:21  HA-X-YkTt9K7c-fkhE-g4Q   \n",
       "\n",
       "                                                     text  \\\n",
       "0       If you decide to eat here, just be aware it is...   \n",
       "1       I've taken a lot of spin classes over the year...   \n",
       "2       Family diner. Had the buffet. Eclectic assortm...   \n",
       "3       Wow!  Yummy, different,  delicious.   Our favo...   \n",
       "4       Cute interior and owner (?) gave us tour of up...   \n",
       "...                                                   ...   \n",
       "151452  Sharlei is the best!!! Every time I get waxed ...   \n",
       "151453  Just moved to the Tampa area, and had some tim...   \n",
       "151454  This is our neighborhood spot! The food is con...   \n",
       "151455  Disappointing.  (No stars deducted for \"just o...   \n",
       "151456  Best Thai in Doylestown. We go here at least o...   \n",
       "\n",
       "                       user_id  funny  cool               review_id  stars  \n",
       "0       mh_-eMZ6K5RLWhZyISBhwA    0.0   0.0  KU_O5udG6zpxOg-VcAEodg    3.0  \n",
       "1       OyoGAe7OKpv6SyGZT5g77Q    0.0   1.0  BiTunyQ73aT9WBnpR9DZGw    5.0  \n",
       "2       8g_iMtfSiwikVnbP2etR0A    0.0   0.0  saUsX_uimxRlCVr67Z4Jig    3.0  \n",
       "3       _7bHUi9Uuf5__HHc_Q8guQ    0.0   1.0  AqPFMleE6RsU23_auESxiA    5.0  \n",
       "4       bcjbaE6dDog4jkNY91ncLQ    0.0   1.0  Sx8TMOWLNuJBWer-0pcmoA    4.0  \n",
       "...                        ...    ...   ...                     ...    ...  \n",
       "151452  5HKAT4-7yHO-kZSXKPSGWQ    0.0   0.0  bLZAPS80ykXHtbseCfnydQ    5.0  \n",
       "151453  xWYEop8hno4KZqcOXyWnaA    0.0   0.0  HOSQsA5Ont5WaE4Uk1Sgvw    4.0  \n",
       "151454  xWo5LSE7Tv3gNp-rGE9qZw    0.0   0.0  SIdugeC_1xVChEJc8T2Blg    5.0  \n",
       "151455  jy-58SJJqv_U0Mt7XJJ5Sg    0.0   0.0  xv8gwdEiLMePeODrLA92LA    2.0  \n",
       "151456  LNEyUWtTpvvMhp9KssJPIg    0.0   1.0  9KjzQVmRCZAhR7uZ-e_8CQ    5.0  \n",
       "\n",
       "[151457 rows x 9 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For stars in review, if stars>=4, we say its sentiment is positive, if stars<=2, we say its sentiment is negative.\n",
    "And we can create the 'sentiment' column for the review dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2018-07-07 22:09:11</td>\n",
       "      <td>XQfwVwDr-v0ZS3_CbbE5Xw</td>\n",
       "      <td>If you decide to eat here, just be aware it is...</td>\n",
       "      <td>mh_-eMZ6K5RLWhZyISBhwA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>KU_O5udG6zpxOg-VcAEodg</td>\n",
       "      <td>3.0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-01-03 15:28:18</td>\n",
       "      <td>7ATYjTIgM3jUlt4UM3IypQ</td>\n",
       "      <td>I've taken a lot of spin classes over the year...</td>\n",
       "      <td>OyoGAe7OKpv6SyGZT5g77Q</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>BiTunyQ73aT9WBnpR9DZGw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-02-05 20:30:30</td>\n",
       "      <td>YjUWPpI6HXG530lwP-fb2A</td>\n",
       "      <td>Family diner. Had the buffet. Eclectic assortm...</td>\n",
       "      <td>8g_iMtfSiwikVnbP2etR0A</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>saUsX_uimxRlCVr67Z4Jig</td>\n",
       "      <td>3.0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-01-04 00:01:03</td>\n",
       "      <td>kxX2SOes4o-D3ZQBkiMRfA</td>\n",
       "      <td>Wow!  Yummy, different,  delicious.   Our favo...</td>\n",
       "      <td>_7bHUi9Uuf5__HHc_Q8guQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AqPFMleE6RsU23_auESxiA</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-01-14 20:54:15</td>\n",
       "      <td>e4Vwtrqf-wpJfwesgvdgxQ</td>\n",
       "      <td>Cute interior and owner (?) gave us tour of up...</td>\n",
       "      <td>bcjbaE6dDog4jkNY91ncLQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Sx8TMOWLNuJBWer-0pcmoA</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151452</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2016-07-30 17:00:18</td>\n",
       "      <td>vyKKuYDTofiV1oJDN-g8Lw</td>\n",
       "      <td>Sharlei is the best!!! Every time I get waxed ...</td>\n",
       "      <td>5HKAT4-7yHO-kZSXKPSGWQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>bLZAPS80ykXHtbseCfnydQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151453</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-08-20 18:05:44</td>\n",
       "      <td>5xokROlB8Zo5T-_qbHzsrQ</td>\n",
       "      <td>Just moved to the Tampa area, and had some tim...</td>\n",
       "      <td>xWYEop8hno4KZqcOXyWnaA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HOSQsA5Ont5WaE4Uk1Sgvw</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151454</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-04-15 01:01:26</td>\n",
       "      <td>WRMlifrYLjRY8q5YUUbfZA</td>\n",
       "      <td>This is our neighborhood spot! The food is con...</td>\n",
       "      <td>xWo5LSE7Tv3gNp-rGE9qZw</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>SIdugeC_1xVChEJc8T2Blg</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151455</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-07-15 01:30:04</td>\n",
       "      <td>wWLic8cT6apGZWK8wWWSQQ</td>\n",
       "      <td>Disappointing.  (No stars deducted for \"just o...</td>\n",
       "      <td>jy-58SJJqv_U0Mt7XJJ5Sg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>xv8gwdEiLMePeODrLA92LA</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151456</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-04-14 03:21:21</td>\n",
       "      <td>HA-X-YkTt9K7c-fkhE-g4Q</td>\n",
       "      <td>Best Thai in Doylestown. We go here at least o...</td>\n",
       "      <td>LNEyUWtTpvvMhp9KssJPIg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9KjzQVmRCZAhR7uZ-e_8CQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151457 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                 date             business_id  \\\n",
       "0          0.0  2018-07-07 22:09:11  XQfwVwDr-v0ZS3_CbbE5Xw   \n",
       "1          1.0  2012-01-03 15:28:18  7ATYjTIgM3jUlt4UM3IypQ   \n",
       "2          0.0  2014-02-05 20:30:30  YjUWPpI6HXG530lwP-fb2A   \n",
       "3          1.0  2015-01-04 00:01:03  kxX2SOes4o-D3ZQBkiMRfA   \n",
       "4          1.0  2017-01-14 20:54:15  e4Vwtrqf-wpJfwesgvdgxQ   \n",
       "...        ...                  ...                     ...   \n",
       "151452     2.0  2016-07-30 17:00:18  vyKKuYDTofiV1oJDN-g8Lw   \n",
       "151453     1.0  2016-08-20 18:05:44  5xokROlB8Zo5T-_qbHzsrQ   \n",
       "151454     0.0  2017-04-15 01:01:26  WRMlifrYLjRY8q5YUUbfZA   \n",
       "151455     1.0  2017-07-15 01:30:04  wWLic8cT6apGZWK8wWWSQQ   \n",
       "151456     0.0  2012-04-14 03:21:21  HA-X-YkTt9K7c-fkhE-g4Q   \n",
       "\n",
       "                                                     text  \\\n",
       "0       If you decide to eat here, just be aware it is...   \n",
       "1       I've taken a lot of spin classes over the year...   \n",
       "2       Family diner. Had the buffet. Eclectic assortm...   \n",
       "3       Wow!  Yummy, different,  delicious.   Our favo...   \n",
       "4       Cute interior and owner (?) gave us tour of up...   \n",
       "...                                                   ...   \n",
       "151452  Sharlei is the best!!! Every time I get waxed ...   \n",
       "151453  Just moved to the Tampa area, and had some tim...   \n",
       "151454  This is our neighborhood spot! The food is con...   \n",
       "151455  Disappointing.  (No stars deducted for \"just o...   \n",
       "151456  Best Thai in Doylestown. We go here at least o...   \n",
       "\n",
       "                       user_id  funny  cool               review_id  stars  \\\n",
       "0       mh_-eMZ6K5RLWhZyISBhwA    0.0   0.0  KU_O5udG6zpxOg-VcAEodg    3.0   \n",
       "1       OyoGAe7OKpv6SyGZT5g77Q    0.0   1.0  BiTunyQ73aT9WBnpR9DZGw    5.0   \n",
       "2       8g_iMtfSiwikVnbP2etR0A    0.0   0.0  saUsX_uimxRlCVr67Z4Jig    3.0   \n",
       "3       _7bHUi9Uuf5__HHc_Q8guQ    0.0   1.0  AqPFMleE6RsU23_auESxiA    5.0   \n",
       "4       bcjbaE6dDog4jkNY91ncLQ    0.0   1.0  Sx8TMOWLNuJBWer-0pcmoA    4.0   \n",
       "...                        ...    ...   ...                     ...    ...   \n",
       "151452  5HKAT4-7yHO-kZSXKPSGWQ    0.0   0.0  bLZAPS80ykXHtbseCfnydQ    5.0   \n",
       "151453  xWYEop8hno4KZqcOXyWnaA    0.0   0.0  HOSQsA5Ont5WaE4Uk1Sgvw    4.0   \n",
       "151454  xWo5LSE7Tv3gNp-rGE9qZw    0.0   0.0  SIdugeC_1xVChEJc8T2Blg    5.0   \n",
       "151455  jy-58SJJqv_U0Mt7XJJ5Sg    0.0   0.0  xv8gwdEiLMePeODrLA92LA    2.0   \n",
       "151456  LNEyUWtTpvvMhp9KssJPIg    0.0   1.0  9KjzQVmRCZAhR7uZ-e_8CQ    5.0   \n",
       "\n",
       "       sentiment  \n",
       "0           None  \n",
       "1              1  \n",
       "2           None  \n",
       "3              1  \n",
       "4              1  \n",
       "...          ...  \n",
       "151452         1  \n",
       "151453         1  \n",
       "151454         1  \n",
       "151455         0  \n",
       "151456         1  \n",
       "\n",
       "[151457 rows x 10 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review['sentiment'] = None\n",
    "review.loc[review['stars']>=3.9, 'sentiment'] = 1\n",
    "review.loc[review['stars']<=2.1, 'sentiment'] = 0\n",
    "review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "review.to_csv('./yelp_academic_dataset_review_first_151K.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((151457, 10), (134392, 10))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review.shape, review.dropna().shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We remove stars=3.0 as this is neither positive or negative sentiment. And we save the remaining part of the review."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-01-03 15:28:18</td>\n",
       "      <td>7ATYjTIgM3jUlt4UM3IypQ</td>\n",
       "      <td>I've taken a lot of spin classes over the year...</td>\n",
       "      <td>OyoGAe7OKpv6SyGZT5g77Q</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>BiTunyQ73aT9WBnpR9DZGw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-01-04 00:01:03</td>\n",
       "      <td>kxX2SOes4o-D3ZQBkiMRfA</td>\n",
       "      <td>Wow!  Yummy, different,  delicious.   Our favo...</td>\n",
       "      <td>_7bHUi9Uuf5__HHc_Q8guQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AqPFMleE6RsU23_auESxiA</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-01-14 20:54:15</td>\n",
       "      <td>e4Vwtrqf-wpJfwesgvdgxQ</td>\n",
       "      <td>Cute interior and owner (?) gave us tour of up...</td>\n",
       "      <td>bcjbaE6dDog4jkNY91ncLQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Sx8TMOWLNuJBWer-0pcmoA</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-09-23 23:10:31</td>\n",
       "      <td>04UD14gamNjLY0IDYVhHJg</td>\n",
       "      <td>I am a long term frequent customer of this est...</td>\n",
       "      <td>eUta8W_HdHMXPzLBBZhL1A</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>JrIxlS1TzJ-iCu79ul40cQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-01-03 23:21:18</td>\n",
       "      <td>gmjsEdUsKpj9Xxu6pdjH0g</td>\n",
       "      <td>Loved this tour! I grabbed a groupon and the p...</td>\n",
       "      <td>r3zeYsv1XFBRA4dJpL78cw</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6AxgBCNX_PNTOxmbRSwcKQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134387</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2016-07-30 17:00:18</td>\n",
       "      <td>vyKKuYDTofiV1oJDN-g8Lw</td>\n",
       "      <td>Sharlei is the best!!! Every time I get waxed ...</td>\n",
       "      <td>5HKAT4-7yHO-kZSXKPSGWQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>bLZAPS80ykXHtbseCfnydQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134388</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-08-20 18:05:44</td>\n",
       "      <td>5xokROlB8Zo5T-_qbHzsrQ</td>\n",
       "      <td>Just moved to the Tampa area, and had some tim...</td>\n",
       "      <td>xWYEop8hno4KZqcOXyWnaA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HOSQsA5Ont5WaE4Uk1Sgvw</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134389</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-04-15 01:01:26</td>\n",
       "      <td>WRMlifrYLjRY8q5YUUbfZA</td>\n",
       "      <td>This is our neighborhood spot! The food is con...</td>\n",
       "      <td>xWo5LSE7Tv3gNp-rGE9qZw</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>SIdugeC_1xVChEJc8T2Blg</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134390</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-07-15 01:30:04</td>\n",
       "      <td>wWLic8cT6apGZWK8wWWSQQ</td>\n",
       "      <td>Disappointing.  (No stars deducted for \"just o...</td>\n",
       "      <td>jy-58SJJqv_U0Mt7XJJ5Sg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>xv8gwdEiLMePeODrLA92LA</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134391</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-04-14 03:21:21</td>\n",
       "      <td>HA-X-YkTt9K7c-fkhE-g4Q</td>\n",
       "      <td>Best Thai in Doylestown. We go here at least o...</td>\n",
       "      <td>LNEyUWtTpvvMhp9KssJPIg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9KjzQVmRCZAhR7uZ-e_8CQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>134392 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                 date             business_id  \\\n",
       "0          1.0  2012-01-03 15:28:18  7ATYjTIgM3jUlt4UM3IypQ   \n",
       "1          1.0  2015-01-04 00:01:03  kxX2SOes4o-D3ZQBkiMRfA   \n",
       "2          1.0  2017-01-14 20:54:15  e4Vwtrqf-wpJfwesgvdgxQ   \n",
       "3          1.0  2015-09-23 23:10:31  04UD14gamNjLY0IDYVhHJg   \n",
       "4          0.0  2015-01-03 23:21:18  gmjsEdUsKpj9Xxu6pdjH0g   \n",
       "...        ...                  ...                     ...   \n",
       "134387     2.0  2016-07-30 17:00:18  vyKKuYDTofiV1oJDN-g8Lw   \n",
       "134388     1.0  2016-08-20 18:05:44  5xokROlB8Zo5T-_qbHzsrQ   \n",
       "134389     0.0  2017-04-15 01:01:26  WRMlifrYLjRY8q5YUUbfZA   \n",
       "134390     1.0  2017-07-15 01:30:04  wWLic8cT6apGZWK8wWWSQQ   \n",
       "134391     0.0  2012-04-14 03:21:21  HA-X-YkTt9K7c-fkhE-g4Q   \n",
       "\n",
       "                                                     text  \\\n",
       "0       I've taken a lot of spin classes over the year...   \n",
       "1       Wow!  Yummy, different,  delicious.   Our favo...   \n",
       "2       Cute interior and owner (?) gave us tour of up...   \n",
       "3       I am a long term frequent customer of this est...   \n",
       "4       Loved this tour! I grabbed a groupon and the p...   \n",
       "...                                                   ...   \n",
       "134387  Sharlei is the best!!! Every time I get waxed ...   \n",
       "134388  Just moved to the Tampa area, and had some tim...   \n",
       "134389  This is our neighborhood spot! The food is con...   \n",
       "134390  Disappointing.  (No stars deducted for \"just o...   \n",
       "134391  Best Thai in Doylestown. We go here at least o...   \n",
       "\n",
       "                       user_id  funny  cool               review_id  stars  \\\n",
       "0       OyoGAe7OKpv6SyGZT5g77Q    0.0   1.0  BiTunyQ73aT9WBnpR9DZGw    5.0   \n",
       "1       _7bHUi9Uuf5__HHc_Q8guQ    0.0   1.0  AqPFMleE6RsU23_auESxiA    5.0   \n",
       "2       bcjbaE6dDog4jkNY91ncLQ    0.0   1.0  Sx8TMOWLNuJBWer-0pcmoA    4.0   \n",
       "3       eUta8W_HdHMXPzLBBZhL1A    2.0   1.0  JrIxlS1TzJ-iCu79ul40cQ    1.0   \n",
       "4       r3zeYsv1XFBRA4dJpL78cw    2.0   0.0  6AxgBCNX_PNTOxmbRSwcKQ    5.0   \n",
       "...                        ...    ...   ...                     ...    ...   \n",
       "134387  5HKAT4-7yHO-kZSXKPSGWQ    0.0   0.0  bLZAPS80ykXHtbseCfnydQ    5.0   \n",
       "134388  xWYEop8hno4KZqcOXyWnaA    0.0   0.0  HOSQsA5Ont5WaE4Uk1Sgvw    4.0   \n",
       "134389  xWo5LSE7Tv3gNp-rGE9qZw    0.0   0.0  SIdugeC_1xVChEJc8T2Blg    5.0   \n",
       "134390  jy-58SJJqv_U0Mt7XJJ5Sg    0.0   0.0  xv8gwdEiLMePeODrLA92LA    2.0   \n",
       "134391  LNEyUWtTpvvMhp9KssJPIg    0.0   1.0  9KjzQVmRCZAhR7uZ-e_8CQ    5.0   \n",
       "\n",
       "       sentiment  \n",
       "0              1  \n",
       "1              1  \n",
       "2              1  \n",
       "3              0  \n",
       "4              1  \n",
       "...          ...  \n",
       "134387         1  \n",
       "134388         1  \n",
       "134389         1  \n",
       "134390         0  \n",
       "134391         1  \n",
       "\n",
       "[134392 rows x 10 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_valid_sentiment = review.dropna(ignore_index=True)\n",
    "review_valid_sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We keep the first 130K entries of sentiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>useful</th>\n",
       "      <th>date</th>\n",
       "      <th>business_id</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-01-03 15:28:18</td>\n",
       "      <td>7ATYjTIgM3jUlt4UM3IypQ</td>\n",
       "      <td>I've taken a lot of spin classes over the year...</td>\n",
       "      <td>OyoGAe7OKpv6SyGZT5g77Q</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>BiTunyQ73aT9WBnpR9DZGw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-01-04 00:01:03</td>\n",
       "      <td>kxX2SOes4o-D3ZQBkiMRfA</td>\n",
       "      <td>Wow!  Yummy, different,  delicious.   Our favo...</td>\n",
       "      <td>_7bHUi9Uuf5__HHc_Q8guQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AqPFMleE6RsU23_auESxiA</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-01-14 20:54:15</td>\n",
       "      <td>e4Vwtrqf-wpJfwesgvdgxQ</td>\n",
       "      <td>Cute interior and owner (?) gave us tour of up...</td>\n",
       "      <td>bcjbaE6dDog4jkNY91ncLQ</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Sx8TMOWLNuJBWer-0pcmoA</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-09-23 23:10:31</td>\n",
       "      <td>04UD14gamNjLY0IDYVhHJg</td>\n",
       "      <td>I am a long term frequent customer of this est...</td>\n",
       "      <td>eUta8W_HdHMXPzLBBZhL1A</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>JrIxlS1TzJ-iCu79ul40cQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-01-03 23:21:18</td>\n",
       "      <td>gmjsEdUsKpj9Xxu6pdjH0g</td>\n",
       "      <td>Loved this tour! I grabbed a groupon and the p...</td>\n",
       "      <td>r3zeYsv1XFBRA4dJpL78cw</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6AxgBCNX_PNTOxmbRSwcKQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-11-15 16:46:40</td>\n",
       "      <td>tuDKqV-4zBZDaAsCoLeGxg</td>\n",
       "      <td>Stayed here for a wedding. Convenient, reasona...</td>\n",
       "      <td>85uM-cor3VXCUyVFEe0YNg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>lRB6hdIPEAJk4grBFfWakg</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129996</th>\n",
       "      <td>4.0</td>\n",
       "      <td>2015-04-19 05:23:13</td>\n",
       "      <td>WStdI9NGcytFT7PEz6aS3g</td>\n",
       "      <td>Horrible, bad service bad food. A bunch of in ...</td>\n",
       "      <td>wGNqqRhYPbh_L3QlntzfkA</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>yEJUiknAwFJxtgrDk9qleA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129997</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2015-08-12 04:09:52</td>\n",
       "      <td>Di6uZDhcwnLsgM66Z4fNfw</td>\n",
       "      <td>Yee-haw, love dive bars! Not always quite so e...</td>\n",
       "      <td>I2XpWCHAom1JRyHXZQrnfg</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>eiD1CL2OXNln9Z5Szx1Fow</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-08-17 19:41:49</td>\n",
       "      <td>bbEXAEFr4RYHLlZ-HFssTA</td>\n",
       "      <td>Great burgers! Only Habit in area where you ca...</td>\n",
       "      <td>3svxI-UDpt8SA-TYZio0iA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>V2wciB0qZ1ARiyj0Q-AmkA</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-05-04 17:59:56</td>\n",
       "      <td>Zi-F-YvyVOK0k5QD7lrLOg</td>\n",
       "      <td>Wow, enjoyed the atmosphere and the wait staff...</td>\n",
       "      <td>1XJsWqvQEHPkQA0sFd829A</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>huB19h2sao1mRcSnEzNUug</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>130000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        useful                 date             business_id  \\\n",
       "0          1.0  2012-01-03 15:28:18  7ATYjTIgM3jUlt4UM3IypQ   \n",
       "1          1.0  2015-01-04 00:01:03  kxX2SOes4o-D3ZQBkiMRfA   \n",
       "2          1.0  2017-01-14 20:54:15  e4Vwtrqf-wpJfwesgvdgxQ   \n",
       "3          1.0  2015-09-23 23:10:31  04UD14gamNjLY0IDYVhHJg   \n",
       "4          0.0  2015-01-03 23:21:18  gmjsEdUsKpj9Xxu6pdjH0g   \n",
       "...        ...                  ...                     ...   \n",
       "129995     0.0  2012-11-15 16:46:40  tuDKqV-4zBZDaAsCoLeGxg   \n",
       "129996     4.0  2015-04-19 05:23:13  WStdI9NGcytFT7PEz6aS3g   \n",
       "129997     2.0  2015-08-12 04:09:52  Di6uZDhcwnLsgM66Z4fNfw   \n",
       "129998     0.0  2016-08-17 19:41:49  bbEXAEFr4RYHLlZ-HFssTA   \n",
       "129999     0.0  2017-05-04 17:59:56  Zi-F-YvyVOK0k5QD7lrLOg   \n",
       "\n",
       "                                                     text  \\\n",
       "0       I've taken a lot of spin classes over the year...   \n",
       "1       Wow!  Yummy, different,  delicious.   Our favo...   \n",
       "2       Cute interior and owner (?) gave us tour of up...   \n",
       "3       I am a long term frequent customer of this est...   \n",
       "4       Loved this tour! I grabbed a groupon and the p...   \n",
       "...                                                   ...   \n",
       "129995  Stayed here for a wedding. Convenient, reasona...   \n",
       "129996  Horrible, bad service bad food. A bunch of in ...   \n",
       "129997  Yee-haw, love dive bars! Not always quite so e...   \n",
       "129998  Great burgers! Only Habit in area where you ca...   \n",
       "129999  Wow, enjoyed the atmosphere and the wait staff...   \n",
       "\n",
       "                       user_id  funny  cool               review_id  stars  \\\n",
       "0       OyoGAe7OKpv6SyGZT5g77Q    0.0   1.0  BiTunyQ73aT9WBnpR9DZGw    5.0   \n",
       "1       _7bHUi9Uuf5__HHc_Q8guQ    0.0   1.0  AqPFMleE6RsU23_auESxiA    5.0   \n",
       "2       bcjbaE6dDog4jkNY91ncLQ    0.0   1.0  Sx8TMOWLNuJBWer-0pcmoA    4.0   \n",
       "3       eUta8W_HdHMXPzLBBZhL1A    2.0   1.0  JrIxlS1TzJ-iCu79ul40cQ    1.0   \n",
       "4       r3zeYsv1XFBRA4dJpL78cw    2.0   0.0  6AxgBCNX_PNTOxmbRSwcKQ    5.0   \n",
       "...                        ...    ...   ...                     ...    ...   \n",
       "129995  85uM-cor3VXCUyVFEe0YNg    0.0   0.0  lRB6hdIPEAJk4grBFfWakg    5.0   \n",
       "129996  wGNqqRhYPbh_L3QlntzfkA    2.0   0.0  yEJUiknAwFJxtgrDk9qleA    1.0   \n",
       "129997  I2XpWCHAom1JRyHXZQrnfg    1.0   1.0  eiD1CL2OXNln9Z5Szx1Fow    4.0   \n",
       "129998  3svxI-UDpt8SA-TYZio0iA    0.0   0.0  V2wciB0qZ1ARiyj0Q-AmkA    5.0   \n",
       "129999  1XJsWqvQEHPkQA0sFd829A    0.0   0.0  huB19h2sao1mRcSnEzNUug    5.0   \n",
       "\n",
       "       sentiment  \n",
       "0              1  \n",
       "1              1  \n",
       "2              1  \n",
       "3              0  \n",
       "4              1  \n",
       "...          ...  \n",
       "129995         1  \n",
       "129996         0  \n",
       "129997         1  \n",
       "129998         1  \n",
       "129999         1  \n",
       "\n",
       "[130000 rows x 10 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_valid_sentiment = review_valid_sentiment.head(130000)\n",
    "review_valid_sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_valid_sentiment.to_csv('./yelp_review_first_130K_with_sentiment.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "539.4040615384615"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# on average, each review entry has ~ 540 characters.\n",
    "# while on average, each IMDB movie review entry has ~ 1300 characters.\n",
    "review_valid_sentiment['text'].apply(len).mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}