2_Netflix_Sim_Matrix.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:32px;text-align:center\"><font color=MediumVioletRed><b>Netflix Movie Recommendations</b></font></p>\n",
    "<p style=\"font-size:20px;text-align:center\"><b><font color=DarkGoldenRod>Part 2</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy import sparse\n",
    "from scipy.sparse import csr_matrix\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import random\n",
    "\n",
    "import os\n",
    "os.chdir(\"D:/Applied_Ai/Case Studies/Netflix Movie Recommendation\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><font color=DarkMagenta>Creating Sparse Matrix From Dataset</font></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "<tr>\n",
    "<td>\n",
    "<img src='data_c.jpg' width='250px' align=left/>\n",
    "</td>\n",
    "<td>\n",
    "<img src='arrow.jpg' width='60px' align=left/>\n",
    "</td>\n",
    "<td>\n",
    "<img src='data_sparse_c.jpg' width='400px' align=left/>\n",
    "</td>\n",
    "</tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Creating Sparse Matrix From Train Dataset</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File is present, getting it from disk...\n",
      "Done...\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile('train_sparse_matrix.npz'):\n",
    "    print(\"File is present, getting it from disk...\")\n",
    "    #just get it from the disk instead of computing it\n",
    "    train_sparse_matrix = sparse.load_npz('train_sparse_matrix.npz')\n",
    "    print(\"Done...\")\n",
    "else:\n",
    "    print(\"We are creating sparse_matrix from the dataframe...\")\n",
    "    start = datetime.now()\n",
    "    \n",
    "    #create sparse_matrix and store it for after usage.\n",
    "    #csr_matrix(data_values, (row_index, col_index), shape_of_matrix)\n",
    "    #it should be in such a way that, MATRIX[row, col] = data\n",
    "    train_sparse_matrix = sparse.csr_matrix((train_data.rating.values, (train_data.user.values,\n",
    "                                               train_data.movie.values)),)\n",
    "    \n",
    "    print('Done... It\\'s shape is : (user, movie) - ',train_sparse_matrix.shape)\n",
    "    print('Saving it into disk for furthur usage...')\n",
    "    #save it into disk\n",
    "    sparse.save_npz(\"train_sparse_matrix.npz\", train_sparse_matrix)\n",
    "    print('Done...\\n')\n",
    "    print(datetime.now() - start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>The Sparsity of Train Sparse Matrix</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparsity of Train Matrix - 99.8292709259195%\n"
     ]
    }
   ],
   "source": [
    "user, movie = train_sparse_matrix.shape\n",
    "\n",
    "ele = train_sparse_matrix.count_nonzero()\n",
    "\n",
    "print(\"Sparsity of Train Matrix - {}%\".format((1-(ele/(user*movie)))*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Creating Sparse Matrix From Test Dataset</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File is present, getting it from disk...\n",
      "Done...\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile('test_sparse_matrix.npz'):\n",
    "    print(\"File is present, getting it from disk...\")\n",
    "    #just get it from the disk instead of computing it\n",
    "    test_sparse_matrix = sparse.load_npz('test_sparse_matrix.npz')\n",
    "    print(\"Done...\")\n",
    "else: \n",
    "    print(\"We are creating sparse_matrix from the dataframe...\")\n",
    "    start = datetime.now()\n",
    "    \n",
    "    #create sparse_matrix and store it for after usage.\n",
    "    #csr_matrix(data_values, (row_index, col_index), shape_of_matrix)\n",
    "    #it should be in such a way that, MATRIX[row, col] = data\n",
    "    test_sparse_matrix = sparse.csr_matrix((test_data.rating.values, (test_data.user.values,\n",
    "                                               test_data.movie.values)))\n",
    "    \n",
    "    print('Done... It\\'s shape is : (user, movie) - ',test_sparse_matrix.shape)\n",
    "    print('Saving it into disk for furthur usage...')\n",
    "    #save it into disk\n",
    "    sparse.save_npz(\"test_sparse_matrix.npz\", test_sparse_matrix)\n",
    "    print('Done...\\n')\n",
    "    print(datetime.now() - start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>The Sparsity of Test Sparse Matrix</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparsity of Test Matrix - 99.95731772988694%\n"
     ]
    }
   ],
   "source": [
    "user, movie = test_sparse_matrix.shape\n",
    "\n",
    "ele = test_sparse_matrix.count_nonzero()\n",
    "\n",
    "print(\"Sparsity of Test Matrix - {}%\".format((1-(ele/(user*movie)))*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><font color=DarkMagenta>Finding Global Average of All Movie Ratings,<br>\n",
    "Average Rating Per User and Average Rating Per Movie</font></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the user averages in dictionary (key: user_id/movie_id, value: avg rating)\n",
    "\n",
    "def get_average_ratings(sparse_matrix, of_users):\n",
    "    \n",
    "    #average ratings of user/axes\n",
    "    ax = 1 if of_users else 0    #1 - User axes, 0 - Movie axes\n",
    "\n",
    "    #\".A1\" is for converting Column_Matrix to 1-D numpy array \n",
    "    sum_of_ratings = sparse_matrix.sum(axis=ax).A1\n",
    "    #boolean matrix of ratings (whether a user rated that movie or not)\n",
    "    is_rated = sparse_matrix != 0\n",
    "    #no of ratings that each user OR movie\n",
    "    no_of_ratings = is_rated.sum(axis=ax).A1\n",
    "    \n",
    "    #max_user and max_movie ids in sparse matrix \n",
    "    u, m = sparse_matrix.shape\n",
    "    #creae a dictonary of users and their average ratigns..\n",
    "    average_ratings = {i : sum_of_ratings[i]/no_of_ratings[i]\n",
    "                                 for i in range(u if of_users else m) \n",
    "                                    if no_of_ratings[i] != 0}\n",
    "    #return that dictionary of average ratings\n",
    "    return average_ratings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Finding Global Average of All Movie Ratings</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'global': 3.582890686321557}\n"
     ]
    }
   ],
   "source": [
    "train_averages = dict()\n",
    "#get the global average of ratings in our train set\n",
    "\n",
    "train_global_average = train_sparse_matrix.sum()/train_sparse_matrix.count_nonzero()\n",
    "train_averages['global'] = train_global_average\n",
    "\n",
    "print(train_averages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Finding Average Rating Per User</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Rating of User 10 - 3.3781094527363185\n"
     ]
    }
   ],
   "source": [
    "train_averages['user'] = get_average_ratings(train_sparse_matrix, of_users=True)\n",
    "\n",
    "print('Average Rating of User 10 -',train_averages['user'][10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Finding Average Rating Per Movie</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Rating of Movie 15 - 3.3038461538461537\n"
     ]
    }
   ],
   "source": [
    "train_averages['movie'] = get_average_ratings(train_sparse_matrix, of_users=False)\n",
    "\n",
    "print('Average Rating of Movie 15 -',train_averages['movie'][15])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><font color=DarkMagenta>Cold Start Problem</font></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Cold Start Problem With Users</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL NUMBER OF USERS - 480189\n",
      "NUMBER OF USERS IN TRAIN DATA - 405041\n",
      "NUMBER OF USERS THAT DIDN'T APPEAR IN TRAIN DATA - 75148 (15.65%)\n"
     ]
    }
   ],
   "source": [
    "ratings_data = pd.read_csv(\"ratings_data.csv\")\n",
    "\n",
    "total_users = len(np.unique(ratings_data.user))\n",
    "users_train = len(train_averages['user'])\n",
    "new_users = total_users - users_train\n",
    "\n",
    "print('TOTAL NUMBER OF USERS -', total_users)\n",
    "print('NUMBER OF USERS IN TRAIN DATA -', users_train)\n",
    "print(\"NUMBER OF USERS THAT DIDN'T APPEAR IN TRAIN DATA - {} ({}%)\".format(new_users, np.round((new_users/total_users)*100,2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> We might have to handle __new users__ (75148) who didn't appear in train data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Cold Start Problem With Movies</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL NUMBER OF MOVIES - 17770\n",
      "NUMBER OF MOVIES IN TRAIN DATA - 17424\n",
      "NUMBER OF MOVIES THAT DIDN'T APPEAR IN TRAIN DATA - 346 (1.95%)\n"
     ]
    }
   ],
   "source": [
    "total_movies = len(np.unique(ratings_data.movie))\n",
    "movies_train = len(train_averages['movie'])\n",
    "new_movies = total_movies - movies_train\n",
    "\n",
    "print('TOTAL NUMBER OF MOVIES -', total_movies)\n",
    "print('NUMBER OF MOVIES IN TRAIN DATA -', movies_train)\n",
    "print(\"NUMBER OF MOVIES THAT DIDN'T APPEAR IN TRAIN DATA - {} ({}%)\".format(new_movies, np.round((new_movies/total_movies)*100,2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> We might have to handle __346 movies__ (small comparatively) in test data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><font color=DarkMagenta>Computing Similarity Matrices</font></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Computing User-User Similarity Matrix</font></b></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Calculating User-User Similarity Matrix is **not very easy** (_unless we have huge Computing Power and lots of time_), because of number of users.\n",
    "\n",
    "    * You can try if you want to. Your system could crash or the program stops with **Memory Error**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <p style=\"font-size:15px\"><b><font color=DarkOliveGreen>Trying With All Dimensions (17k Dimensions Per User)</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def compute_user_similarity(sparse_matrix, compute_for_few=False, top = 100, verbose=False, verb_for_n_rows = 20,\n",
    "                            draw_time_taken=True):\n",
    "    no_of_users = sparse_matrix.shape\n",
    "    #get the indices of  non zero rows(users) from our sparse matrix\n",
    "    row_ind, col_ind = sparse_matrix.nonzero()\n",
    "    \n",
    "    row_ind = sorted(set(row_ind)) #we don't have to\n",
    "    time_taken = list() #time taken for finding similar users for an user\n",
    "    \n",
    "    #we create rows, cols, and data lists, which can be used to create sparse matrices\n",
    "    rows, cols, data = list(), list(), list()\n",
    "    if verbose: print(\"Computing Top\",top,\"Similarities For Each User..\")\n",
    "    \n",
    "    start = datetime.now()\n",
    "    temp = 0\n",
    "    \n",
    "    for row in row_ind[:top] if compute_for_few else row_ind:\n",
    "        temp = temp+1\n",
    "        prev = datetime.now()\n",
    "        \n",
    "        #get the similarity row for this user with all other users\n",
    "        sim = cosine_similarity(sparse_matrix.getrow(row), sparse_matrix).ravel()\n",
    "        #we will get only the top ''top'' most similar users and ignore rest of them..\n",
    "        top_sim_ind = sim.argsort()[-top:]\n",
    "        top_sim_val = sim[top_sim_ind]\n",
    "        \n",
    "        #add them to our rows, cols and data\n",
    "        rows.extend([row]*top)\n",
    "        cols.extend(top_sim_ind)\n",
    "        data.extend(top_sim_val)\n",
    "        time_taken.append(datetime.now().timestamp() - prev.timestamp())\n",
    "        if verbose:\n",
    "            if temp%verb_for_n_rows == 0:\n",
    "                print(\"Computing Done For {} Users [Time Elapsed : {}]\"\n",
    "                      .format(temp, datetime.now()-start))\n",
    "            \n",
    "    #lets create sparse matrix out of these and return it\n",
    "    if verbose: print('Creating Sparse Matrix From The Computed Similarities')\n",
    "    #return rows, cols, data\n",
    "    \n",
    "    if draw_time_taken:\n",
    "        plt.plot(time_taken, label = 'Time Taken For Each User')\n",
    "        plt.plot(np.cumsum(time_taken), label='Total Time')\n",
    "        plt.legend(loc='best')\n",
    "        plt.xlabel('User')\n",
    "        plt.ylabel('Time (Seconds)')\n",
    "        plt.show()\n",
    "        \n",
    "    return sparse.csr_matrix((data, (rows, cols)), shape=(no_of_users, no_of_users)), time_taken      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing Top 100 Similarities For Each User..\n",
      "\n",
      "Computing Done For 20 Users [Time Elapsed : 0:03:20.300488]\n",
      "Computing Done For 40 Users [Time Elapsed : 0:06:38.518391]\n",
      "Computing Done For 60 Users [Time Elapsed : 0:09:53.143126]\n",
      "Computing Done For 80 Users [Time Elapsed : 0:13:10.080447]\n",
      "Computing Done For 100 Users [Time Elapsed : 0:16:24.711032]\n",
      "\n",
      "Time Taken : 0:16:24.711032\n"
     ]
    }
   ],
   "source": [
    "start = datetime.now()\n",
    "u_u_sim_sparse = compute_user_similarity(train_sparse_matrix, compute_for_few=True, top = 100, verbose=True)\n",
    "print(\"-\"*100)\n",
    "print(\"Time Taken :\",datetime.now()-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <p style=\"font-size:15px\"><b><font color=DarkOliveGreen>Trying With Reduced Dimensions (Using TruncatedSVD For Dimensionality Reduction of User Vector)</font></b></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We have  **405,041 users** in out training set and computing similarities between them (**17K dimensional vector**) is time consuming.\n",
    "\n",
    "- From above plot, It took roughly __8.88 sec__ for computing simlilar users for __one user__\n",
    "    \n",
    "    \n",
    "- We have __405,041 users__ with us in training set.\n",
    "\n",
    "\n",
    "- ${ 405041 \\times 8.88 = 3596764.08  \\sec } =  59946.068 \\min = 999.101133333 \\text{ hours}\n",
    "= 41.629213889 \\text{ days}...$\n",
    "\n",
    "    - Even if we run on 4 cores parallelly (a typical system now a days), It will still take almost __10 and 1/2__ days.\n",
    "    \n",
    "**IDEA:** Instead, we will try to reduce the dimentsions using SVD, so that __it might__ speed up the process."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This is taking more time for each user than Original one.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- It took almost __12.18__ for computing simlilar users for __one user__.\n",
    "    \n",
    "    \n",
    "- We have __405041 users__ with us in training set.\n",
    "\n",
    "\n",
    "- ${ 405041 \\times 12.18 ==== 4933399.38 \\sec } ====  82223.323 \\min ==== 1370.388716667 \\text{ hours}\n",
    "==== 57.099529861 \\text{ days}...$\n",
    "\n",
    "    - Even we run on 4 cores parallelly (a typical system now a days), It will still take almost __(14 - 15) __ days.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><font color=red>Is there any other way to compute user user similarity ?</font></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- An alternative is to compute similar users for a particular user,  whenenver required (**ie., Run time**)\n",
    "    - We maintain a binary Vector for users, which tells us whether we already computed or not..\n",
    "    - ***If not*** : \n",
    "        - Compute top (let's just say, 1000) most similar users for this given user, and add this to our datastructure, so that we can just access it(similar users) without recomputing it again.\n",
    "        - \n",
    "    - ***If It is already Computed***:\n",
    "        - Just get it directly from our datastructure, which has that information.\n",
    "        - In production time, We might have to recompute similarities, if it is computed a long time ago. Because user preferences changes over time. If we could maintain some kind of Timer, which when expires, we have to update it ( recompute it ). \n",
    "        - \n",
    "    - ***Which datastructure to use:***\n",
    "        - It is purely implementation dependant. \n",
    "        - One simple method is to maintain a **Dictionary Of Dictionaries**.\n",
    "            - \n",
    "            - **key    :** _userid_ \n",
    "            - __value__: _Again a dictionary_\n",
    "                - __key__  : _Similar User_\n",
    "                - __value__: _Similarity Value_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Computing Movie-Movie Similarity Matrix</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File Is There, We Will Get It...\n",
      "Done...\n",
      "It's a (17771, 17771) Dimensional Matrix.\n"
     ]
    }
   ],
   "source": [
    "if not os.path.isfile('m_m_sim_sparse.npz'):\n",
    "    print(\"It Seems You Don't Have That File. Computing Movie-Movie Similarity...\")\n",
    "    start = datetime.now()\n",
    "    \n",
    "    m_m_sim_sparse = cosine_similarity(X=train_sparse_matrix.T, dense_output=False)\n",
    "    print(\"Done...\")\n",
    "    \n",
    "    #store this sparse matrix in disk before using it. For future purposes.\n",
    "    print(\"Saving File To Disk...\")\n",
    "    sparse.save_npz(\"m_m_sim_sparse.npz\", m_m_sim_sparse)\n",
    "    print(\"Done...\")\n",
    "    print(datetime.now() - start)\n",
    "else:\n",
    "    print(\"File Is There, We Will Get It...\")\n",
    "    m_m_sim_sparse = sparse.load_npz(\"m_m_sim_sparse.npz\")\n",
    "    print(\"Done...\")\n",
    "\n",
    "print(\"It's a\",m_m_sim_sparse.shape,\"Dimensional Matrix.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Even though we have similarity measure of each movie, with all other movies, We generally don't care much about least similar movies.\n",
    "\n",
    "- Most of the times, only top_xxx similar items matters. It may be 10 or 100.\n",
    "\n",
    "- We take only those top similar movie ratings and store them  in a saperate dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_ids = np.unique(m_m_sim_sparse.nonzero()[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:33.411700\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 8279,  8013, 16528,  5927, 13105, 12049,  4424, 10193, 17590,\n",
       "        4549,  3755,   590, 14059, 15144, 15054,  9584,  9071,  6349,\n",
       "       16402,  3973,  1720,  5370, 16309,  9376,  6116,  4706,  2818,\n",
       "         778, 15331,  1416, 12979, 17139, 17710,  5452,  2534,   164,\n",
       "       15188,  8323,  2450, 16331,  9566, 15301, 13213, 14308, 15984,\n",
       "       10597,  6426,  5500,  7068,  7328,  5720,  9802,   376, 13013,\n",
       "        8003, 10199,  3338, 15390,  9688, 16455, 11730,  4513,   598,\n",
       "       12762,  2187,   509,  5865,  9166, 17115, 16334,  1942,  7282,\n",
       "       17584,  4376,  8988,  8873,  5921,  2716, 14679, 11947, 11981,\n",
       "        4649,   565, 12954, 10788, 10220, 10963,  9427,  1690,  5107,\n",
       "        7859,  5969,  1510,  2429,   847,  7845,  6410, 13931,  9840,\n",
       "        3706])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = datetime.now()\n",
    "similar_movies = dict()\n",
    "for movie in movie_ids:\n",
    "    #get the top similar movies and store them in the dictionary\n",
    "    sim_movies = m_m_sim_sparse[movie].toarray().ravel().argsort()[::-1][1:]\n",
    "    similar_movies[movie] = sim_movies[:100]\n",
    "print(datetime.now() - start)\n",
    "\n",
    "#just testing similar movies for movie_15\n",
    "similar_movies[15]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=SaddleBrown>Finding Most Similar Movies Using Similarity Matrix</font></b></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Does Similarity really works as the way we expected ?**<br>\n",
    "_Let's pick some random movie and check for its similar movies..._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenization took: 15.57 ms\n",
      "Type conversion took: 15.62 ms\n",
      "Parser memory cleanup took: 0.00 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year_of_release</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movie_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2003.0</td>\n",
       "      <td>Dinosaur Planet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2004.0</td>\n",
       "      <td>Isle of Man TT 2004 Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>Character</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994.0</td>\n",
       "      <td>Paula Abdul's Get Up &amp; Dance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2004.0</td>\n",
       "      <td>The Rise and Fall of ECW</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          year_of_release                         title\n",
       "movie_id                                               \n",
       "1                  2003.0               Dinosaur Planet\n",
       "2                  2004.0    Isle of Man TT 2004 Review\n",
       "3                  1997.0                     Character\n",
       "4                  1994.0  Paula Abdul's Get Up & Dance\n",
       "5                  2004.0      The Rise and Fall of ECW"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#first Let's load the movie details into see dataframe\n",
    "\n",
    "movie_titles = pd.read_csv(\"movie_titles.csv\", sep=',', header = None,\n",
    "                           names=['movie_id', 'year_of_release', 'title'], verbose=True,\n",
    "                      index_col = 'movie_id', encoding = \"ISO-8859-1\")\n",
    "\n",
    "movie_titles.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=DarkOliveGreen>Similar Movies for 'Vampire Journals'</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Movie -----> Vampire Journals\n",
      "\n",
      "It Has 270 Ratings From Users.\n",
      "We Have 17284 Movies Which Are Similar To This Movie & We Will Get Only Top Most.\n"
     ]
    }
   ],
   "source": [
    "mv_id = 67\n",
    "\n",
    "print(\"Movie ----->\",movie_titles.loc[mv_id].values[1])\n",
    "\n",
    "print(\"It Has {} Ratings From Users.\".format(train_sparse_matrix[:,mv_id].getnnz()))\n",
    "\n",
    "print(\"We Have {} Movies Which Are Similar To This Movie & We Will Get Only Top Most.\".format(m_m_sim_sparse[:,mv_id].getnnz()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities = m_m_sim_sparse[mv_id].toarray().ravel()\n",
    "\n",
    "similar_indices = similarities.argsort()[::-1][1:]\n",
    "\n",
    "similarities[similar_indices]\n",
    "\n",
    "sim_indices = similarities.argsort()[::-1][1:] #it will sort and reverse the array and ignore its similarity (ie.,1)\n",
    "                                               #and return its indices(movie_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(similarities[sim_indices], label='All The Ratings')\n",
    "plt.plot(similarities[sim_indices[:100]], label='Top 100 Similar Movies')\n",
    "plt.title(\"Similar Movies of {}(movie_id)\".format(mv_id), fontsize=20)\n",
    "plt.xlabel(\"Movies (Not Movie_Ids)\", fontsize=15)\n",
    "plt.ylabel(\"Cosine Similarity\",fontsize=15)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:15px\"><b><font color=DarkOliveGreen>Top 10 Similar Movies</font></b></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year_of_release</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movie_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>1999</td>\n",
       "      <td>Modern Vampires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4044</th>\n",
       "      <td>1998</td>\n",
       "      <td>Subspecies 4: Bloodstorm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1688</th>\n",
       "      <td>1993</td>\n",
       "      <td>To Sleep With a Vampire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13962</th>\n",
       "      <td>2001</td>\n",
       "      <td>Dracula: The Dark Prince</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12053</th>\n",
       "      <td>1993</td>\n",
       "      <td>Dracula Rising</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16279</th>\n",
       "      <td>2002</td>\n",
       "      <td>Vampires: Los Muertos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4667</th>\n",
       "      <td>1996</td>\n",
       "      <td>Vampirella</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1900</th>\n",
       "      <td>1997</td>\n",
       "      <td>Club Vampire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13873</th>\n",
       "      <td>2001</td>\n",
       "      <td>The Breed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15867</th>\n",
       "      <td>2003</td>\n",
       "      <td>Dracula II: Ascension</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          year_of_release                     title\n",
       "movie_id                                           \n",
       "323                  1999           Modern Vampires\n",
       "4044                 1998  Subspecies 4: Bloodstorm\n",
       "1688                 1993   To Sleep With a Vampire\n",
       "13962                2001  Dracula: The Dark Prince\n",
       "12053                1993            Dracula Rising\n",
       "16279                2002     Vampires: Los Muertos\n",
       "4667                 1996                Vampirella\n",
       "1900                 1997              Club Vampire\n",
       "13873                2001                 The Breed\n",
       "15867                2003     Dracula II: Ascension"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_titles.loc[sim_indices[:10]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Similarly, we can ___find similar users___ and compare how similar they are. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"font-size:20px;text-align:center\"><b>For Machine Learning Models - Refer <a href=\"https://gitlab.com/akashbangalkar/netflix-movie-recommendation/-/blob/main/3_Netflix_Machine_Learning.ipynb\">Part 3</a> Notebook</b></p>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}