MovieLens_Autodiff.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using ZipFile,\n",
    "using CSV\n",
    "using DataFrames\n",
    "using StatsBase\n",
    "using Random, Distributions\n",
    "using Flux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>rowN</th><th>userId</th><th>movieId</th><th>rating</th><th>timestamp</th><th>title</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th><th>Float64</th><th>Int64</th><th>String</th></tr></thead><tbody><p>999,999 rows × 7 columns (omitted printing of 1 columns)</p><tr><th>1</th><td>1</td><td>1</td><td>231</td><td>5.0</td><td>838983392</td><td>Dumb &amp; Dumber (1994)</td></tr><tr><th>2</th><td>2</td><td>1</td><td>480</td><td>5.0</td><td>838983653</td><td>Jurassic Park (1993)</td></tr><tr><th>3</th><td>3</td><td>1</td><td>586</td><td>5.0</td><td>838984068</td><td>Home Alone (1990)</td></tr><tr><th>4</th><td>4</td><td>2</td><td>151</td><td>3.0</td><td>868246450</td><td>Rob Roy (1995)</td></tr><tr><th>5</th><td>5</td><td>2</td><td>858</td><td>2.0</td><td>868245645</td><td>Godfather, The (1972)</td></tr><tr><th>6</th><td>6</td><td>2</td><td>1544</td><td>3.0</td><td>868245920</td><td>Lost World: Jurassic Park, The (Jurassic Park 2) (1997)</td></tr><tr><th>7</th><td>7</td><td>3</td><td>590</td><td>3.5</td><td>1136075494</td><td>Dances with Wolves (1990)</td></tr><tr><th>8</th><td>8</td><td>3</td><td>4995</td><td>4.5</td><td>1133571200</td><td>Beautiful Mind, A (2001)</td></tr><tr><th>9</th><td>9</td><td>4</td><td>34</td><td>5.0</td><td>844416936</td><td>Babe (1995)</td></tr><tr><th>10</th><td>10</td><td>4</td><td>432</td><td>3.0</td><td>844417070</td><td>City Slickers II: The Legend of Curly's Gold (1994)</td></tr><tr><th>11</th><td>11</td><td>4</td><td>434</td><td>3.0</td><td>844416796</td><td>Cliffhanger (1993)</td></tr><tr><th>12</th><td>12</td><td>5</td><td>85</td><td>3.0</td><td>857912791</td><td>Angels and Insects (1995)</td></tr><tr><th>13</th><td>13</td><td>5</td><td>171</td><td>3.0</td><td>857912492</td><td>Jeffrey (1995)</td></tr><tr><th>14</th><td>14</td><td>5</td><td>232</td><td>3.0</td><td>857912933</td><td>Eat Drink Man Woman (Yin shi nan nu) (1994)</td></tr><tr><th>15</th><td>15</td><td>5</td><td>242</td><td>3.0</td><td>857912738</td><td>Farinelli: il castrato (1994)</td></tr><tr><th>16</th><td>16</td><td>5</td><td>306</td><td>3.0</td><td>857912441</td><td>Three Colors: Red (Trois couleurs: Rouge) (1994)</td></tr><tr><th>17</th><td>17</td><td>5</td><td>345</td><td>3.0</td><td>857912365</td><td>Adventures of Priscilla, Queen of the Desert, The (1994)</td></tr><tr><th>18</th><td>18</td><td>5</td><td>592</td><td>3.0</td><td>857912657</td><td>Batman (1989)</td></tr><tr><th>19</th><td>19</td><td>5</td><td>593</td><td>4.0</td><td>857912364</td><td>Silence of the Lambs, The (1991)</td></tr><tr><th>20</th><td>20</td><td>5</td><td>920</td><td>5.0</td><td>857912840</td><td>Gone with the Wind (1939)</td></tr><tr><th>21</th><td>21</td><td>5</td><td>1097</td><td>3.0</td><td>857912738</td><td>E.T. the Extra-Terrestrial (1982)</td></tr><tr><th>22</th><td>22</td><td>5</td><td>1206</td><td>4.0</td><td>857912791</td><td>Clockwork Orange, A (1971)</td></tr><tr><th>23</th><td>23</td><td>5</td><td>1244</td><td>4.0</td><td>857913143</td><td>Manhattan (1979)</td></tr><tr><th>24</th><td>24</td><td>5</td><td>1280</td><td>4.0</td><td>857913337</td><td>Raise the Red Lantern (Da hong deng long gao gao gua) (1991)</td></tr><tr><th>25</th><td>25</td><td>6</td><td>260</td><td>5.0</td><td>1001083120</td><td>Star Wars: Episode IV - A New Hope (a.k.a. Star Wars) (1977)</td></tr><tr><th>26</th><td>26</td><td>6</td><td>3755</td><td>4.0</td><td>1001083338</td><td>Perfect Storm, The (2000)</td></tr><tr><th>27</th><td>27</td><td>6</td><td>4369</td><td>2.0</td><td>1001083221</td><td>Fast and the Furious, The (2001)</td></tr><tr><th>28</th><td>28</td><td>7</td><td>599</td><td>5.0</td><td>1049764568</td><td>Wild Bunch, The (1969)</td></tr><tr><th>29</th><td>29</td><td>7</td><td>904</td><td>5.0</td><td>1050098597</td><td>Rear Window (1954)</td></tr><tr><th>30</th><td>30</td><td>7</td><td>912</td><td>5.0</td><td>1049764461</td><td>Casablanca (1942)</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccc}\n",
       "\t& rowN & userId & movieId & rating & timestamp & title & \\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Float64 & Int64 & String & \\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 231 & 5.0 & 838983392 & Dumb \\& Dumber (1994) & $\\dots$ \\\\\n",
       "\t2 & 2 & 1 & 480 & 5.0 & 838983653 & Jurassic Park (1993) & $\\dots$ \\\\\n",
       "\t3 & 3 & 1 & 586 & 5.0 & 838984068 & Home Alone (1990) & $\\dots$ \\\\\n",
       "\t4 & 4 & 2 & 151 & 3.0 & 868246450 & Rob Roy (1995) & $\\dots$ \\\\\n",
       "\t5 & 5 & 2 & 858 & 2.0 & 868245645 & Godfather, The (1972) & $\\dots$ \\\\\n",
       "\t6 & 6 & 2 & 1544 & 3.0 & 868245920 & Lost World: Jurassic Park, The (Jurassic Park 2) (1997) & $\\dots$ \\\\\n",
       "\t7 & 7 & 3 & 590 & 3.5 & 1136075494 & Dances with Wolves (1990) & $\\dots$ \\\\\n",
       "\t8 & 8 & 3 & 4995 & 4.5 & 1133571200 & Beautiful Mind, A (2001) & $\\dots$ \\\\\n",
       "\t9 & 9 & 4 & 34 & 5.0 & 844416936 & Babe (1995) & $\\dots$ \\\\\n",
       "\t10 & 10 & 4 & 432 & 3.0 & 844417070 & City Slickers II: The Legend of Curly's Gold (1994) & $\\dots$ \\\\\n",
       "\t11 & 11 & 4 & 434 & 3.0 & 844416796 & Cliffhanger (1993) & $\\dots$ \\\\\n",
       "\t12 & 12 & 5 & 85 & 3.0 & 857912791 & Angels and Insects (1995) & $\\dots$ \\\\\n",
       "\t13 & 13 & 5 & 171 & 3.0 & 857912492 & Jeffrey (1995) & $\\dots$ \\\\\n",
       "\t14 & 14 & 5 & 232 & 3.0 & 857912933 & Eat Drink Man Woman (Yin shi nan nu) (1994) & $\\dots$ \\\\\n",
       "\t15 & 15 & 5 & 242 & 3.0 & 857912738 & Farinelli: il castrato (1994) & $\\dots$ \\\\\n",
       "\t16 & 16 & 5 & 306 & 3.0 & 857912441 & Three Colors: Red (Trois couleurs: Rouge) (1994) & $\\dots$ \\\\\n",
       "\t17 & 17 & 5 & 345 & 3.0 & 857912365 & Adventures of Priscilla, Queen of the Desert, The (1994) & $\\dots$ \\\\\n",
       "\t18 & 18 & 5 & 592 & 3.0 & 857912657 & Batman (1989) & $\\dots$ \\\\\n",
       "\t19 & 19 & 5 & 593 & 4.0 & 857912364 & Silence of the Lambs, The (1991) & $\\dots$ \\\\\n",
       "\t20 & 20 & 5 & 920 & 5.0 & 857912840 & Gone with the Wind (1939) & $\\dots$ \\\\\n",
       "\t21 & 21 & 5 & 1097 & 3.0 & 857912738 & E.T. the Extra-Terrestrial (1982) & $\\dots$ \\\\\n",
       "\t22 & 22 & 5 & 1206 & 4.0 & 857912791 & Clockwork Orange, A (1971) & $\\dots$ \\\\\n",
       "\t23 & 23 & 5 & 1244 & 4.0 & 857913143 & Manhattan (1979) & $\\dots$ \\\\\n",
       "\t24 & 24 & 5 & 1280 & 4.0 & 857913337 & Raise the Red Lantern (Da hong deng long gao gao gua) (1991) & $\\dots$ \\\\\n",
       "\t25 & 25 & 6 & 260 & 5.0 & 1001083120 & Star Wars: Episode IV - A New Hope (a.k.a. Star Wars) (1977) & $\\dots$ \\\\\n",
       "\t26 & 26 & 6 & 3755 & 4.0 & 1001083338 & Perfect Storm, The (2000) & $\\dots$ \\\\\n",
       "\t27 & 27 & 6 & 4369 & 2.0 & 1001083221 & Fast and the Furious, The (2001) & $\\dots$ \\\\\n",
       "\t28 & 28 & 7 & 599 & 5.0 & 1049764568 & Wild Bunch, The (1969) & $\\dots$ \\\\\n",
       "\t29 & 29 & 7 & 904 & 5.0 & 1050098597 & Rear Window (1954) & $\\dots$ \\\\\n",
       "\t30 & 30 & 7 & 912 & 5.0 & 1049764461 & Casablanca (1942) & $\\dots$ \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ &  \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "999999×7 DataFrame. Omitted printing of 2 columns\n",
       "│ Row    │ rowN   │ userId │ movieId │ rating  │ timestamp  │\n",
       "│        │ \u001b[90mInt64\u001b[39m  │ \u001b[90mInt64\u001b[39m  │ \u001b[90mInt64\u001b[39m   │ \u001b[90mFloat64\u001b[39m │ \u001b[90mInt64\u001b[39m      │\n",
       "├────────┼────────┼────────┼─────────┼─────────┼────────────┤\n",
       "│ 1      │ 1      │ 1      │ 231     │ 5.0     │ 838983392  │\n",
       "│ 2      │ 2      │ 1      │ 480     │ 5.0     │ 838983653  │\n",
       "│ 3      │ 3      │ 1      │ 586     │ 5.0     │ 838984068  │\n",
       "│ 4      │ 4      │ 2      │ 151     │ 3.0     │ 868246450  │\n",
       "│ 5      │ 5      │ 2      │ 858     │ 2.0     │ 868245645  │\n",
       "│ 6      │ 6      │ 2      │ 1544    │ 3.0     │ 868245920  │\n",
       "│ 7      │ 7      │ 3      │ 590     │ 3.5     │ 1136075494 │\n",
       "│ 8      │ 8      │ 3      │ 4995    │ 4.5     │ 1133571200 │\n",
       "│ 9      │ 9      │ 4      │ 34      │ 5.0     │ 844416936  │\n",
       "│ 10     │ 10     │ 4      │ 432     │ 3.0     │ 844417070  │\n",
       "⋮\n",
       "│ 999989 │ 999989 │ 71565  │ 2851    │ 3.0     │ 974294986  │\n",
       "│ 999990 │ 999990 │ 71565  │ 3090    │ 3.0     │ 974295544  │\n",
       "│ 999991 │ 999991 │ 71565  │ 3742    │ 3.0     │ 974295077  │\n",
       "│ 999992 │ 999992 │ 71566  │ 25      │ 4.0     │ 830341062  │\n",
       "│ 999993 │ 999993 │ 71566  │ 161     │ 4.0     │ 830340834  │\n",
       "│ 999994 │ 999994 │ 71566  │ 186     │ 3.0     │ 830341062  │\n",
       "│ 999995 │ 999995 │ 71566  │ 235     │ 5.0     │ 830341062  │\n",
       "│ 999996 │ 999996 │ 71566  │ 273     │ 3.0     │ 830341118  │\n",
       "│ 999997 │ 999997 │ 71566  │ 434     │ 3.0     │ 830340953  │\n",
       "│ 999998 │ 999998 │ 71567  │ 480     │ 4.0     │ 912580688  │\n",
       "│ 999999 │ 999999 │ 71567  │ 898     │ 4.0     │ 912649403  │"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import ratings file\n",
    "cd(raw\"/home/emmanuel/Development/Learning/capstone-movielens/\")\n",
    "# z = ZipFile.Reader(\"datasets/ratings.zip\")\n",
    "# ratings = CSV.read(z.files[1];\n",
    "#                 delim=\"::\",\n",
    "#                 header=[\"userId\", \"movieId\", \"rating\", \"timestamp\"],\n",
    "#                 types = [Int, Int, Float64, Int])\n",
    "\n",
    "edx = CSV.read(\"datasets/edx.csv\";\n",
    "               delim=\",\",\n",
    "               copycols=true,\n",
    "               datarow=2,\n",
    "               header=[\"rowN\", \"userId\", \"movieId\", \"rating\", \"timestamp\", \"title\", \"genres\"],\n",
    "               types = [Int, Int, Int, Float64, Int, String, String])\n",
    "\n",
    "validation = CSV.read(\"datasets/validation.csv\";\n",
    "                      delim=\",\",\n",
    "                      copycols=true,\n",
    "                      datarow=2,\n",
    "                      header=[\"rowN\", \"userId\", \"movieId\", \"rating\", \"timestamp\", \"title\", \"genres\"],\n",
    "                      types = [Int, Int, Int, Float64, Int, String, String])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "999999-element Array{Float64,1}:\n",
       "  1.4028961120912835 \n",
       "  1.4028961120912835 \n",
       "  1.4028961120912835 \n",
       " -0.483306635705546  \n",
       " -1.4264080096039606 \n",
       " -0.483306635705546  \n",
       " -0.01175594875633861\n",
       "  0.9313454251420761 \n",
       "  1.4028961120912835 \n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       "  ⋮                  \n",
       "  0.45979473819286876\n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       "  0.45979473819286876\n",
       "  0.45979473819286876\n",
       " -0.483306635705546  \n",
       "  1.4028961120912835 \n",
       " -0.483306635705546  \n",
       " -0.483306635705546  \n",
       "  0.45979473819286876\n",
       "  0.45979473819286876"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_m, r_sd = mean_and_std(edx.rating)\n",
    "\n",
    "edx.rating_z        = (edx.rating .- r_m) ./ r_sd\n",
    "validation.rating_z = (validation.rating .- r_m) ./ r_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>userId</th><th>user_mean</th></tr><tr><th></th><th>Int64</th><th>Float64</th></tr></thead><tbody><p>69,878 rows × 2 columns</p><tr><th>1</th><td>1</td><td>1.4029</td></tr><tr><th>2</th><td>2</td><td>-0.205924</td></tr><tr><th>3</th><td>3</td><td>0.398949</td></tr><tr><th>4</th><td>4</td><td>0.513686</td></tr><tr><th>5</th><td>5</td><td>0.383327</td></tr><tr><th>6</th><td>6</td><td>0.411431</td></tr><tr><th>7</th><td>7</td><td>0.332083</td></tr><tr><th>8</th><td>8</td><td>-0.118779</td></tr><tr><th>9</th><td>9</td><td>0.504704</td></tr><tr><th>10</th><td>10</td><td>0.299804</td></tr><tr><th>11</th><td>11</td><td>0.720961</td></tr><tr><th>12</th><td>12</td><td>-0.0418549</td></tr><tr><th>13</th><td>13</td><td>-0.198879</td></tr><tr><th>14</th><td>14</td><td>-0.244882</td></tr><tr><th>15</th><td>16</td><td>0.247597</td></tr><tr><th>16</th><td>17</td><td>0.32743</td></tr><tr><th>17</th><td>18</td><td>-0.0547426</td></tr><tr><th>18</th><td>19</td><td>0.210921</td></tr><tr><th>19</th><td>22</td><td>-0.12403</td></tr><tr><th>20</th><td>23</td><td>0.658342</td></tr><tr><th>21</th><td>24</td><td>0.577682</td></tr><tr><th>22</th><td>26</td><td>0.529654</td></tr><tr><th>23</th><td>27</td><td>-0.344615</td></tr><tr><th>24</th><td>28</td><td>0.158002</td></tr><tr><th>25</th><td>29</td><td>-1.45261</td></tr><tr><th>26</th><td>30</td><td>0.93469</td></tr><tr><th>27</th><td>33</td><td>-0.208235</td></tr><tr><th>28</th><td>34</td><td>-0.733384</td></tr><tr><th>29</th><td>35</td><td>-0.295486</td></tr><tr><th>30</th><td>36</td><td>-0.143402</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& userId & user\\_mean\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1.4029 \\\\\n",
       "\t2 & 2 & -0.205924 \\\\\n",
       "\t3 & 3 & 0.398949 \\\\\n",
       "\t4 & 4 & 0.513686 \\\\\n",
       "\t5 & 5 & 0.383327 \\\\\n",
       "\t6 & 6 & 0.411431 \\\\\n",
       "\t7 & 7 & 0.332083 \\\\\n",
       "\t8 & 8 & -0.118779 \\\\\n",
       "\t9 & 9 & 0.504704 \\\\\n",
       "\t10 & 10 & 0.299804 \\\\\n",
       "\t11 & 11 & 0.720961 \\\\\n",
       "\t12 & 12 & -0.0418549 \\\\\n",
       "\t13 & 13 & -0.198879 \\\\\n",
       "\t14 & 14 & -0.244882 \\\\\n",
       "\t15 & 16 & 0.247597 \\\\\n",
       "\t16 & 17 & 0.32743 \\\\\n",
       "\t17 & 18 & -0.0547426 \\\\\n",
       "\t18 & 19 & 0.210921 \\\\\n",
       "\t19 & 22 & -0.12403 \\\\\n",
       "\t20 & 23 & 0.658342 \\\\\n",
       "\t21 & 24 & 0.577682 \\\\\n",
       "\t22 & 26 & 0.529654 \\\\\n",
       "\t23 & 27 & -0.344615 \\\\\n",
       "\t24 & 28 & 0.158002 \\\\\n",
       "\t25 & 29 & -1.45261 \\\\\n",
       "\t26 & 30 & 0.93469 \\\\\n",
       "\t27 & 33 & -0.208235 \\\\\n",
       "\t28 & 34 & -0.733384 \\\\\n",
       "\t29 & 35 & -0.295486 \\\\\n",
       "\t30 & 36 & -0.143402 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "69878×2 DataFrame\n",
       "│ Row   │ userId │ user_mean  │\n",
       "│       │ \u001b[90mInt64\u001b[39m  │ \u001b[90mFloat64\u001b[39m    │\n",
       "├───────┼────────┼────────────┤\n",
       "│ 1     │ 1      │ 1.4029     │\n",
       "│ 2     │ 2      │ -0.205924  │\n",
       "│ 3     │ 3      │ 0.398949   │\n",
       "│ 4     │ 4      │ 0.513686   │\n",
       "│ 5     │ 5      │ 0.383327   │\n",
       "│ 6     │ 6      │ 0.411431   │\n",
       "│ 7     │ 7      │ 0.332083   │\n",
       "│ 8     │ 8      │ -0.118779  │\n",
       "│ 9     │ 9      │ 0.504704   │\n",
       "│ 10    │ 10     │ 0.299804   │\n",
       "⋮\n",
       "│ 69868 │ 71557  │ 0.135804   │\n",
       "│ 69869 │ 71558  │ 0.367334   │\n",
       "│ 69870 │ 71559  │ 0.703518   │\n",
       "│ 69871 │ 71560  │ -0.0117559 │\n",
       "│ 69872 │ 71561  │ 0.626224   │\n",
       "│ 69873 │ 71562  │ 0.45157    │\n",
       "│ 69874 │ 71563  │ -0.0672325 │\n",
       "│ 69875 │ 71564  │ -0.451517  │\n",
       "│ 69876 │ 71565  │ 0.364011   │\n",
       "│ 69877 │ 71566  │ 0.328199   │\n",
       "│ 69878 │ 71567  │ -0.316877  │"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movieIndex = DataFrame(movieId = unique(edx, :movieId).movieId)\n",
    "nMovie = size(movieIndex)[1]\n",
    "movieIndex.movieN = 1:nMovie\n",
    "\n",
    "userIndex = DataFrame(userId = unique(edx, :userId).userId)\n",
    "nUser = size(userIndex)[1]\n",
    "userIndex.userN = 1:nUser\n",
    "\n",
    "movieMean = by(edx, :movieId, movie_mean = :rating_z => mean)\n",
    "userMean = by(edx, :userId, user_mean = :rating_z => mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "999999"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training = join(edx, movieIndex, on = :movieId )\n",
    "training = join(training, userIndex, on = :userId )\n",
    "training = training[:, [:userN, :movieN, :rating_z]]\n",
    "nSamples = size(training)[1]\n",
    "\n",
    "test = join(validation, movieIndex, on = :movieId )\n",
    "test = join(test, userIndex, on = :userId )\n",
    "test = test[:, [:userN, :movieN, :rating, :rating_z]]\n",
    "nTest = size(test)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tracked 10677×3 Array{Float64,2}:\n",
       " -0.616675   1.0  -0.0008925  \n",
       " -0.361332   1.0  -0.000540742\n",
       " -0.0890802  1.0  -0.00165577 \n",
       " -0.153526   1.0  -0.000116193\n",
       " -0.16505    1.0  -0.00052468 \n",
       " -0.966375   1.0  -0.000815219\n",
       "  0.471887   1.0  -0.00100502 \n",
       " -0.0541851  1.0   0.000336547\n",
       "  0.225951   1.0  -0.000794165\n",
       " -0.52383    1.0  -0.000984026\n",
       "  0.0135865  1.0   5.96994e-5 \n",
       " -0.71666    1.0   0.000200833\n",
       " -0.575833   1.0   0.000602939\n",
       "  ⋮                           \n",
       "  0.459795   1.0  -0.000875203\n",
       " -2.84106    1.0   0.00135934 \n",
       "  0.459795   1.0   0.000346972\n",
       " -1.89796    1.0   0.000240046\n",
       " -0.483307   1.0  -0.000962833\n",
       "  0.459795   1.0   1.02952e-5 \n",
       " -2.84106    1.0  -0.00129574 \n",
       " -0.0117559  1.0  -0.000742125\n",
       " -0.483307   1.0   0.000798655\n",
       " -1.42641    1.0  -0.00132777 \n",
       " -1.42641    1.0  -0.00228968 \n",
       " -0.954857   1.0  -0.00110868 "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nLF = 3\n",
    "\n",
    "P = zeros(Float64, nUser, nLF)\n",
    "Q = zeros(Float64, nMovie, nLF)\n",
    "\n",
    "P[:,1] .= 1\n",
    "P[:,2] = sort(join(userIndex, userMean, on = :userId, kind = :left), :userN)[:, :user_mean]\n",
    "\n",
    "Q[:,1] = sort(join(movieIndex, movieMean, on = :movieId, kind = :left), :movieN)[:, :movie_mean]\n",
    "Q[:,2] .= 1\n",
    "\n",
    "Random.seed!(42)\n",
    "P[:, 3] = rand(Normal(), nUser) / 1000\n",
    "Q[:, 3] = rand(Normal(), nMovie) / 1000\n",
    "\n",
    "\n",
    "# Make P and Q Flux parameters\n",
    "P = param(P)\n",
    "Q = param(Q)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "J (generic function with 3 methods)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cost function on sample\n",
    "\n",
    "function J(p, q)\n",
    "    error = sum( p[test[:,:userN], :] .* q[test[:,:movieN], :]; dims = 2) .* r_sd .+ r_m\n",
    "    error = sqrt( sum((test.rating - error).^2) / nTest )\n",
    "    \n",
    "    return error\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tracked 3-element Array{Float64,1}:\n",
       "  1.0                  \n",
       " -0.1988792372282461   \n",
       "  0.0005181487878771377"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "P[13, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "J (generic function with 3 methods)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cost function on full set\n",
    "\n",
    "function prediction(user_n, movie_n) = sum( P[user_n, :] .* Q[movie_n, :], dims = 2) .* r_sd + r_m \n",
    "    \n",
    "\n",
    "function J(p, q)\n",
    "    error = sum( p[training[:,:userN], :] .* q[training[:,:movieN], :]; dims = 2) .* r_sd .+ r_m\n",
    "    error = sqrt( sum((training.rating - error).^2) / nTest )\n",
    "    \n",
    "    return error\n",
    "end\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "UndefVarError",
     "evalue": "UndefVarError: p not defined",
     "output_type": "error",
     "traceback": [
      "UndefVarError: p not defined",
      "",
      "Stacktrace:",
      " [1] (::getfield(Main, Symbol(\"##6#7\")))() at ./In[30]:3",
      " [2] forward(::getfield(Main, Symbol(\"##6#7\")), ::Tracker.Params) at /home/emmanuel/.julia/packages/Tracker/SAr25/src/back.jl:135",
      " [3] gradient_nested(::Function, ::Tracker.Params) at /home/emmanuel/.julia/packages/Tracker/SAr25/src/back.jl:159",
      " [4] #gradient#24(::Bool, ::Function, ::Function, ::Tracker.Params) at /home/emmanuel/.julia/packages/Tracker/SAr25/src/back.jl:164",
      " [5] (::getfield(Tracker, Symbol(\"#kw##gradient\")))(::NamedTuple{(:nest,),Tuple{Bool}}, ::typeof(Tracker.gradient), ::Function, ::Tracker.Params) at ./none:0",
      " [6] top-level scope at In[30]:1"
     ]
    }
   ],
   "source": [
    "# Gradient\n",
    "\n",
    "dJ = Tracker.gradient(() -> J(p, q), Flux.params(P, Q), nest = true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "function stochastic_grad_descent(P::Array{Float64, 2}, Q::Array{Float64, 2}, grad;\n",
    "  times = 1, batch_size = 10000, λ = 0.1, α = 0.01, verbose = true)\n",
    "\n",
    "  for i = 1:times\n",
    "        \n",
    "    spl_P = P[spl_training[:,:userN], :]\n",
    "    spl_Q = Q[spl_training[:,:movieN], :]\n",
    "    \n",
    "       \n",
    "        \n",
    "    P[spl_training[:,:userN], :]  = spl_P - α * ΔP\n",
    "    Q[spl_training[:,:movieN], :] = spl_Q - α * ΔQ\n",
    "  end # for loop\n",
    "\n",
    "  return P, Q\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "float_t = sum( P[test[:,:userN], :] .* Q[test[:,:movieN], :]; dims = 2) .* r_sd .+ r_m\n",
    "float_t = sqrt( sum((test.rating - float_t).^2) / nTest )\n",
    "println(\"Initial floating point RMSE test = \",float_t)\n",
    "\n",
    "round_t = round.(P[test[:,:userN], :] .* Q[test[:,:movieN], :])\n",
    "round_t = sum(round_t ; dims = 2) .* r_sd .+ r_m\n",
    "round_t = sqrt( sum((test.rating - round_t).^2) / nTest )\n",
    "println(\"Initial rounded RMSE test = \", round_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "starting_α = 0.01\n",
    "validation_results =[0 float_t round_t]\n",
    "\n",
    "for n = 1:500\n",
    "  batch_size = 10000\n",
    "  α = starting_α\n",
    "  nFeatures = size(P)[2]\n",
    "  λ = 0.1 * (nUser + nMovie) * nFeatures / 2_000_000\n",
    "\n",
    "  RMSE = sqrt(sum((training.rating_z -\n",
    "                  sum( P[training[:,:userN], :] .* Q[training[:,:movieN], :]; dims = 2)).^2) / nSamples)\n",
    "\n",
    "  for i = 1:250\n",
    "    RMSE_tmp =   RMSE\n",
    "\n",
    "    (newP, newQ) = stochastic_grad_descent(P, Q; times = 100, batch_size = 1000 * nFeatures, λ = λ, α = α)\n",
    "    global P = newP\n",
    "    global Q = newQ\n",
    "\n",
    "    RMSE = sqrt(sum((training.rating_z -\n",
    "                     sum( P[training[:,:userN], :] .* Q[training[:,:movieN], :]; dims = 2)).^2) / nSamples)\n",
    "\n",
    "    println(\"Step: \", i, \"   RMSE of z-score training = \", RMSE)\n",
    "\n",
    "    if (RMSE > RMSE_tmp)\n",
    "      α /= 2\n",
    "      println(\"α decreased to \", α)\n",
    "    end\n",
    "\n",
    "    if (starting_α / α > 10000) | (abs((RMSE - RMSE_tmp) / RMSE_tmp) < 1e-6)\n",
    "      break\n",
    "    end\n",
    "\n",
    "  end\n",
    "\n",
    "\n",
    "  test_RMSE = P[test[:,:userN], :] .* Q[test[:,:movieN], :]\n",
    "\n",
    "  # floating point ratings float_RMSE = sum(test_RMSE ; dims = 2).* r_sd .+ r_m float_RMSE = sqrt( sum((test.rating - float_RMSE).^2) / nTest ) println(\"Step: \", n, \"    float RMSE test = \",float_RMSE, \" with number features = \", nFeatures) # Round to only obtain legal ratings round_RMSE = sum( round.(test_RMSE) ; dims = 2).* r_sd .+ r_m round_RMSE = sqrt( sum((test.rating - round_RMSE).^2) / nTest ) println(\"Step: \", n, \"    round RMSE test = \",round_RMSE, \" with number features = \", nFeatures) global validation_results = [validation_results; nFeatures float_RMSE round_RMSE] # Add 1 features global P = [P rand(Normal(), nUser)/1000] global Q = [Q rand(Normal(), nMovie)/1000]\n",
    "  nFeatures += 1\n",
    "\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "stochastic_grad_descent_simple (generic function with 1 method)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function stochastic_grad_descent_simple(P::Array{Float64, 2}, Q::Array{Float64, 2};\n",
    "  times = 1, batch_size = 10000, λ = 0.1, α = 0.01, verbose = true)\n",
    "\n",
    "  for i = 1:times\n",
    "    spl_training = view(training, sample(axes(training, 1), batch_size; replace = false, ordered = true), :)\n",
    "\n",
    "    spl_P = P[spl_training[:,:userN], :]\n",
    "    spl_Q = Q[spl_training[:,:movieN], :]\n",
    "\n",
    "    err = spl_training.rating_z - sum(spl_P .* spl_Q; dims = 2)\n",
    "    ΔP = - 2 * err .* spl_Q + λ * spl_P\n",
    "    ΔQ = - 2 * err .* spl_P + λ * spl_Q\n",
    "\n",
    "    P[spl_training[:,:userN], :]  = spl_P - α * ΔP\n",
    "    Q[spl_training[:,:movieN], :] = spl_Q - α * ΔQ\n",
    "  end # for loop\n",
    "\n",
    "  return P, Q\n",
    "end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.1.1",
   "language": "julia",
   "name": "julia-1.1"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}