Week-11.2_Supervised-learning_solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# *Supervised learning*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# Include the line above to hide a cell's text output.\n",
    "\n",
    "## Download sample 'sports' and 'world' articles sets, then unzip.\n",
    "\n",
    "import os\n",
    "\n",
    "os.chdir('/sharedfolder/')\n",
    "\n",
    "!wget -N https://github.com/pcda17/pcda17.github.io/raw/master/week/11/nyt_world_11-19-2017.zip\n",
    "!unzip -o nyt_world_11-19-2017.zip\n",
    "\n",
    "!wget -N https://github.com/pcda17/pcda17.github.io/raw/master/week/11/nyt_sports_11-19-2017.zip\n",
    "!unzip -o nyt_sports_11-19-2017.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45\n"
     ]
    }
   ],
   "source": [
    "## Loading 'world' articles as a list of strings\n",
    "\n",
    "os.chdir('/sharedfolder/nyt_world_11-19-2017/')\n",
    "\n",
    "nyt_world_texts = []\n",
    "\n",
    "for filename in os.listdir('./'):\n",
    "    text_data = open(filename).read().replace('\\n', ' ')\n",
    "    nyt_world_texts.append(text_data)\n",
    "\n",
    "print(len(nyt_world_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "## Loading 'sports' articles as a list of strings\n",
    "\n",
    "os.chdir('/sharedfolder/nyt_sports_11-19-2017/')\n",
    "\n",
    "nyt_sports_texts = []\n",
    "\n",
    "for filename in os.listdir('./'):\n",
    "    text_data = open(filename).read().replace('\\n', ' ')\n",
    "    nyt_sports_texts.append(text_data)\n",
    "\n",
    "print(len(nyt_sports_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set lengths:\n",
      "37\n",
      "32\n",
      "\n",
      "Test set lengths:\n",
      "8\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "## Divide articles into training and test sets (reserving 8 articles for test set)\n",
    "\n",
    "nyt_world_train = nyt_world_texts[:-8]\n",
    "nyt_sports_train = nyt_sports_texts[:-8]\n",
    "\n",
    "nyt_world_test = nyt_world_texts[-8:]\n",
    "nyt_sports_test = nyt_sports_texts[-8:]\n",
    "\n",
    "print('Training set lengths:')\n",
    "print(len(nyt_world_train))\n",
    "print(len(nyt_sports_train))\n",
    "\n",
    "print() ## empty line\n",
    "\n",
    "print('Test set lengths:')\n",
    "print(len(nyt_world_test))\n",
    "print(len(nyt_sports_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Note that the '*' operator can be used to loop a list:\n",
    "\n",
    "repeated_list = [0]*5\n",
    "\n",
    "repeated_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'j', 'j', 'j']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## And we use '+' to concatenate lists:\n",
    "\n",
    "repeated_list_2 = [1]*10 + [0]*9 + ['j']*3\n",
    "\n",
    "repeated_list_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Combing training data\n",
    "\n",
    "combined_texts = nyt_world_train + nyt_sports_train\n",
    "\n",
    "## Creating list of associated class values: \n",
    "## 0 for 'world', 1 for 'sports'\n",
    "\n",
    "y = [0]*len(nyt_world_train) + [1]*len(nyt_sports_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(69, 6891)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Creating vectorized training set using our combined sentence list\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "\n",
    "X = vectorizer.fit_transform(combined_texts)\n",
    "\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Training a multinomial naive Bayes classifier\n",
    "## X is a combined list of 'world' and 'sports' vectors\n",
    "## y is a list of classes (0 or 1)\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "naive_bayes_classifier = MultinomialNB().fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Classifying 'world' test set\n",
    "\n",
    "input_vector = vectorizer.transform(nyt_world_test) ## Converting a list of string to vector format established above\n",
    "\n",
    "naive_bayes_classifier.predict(input_vector) ## Classifying each article in the list. '0' is correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Classifying 'sports' test set\n",
    "\n",
    "input_vector = vectorizer.transform(nyt_sports_test) ## Converting a list of string to vector format established above\n",
    "\n",
    "naive_bayes_classifier.predict(input_vector) ## Classifying each article in the list. '1' is correct.\n",
    "\n",
    "## We'll continue using this set of vectors in the cells below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## k-nearest neighbor classifier\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn_classifier = KNeighborsClassifier(n_neighbors=5)\n",
    "\n",
    "knn_classifier.fit(X, y)\n",
    "\n",
    "knn_classifier.predict(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 1, 1, 0])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Logistic Regression classifier\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr_classifier = LogisticRegression()\n",
    "\n",
    "lr_classifier.fit(X, y)\n",
    "\n",
    "lr_classifier.predict(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Support Vector Machine (SVM) classifer\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_classifier = SVC(kernel='linear')\n",
    "\n",
    "svm_classifier.fit(X, y)\n",
    "\n",
    "svm_classifier.predict(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Multi-layer perceptron classifier (a shallow neural network)\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "mlp_classifier = MLPClassifier()\n",
    "\n",
    "mlp_classifier.fit(X, y)\n",
    "\n",
    "mlp_classifier.predict(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Assignment *\n",
    "    \n",
    "    Write some code that downloads a live New York Times page and classifies it as 'world' or 'sports'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "!apt-get -y install libxml2-dev libxslt-dev \n",
    "!pip3 install newspaper3k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Using the 'newspaper' package to extract article text\n",
    "\n",
    "from newspaper import Article\n",
    "\n",
    "def url_to_article_text(url):\n",
    "    article = Article(url)\n",
    "    article.download()\n",
    "    article.parse()\n",
    "    article_text = article.text.replace('\\n', ' ')\n",
    "    return article_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'It gave a jolt of momentum to the Patriots (8-2), as if they needed it in a blowout that laid bare Oakland’s anemic defense. Oakland, trailing by 30-0, finally scored in the fourth quarter, when Amari Cooper scored a touchdown on a pass from Derek Carr.  Photo  The win improved the Patriots chances for the top seed in the A.F.C. playoffs. As the game wound down, thick storm clouds menaced, mirroring the increasingly gloomy postseason outlook for Oakland (4-6).  No matter.  The announced crowd of 77,000 treated this event like a Super Bowl nearly the entire time: The fans’ excitement was palpable; their face paint, jerseys and carousing inside and outside the stadium a testament to their anticipation for this game.  “If I were in Oakland, I would care more if they won or lost, but just that they are playing here, I cannot describe how that feels,” said Emilio Carreño, 28, a government accountant and Raiders fan. “To see the Raiders here in Azteca is something I will remember for a long time.”  Just about everything on the field drew an emotional, throaty response: the unfurling of giant flags of the United States and Mexico for the anthems before the game; the halftime salute to rescue teams — in their protective gear and with their search dogs — from the earthquakes in September that killed more than 450 people, most of them in and around Mexico City; even dropped balls and middling runs from both teams.  The grounds crew surely would have been cheered, too, if the workers were seen.  Even as New England was running away with the game, the crowd stuck with it, doing the wave so vigorously the stadium shook.  Another regular-season game will be played here in 2018, the third since 2016, when Oakland beat the Houston Texans, 27-20. Shortly before this game, the N.F.L. and its broadcast partner here and stadium landlord, Televisa, announced that three more regular-season games will be played from 2019 to 2021.  Photo  The N.F.L. is striving to expand its presence inter'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.nytimes.com/2017/11/19/sports/patriots-beat-raiders-mexico.html'\n",
    "\n",
    "article_text = url_to_article_text(url)\n",
    "\n",
    "article_text[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Solution*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.nytimes.com/2017/11/19/sports/patriots-beat-raiders-mexico.html\n",
      "naive Bayes: [1]\n",
      "k-nearest neighbors: [0]\n",
      "logistic regression: [1]\n",
      "support vector machine: [1]\n",
      "multi-layer perceptron classifier: [1]\n",
      "\n",
      "https://www.nytimes.com/2017/11/21/world/middleeast/syria-damascus-war.html\n",
      "naive Bayes: [0]\n",
      "k-nearest neighbors: [0]\n",
      "logistic regression: [0]\n",
      "support vector machine: [0]\n",
      "multi-layer perceptron classifier: [0]\n",
      "\n",
      "https://www.washingtonpost.com/world/asia_pacific/with-technology-these-researchers-are-figuring-out-north-koreas-nuclear-secrets/2017/11/20/274d9786-c9e2-11e7-b244-2d22ac912500_story.html\n",
      "naive Bayes: [0]\n",
      "k-nearest neighbors: [0]\n",
      "logistic regression: [0]\n",
      "support vector machine: [1]\n",
      "multi-layer perceptron classifier: [0]\n",
      "\n",
      "https://www.washingtonpost.com/news/football-insider/wp/2017/11/20/we-have-so-many-question-marks-redskins-starters-ailing-at-safety-on-both-lines/\n",
      "naive Bayes: [1]\n",
      "k-nearest neighbors: [1]\n",
      "logistic regression: [1]\n",
      "support vector machine: [1]\n",
      "multi-layer perceptron classifier: [1]\n",
      "\n",
      "https://www.nytimes.com/2017/11/21/movies/mr-roosevelt-review-noel-wells.html\n",
      "naive Bayes: [0]\n",
      "k-nearest neighbors: [1]\n",
      "logistic regression: [0]\n",
      "support vector machine: [0]\n",
      "multi-layer perceptron classifier: [0]\n",
      "\n",
      "https://www.washingtonpost.com/entertainment/music/silly-humor-of-dated-plot-underlines-joy-of-opera/2017/11/20/c5521c3c-ce1e-11e7-9d3a-bcbe2af58c3a_story.html\n",
      "naive Bayes: [1]\n",
      "k-nearest neighbors: [0]\n",
      "logistic regression: [1]\n",
      "support vector machine: [0]\n",
      "multi-layer perceptron classifier: [0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Classifying a list of articles 5 ways\n",
    "\n",
    "urls = ['https://www.nytimes.com/2017/11/19/sports/patriots-beat-raiders-mexico.html', \n",
    "        'https://www.nytimes.com/2017/11/21/world/middleeast/syria-damascus-war.html',\n",
    "        'https://www.washingtonpost.com/world/asia_pacific/with-technology-these-researchers-are-figuring-out-north-koreas-nuclear-secrets/2017/11/20/274d9786-c9e2-11e7-b244-2d22ac912500_story.html',\n",
    "        'https://www.washingtonpost.com/news/football-insider/wp/2017/11/20/we-have-so-many-question-marks-redskins-starters-ailing-at-safety-on-both-lines/',\n",
    "        'https://www.nytimes.com/2017/11/21/movies/mr-roosevelt-review-noel-wells.html',\n",
    "        'https://www.washingtonpost.com/entertainment/music/silly-humor-of-dated-plot-underlines-joy-of-opera/2017/11/20/c5521c3c-ce1e-11e7-9d3a-bcbe2af58c3a_story.html']\n",
    "\n",
    "for url in urls:\n",
    "    article_text = url_to_article_text(url)     ## Get the article at a given URL as a string.\n",
    "    input_vector = vectorizer.transform([article_text])[0] ## Vectorize article.\n",
    "    print(url)                        # ^ Enclosing the string in a list, because that's what vectorizer.transform() expects. \n",
    "    print('naive Bayes: ' + str(naive_bayes_classifier.predict(input_vector)))\n",
    "    print('k-nearest neighbors: ' + str(knn_classifier.predict(input_vector)))\n",
    "    print('logistic regression: ' + str(lr_classifier.predict(input_vector)))\n",
    "    print('support vector machine: ' + str(svm_classifier.predict(input_vector)))\n",
    "    print('multi-layer perceptron classifier: ' + str(mlp_classifier.predict(input_vector)))\n",
    "    print()\n",
    "\n",
    "## Recall that '0' = 'world' and '1' = 'sports'\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.nytimes.com/2017/11/19/sports/patriots-beat-raiders-mexico.html\n",
      "naive Bayes: sports\n",
      "k-nearest neighbors: world\n",
      "logistic regression: sports\n",
      "support vector machine: sports\n",
      "multi-layer perceptron classifier: sports\n",
      "\n",
      "https://www.nytimes.com/2017/11/21/world/middleeast/syria-damascus-war.html\n",
      "naive Bayes: world\n",
      "k-nearest neighbors: world\n",
      "logistic regression: world\n",
      "support vector machine: world\n",
      "multi-layer perceptron classifier: world\n",
      "\n",
      "https://www.washingtonpost.com/world/asia_pacific/with-technology-these-researchers-are-figuring-out-north-koreas-nuclear-secrets/2017/11/20/274d9786-c9e2-11e7-b244-2d22ac912500_story.html\n",
      "naive Bayes: world\n",
      "k-nearest neighbors: world\n",
      "logistic regression: world\n",
      "support vector machine: sports\n",
      "multi-layer perceptron classifier: world\n",
      "\n",
      "https://www.washingtonpost.com/news/football-insider/wp/2017/11/20/we-have-so-many-question-marks-redskins-starters-ailing-at-safety-on-both-lines/\n",
      "naive Bayes: sports\n",
      "k-nearest neighbors: sports\n",
      "logistic regression: sports\n",
      "support vector machine: sports\n",
      "multi-layer perceptron classifier: sports\n",
      "\n",
      "https://www.nytimes.com/2017/11/21/movies/mr-roosevelt-review-noel-wells.html\n",
      "naive Bayes: world\n",
      "k-nearest neighbors: sports\n",
      "logistic regression: world\n",
      "support vector machine: world\n",
      "multi-layer perceptron classifier: world\n",
      "\n",
      "https://www.washingtonpost.com/entertainment/music/silly-humor-of-dated-plot-underlines-joy-of-opera/2017/11/20/c5521c3c-ce1e-11e7-9d3a-bcbe2af58c3a_story.html\n",
      "naive Bayes: sports\n",
      "k-nearest neighbors: world\n",
      "logistic regression: sports\n",
      "support vector machine: world\n",
      "multi-layer perceptron classifier: world\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Classifying a list of articles 5 ways\n",
    "\n",
    "## If we define a list of classes (i.e., ['world', 'sports']), we can display class names instead of 0s and 1s.\n",
    "\n",
    "urls = ['https://www.nytimes.com/2017/11/19/sports/patriots-beat-raiders-mexico.html', \n",
    "        'https://www.nytimes.com/2017/11/21/world/middleeast/syria-damascus-war.html',\n",
    "        'https://www.washingtonpost.com/world/asia_pacific/with-technology-these-researchers-are-figuring-out-north-koreas-nuclear-secrets/2017/11/20/274d9786-c9e2-11e7-b244-2d22ac912500_story.html',\n",
    "        'https://www.washingtonpost.com/news/football-insider/wp/2017/11/20/we-have-so-many-question-marks-redskins-starters-ailing-at-safety-on-both-lines/',\n",
    "        'https://www.nytimes.com/2017/11/21/movies/mr-roosevelt-review-noel-wells.html',\n",
    "        'https://www.washingtonpost.com/entertainment/music/silly-humor-of-dated-plot-underlines-joy-of-opera/2017/11/20/c5521c3c-ce1e-11e7-9d3a-bcbe2af58c3a_story.html']\n",
    "\n",
    "for url in urls:\n",
    "    article_text = url_to_article_text(url)     ## Get the article at a given URL as a string.\n",
    "    input_vector = vectorizer.transform([article_text])[0] ## Vectorize article.\n",
    "    classes = ['world', 'sports']\n",
    "    print(url)                        # ^ Enclosing the string in a list, because that's what vectorizer.transform() expects. \n",
    "    print('naive Bayes: ' + str(classes[naive_bayes_classifier.predict(input_vector)[0]]))\n",
    "    print('k-nearest neighbors: ' + str(classes[knn_classifier.predict(input_vector)[0]]))\n",
    "    print('logistic regression: ' + str(classes[lr_classifier.predict(input_vector)[0]]))\n",
    "    print('support vector machine: ' + str(classes[svm_classifier.predict(input_vector)[0]]))\n",
    "    print('multi-layer perceptron classifier: ' + str(classes[mlp_classifier.predict(input_vector)[0]]))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}