Week-11.1_Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# *Unsupervised learning: k-means clustering*\n",
    "\n",
    "[http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html](http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Importing basic packages\n",
    "\n",
    "import os\n",
    "import random\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "## Downloading several hundred New York Times articles as text files\n",
    "\n",
    "os.chdir('/sharedfolder/')\n",
    "\n",
    "!wget -N https://github.com/pcda17/pcda17.github.io/raw/master/week/10/nyt_articles_11-9-2017.zip\n",
    "!unzip -o nyt_articles_11-9-2017.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Loading all text files in the current directory as a list of strings\n",
    "\n",
    "os.chdir('/sharedfolder/nyt_articles_11-9-2017/')\n",
    "\n",
    "document_list = []\n",
    "\n",
    "for filename in [item for item in os.listdir('./') if '.txt' in item]: # Excluding files other than .txt with a list comprehension\n",
    "    text_data = open(filename).read()\n",
    "    document_list.append(text_data)\n",
    "\n",
    "print(len(document_list)) # Printing number of documents in list\n",
    "\n",
    "random.choice(document_list) # Viewing a single document chosen at random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating stop word list\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "\n",
    "stop_words = stopwords.words('english') + [\"'s\", \"'re\", '”', '“', '‘', '’', '—'] + list(string.punctuation)\n",
    "\n",
    "stop_words[:10]     ## Viewing first 10 stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Tokenizing, stemming, and removing stop words from our list of documents\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "documents_filtered = []\n",
    "\n",
    "for document in document_list:\n",
    "    token_list = word_tokenize(document.lower())                                 ## Tokenizing\n",
    "    tokens_filtered = [item for item in token_list if (item not in stop_words)]  ## Removing stop words\n",
    "    tokens_filtered = [stemmer.stem(item) for item in tokens_filtered]           ## Stemming\n",
    "    documents_filtered.append(' '.join(tokens_filtered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Viewing a preprocessed document\n",
    "\n",
    "random.choice(documents_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Vectorizing preprocessed documents\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "\n",
    "X = vectorizer.fit_transform(documents_filtered) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating a vocabulary list corresponding to the vectors we created above\n",
    "\n",
    "vocabulary = vectorizer.get_feature_names()\n",
    "\n",
    "vocabulary[1140:1160]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Clustering documents\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "kmeans_classifier = KMeans(n_clusters=8, init='k-means++', n_init=10, max_iter=500, algorithm='auto')\n",
    "                                   ## ^ docuements to be grouped in 8 clusters \n",
    "\n",
    "cluster_classes = kmeans_classifier.fit_predict(X) \n",
    "\n",
    "cluster_classes[:20]               ## Viewing first 20 cluster assignments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Our cluster assignments and document lists are the same size, in the same order.\n",
    "\n",
    "print(len(cluster_classes))\n",
    "\n",
    "print(len(documents_filtered))\n",
    "\n",
    "print(len(document_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## We can view a document and its assigned cluster by accessing the same index in both lists\n",
    "\n",
    "index_num = 130\n",
    "\n",
    "print('Cluster assignment:')\n",
    "print(cluster_classes[index_num])\n",
    "print()\n",
    "print('Document:')\n",
    "print(document_list[index_num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write each document to a new text file, with each cluster of documents in its own directory\n",
    "\n",
    "try: os.mkdir('/sharedfolder/nyt_clusters_11-9-2017/')\n",
    "except: pass\n",
    "\n",
    "for i in range(len(documents_filtered)):\n",
    "    \n",
    "    out_dir = '/sharedfolder/nyt_clusters_11-9-2017/Cluster_' + str(cluster_classes[i])  ## Creating a directory pathname that\n",
    "                                                                                         ## includes the assigned cluster number.\n",
    "    try: os.mkdir(out_dir)  ## Creating the out_dir directory if it does not yet exist\n",
    "    except: pass\n",
    "    \n",
    "    os.chdir(out_dir)\n",
    "    \n",
    "    out_filename = 'Document_' + str(i) + '.txt'    ## Creating a filename for the text file\n",
    "    \n",
    "    with open(out_filename, 'w') as file_out:\n",
    "        file_out.write(document_list[i])            ## Writing text from original (non-preprocessed) document list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Classifying a new text into an existing cluster\n",
    "\n",
    "input_vector = vectorizer.transform(['Even the budget office is revising its estimates and has predicted the new numbers would be smaller.'])\n",
    "\n",
    "kmeans_classifier.predict(input_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Scatter Plots*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## A simple scatter plot example\n",
    "\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 7)  # Setting default plot size\n",
    "\n",
    "x_vals_1 = [1.92, 1.79, 1.96, 1.4, 1.61, 1.23, 1.43, 1.85, 2.07, 2.24, 2.11, 1.78, 2.21, 1.79, 1.33]\n",
    "y_vals_1 = [2.5, 2.11, 2.19, 1.6, 2.83, 2.55, 2.33, 2.09, 2.32, 2.72, 2.05, 2.4, 2.55, 2.83, 1.58]\n",
    "\n",
    "x_vals_2 = [3.63, 3.12, 3.21, 3.15, 3.56, 3.17, 3.05, 3.14, 2.87, 3.65, 2.82, 3.34, 3.7, 2.95, 3.15]\n",
    "y_vals_2 = [3.1, 4.27, 4.03, 3.37, 3.22, 3.89, 3.27, 2.64, 3.09, 4.1, 3.61, 3.74, 3.71, 3.51, 2.9]\n",
    "\n",
    "\n",
    "plt.scatter(x_vals_1, y_vals_1, label='Class 1', c='indigo', alpha=0.5)\n",
    "\n",
    "plt.scatter(x_vals_2, y_vals_2, label='Class 2', c='blue', alpha=0.5)\n",
    "\n",
    "plt.ylim(ymin=0, ymax=5)\n",
    "plt.xlim(xmin=0, xmax=5)\n",
    "\n",
    "plt.legend(loc=4)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Projecting all vectors to 2 dimensions using linear discriminant analysis (LDA) and viewing a scatter plot\n",
    "\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 7)  # Setting default plot size\n",
    "\n",
    "lda = LDA(n_components=2) #2-dimensional LDA\n",
    "\n",
    "y = cluster_classes\n",
    "\n",
    "lda_transformed = pd.DataFrame(lda.fit_transform(X.toarray(), y))\n",
    "\n",
    "lda_transformed['y'] = y\n",
    "\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==0][0], lda_transformed[lda_transformed['y']==0][1], label='0', c='blue', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==1][0], lda_transformed[lda_transformed['y']==1][1], label='1', c='green', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==2][0], lda_transformed[lda_transformed['y']==2][1], label='2', c='red', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==3][0], lda_transformed[lda_transformed['y']==3][1], label='3', c='violet', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==4][0], lda_transformed[lda_transformed['y']==4][1], label='4', c='orange', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==5][0], lda_transformed[lda_transformed['y']==5][1], label='5', c='magenta', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==6][0], lda_transformed[lda_transformed['y']==6][1], label='6', c='black', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==7][0], lda_transformed[lda_transformed['y']==7][1], label='7', c='indigo', alpha=0.4)\n",
    "\n",
    "plt.legend(loc=2)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Projecting all vectors to 2 dimensions using principal component analysis (PCA) and viewing a scatter plot\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 7)  # Setting default plot size\n",
    "\n",
    "pca = PCA(n_components=2) #2-dimensional LDA\n",
    "\n",
    "pca_transformed = pca.fit_transform(X.toarray())\n",
    "\n",
    "plt.scatter([pair[0] for pair in pca_transformed], [pair[1] for pair in pca_transformed], label='0', c='blue', alpha=0.4)\n",
    "\n",
    "plt.legend(loc=2)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Clustering in topic space*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip3 install lda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lda\n",
    "\n",
    "model = lda.LDA(n_topics=8, n_iter=1000, random_state=1)\n",
    "\n",
    "model.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_features = model.doc_topic_\n",
    "\n",
    "print(len(topic_features))\n",
    "\n",
    "topic_features[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Clustering documents using topic vectors (instead of vocabulary vectors, which we used above)\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "kmeans_topic_classifier = KMeans(n_clusters=8, init='k-means++', n_init=15, max_iter=1000, algorithm='auto')\n",
    "                                         ## ^ docuements to be grouped in 8 clusters \n",
    "\n",
    "topic_cluster_classes = kmeans_topic_classifier.fit_predict(topic_features) \n",
    "\n",
    "print(len(topic_cluster_classes))\n",
    "\n",
    "topic_cluster_classes[:20]               ## Viewing first 20 cluster assignments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write each document to a new text file, with each cluster of documents in its own directory\n",
    "\n",
    "try: os.mkdir('/sharedfolder/nyt_topic_clusters_11-9-2017/')\n",
    "except: pass\n",
    "\n",
    "for i in range(len(documents_filtered)):\n",
    "    \n",
    "    out_dir = '/sharedfolder/nyt_topic_clusters_11-9-2017/Cluster_' + str(topic_cluster_classes[i])  ## Creating a directory pathname that\n",
    "                                                                                         ## includes the assigned cluster number.\n",
    "    try: os.mkdir(out_dir)  ## Creating the out_dir directory if it does not yet exist\n",
    "    except: pass\n",
    "    \n",
    "    os.chdir(out_dir)\n",
    "    \n",
    "    out_filename = 'Document_' + str(i) + '.txt'    ## Creating a filename for the text file\n",
    "    \n",
    "    with open(out_filename, 'w') as file_out:\n",
    "        file_out.write(document_list[i])            ## Writing text from original (non-preprocessed) document list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Projecting all topic vectors to 2 dimensions using linear discriminant analysis (LDA) and viewing a scatter plot\n",
    "\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 7)  # Setting default plot size\n",
    "\n",
    "lda = LDA(n_components=2) #2-dimensional LDA\n",
    "\n",
    "y = topic_cluster_classes\n",
    "\n",
    "lda_transformed = pd.DataFrame(lda.fit_transform(topic_features, y))\n",
    "\n",
    "lda_transformed['y'] = y\n",
    "\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==0][0], lda_transformed[lda_transformed['y']==0][1], label='0', c='blue', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==1][0], lda_transformed[lda_transformed['y']==1][1], label='1', c='green', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==2][0], lda_transformed[lda_transformed['y']==2][1], label='2', c='red', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==3][0], lda_transformed[lda_transformed['y']==3][1], label='3', c='violet', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==4][0], lda_transformed[lda_transformed['y']==4][1], label='4', c='yellow', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==5][0], lda_transformed[lda_transformed['y']==5][1], label='5', c='magenta', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==6][0], lda_transformed[lda_transformed['y']==6][1], label='6', c='black', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==7][0], lda_transformed[lda_transformed['y']==7][1], label='7', c='orange', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==8][0], lda_transformed[lda_transformed['y']==8][1], label='8', c='indigo', alpha=0.4)\n",
    "plt.scatter(lda_transformed[lda_transformed['y']==9][0], lda_transformed[lda_transformed['y']==9][1], label='9', c='purple', alpha=0.4)\n",
    "\n",
    "plt.legend(loc=2)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Viewing the most frequently used words in each topic cluster\n",
    "\n",
    "import collections\n",
    "from pprint import pprint\n",
    "\n",
    "\n",
    "for topic_cluster_id in range(8):\n",
    "    \n",
    "    print('Topic cluster: ' + str(topic_cluster_id))    \n",
    "    \n",
    "    words = []\n",
    "    \n",
    "    for i in range(len(documents_filtered)):\n",
    "        if topic_cluster_classes[i] == topic_cluster_id:\n",
    "            words += documents_filtered[i].split(' ')\n",
    "\n",
    "    counter = collections.Counter(words)\n",
    "    pprint(counter.most_common()[:25])\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}