1.train-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import sys\n",
    "import numpy as np\n",
    "from scipy.sparse import csr_matrix\n",
    "import xgboost as xgb\n",
    "from sklearn.externals import joblib\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eliminate_whitespace(somestr):\n",
    "    newstr = re.sub(r\"[\\n\\t\\s]*\", \"\", somestr) #get rid of all whitespace characters including newline\n",
    "    return newstr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_all_4grams(line_of_chars):\n",
    "    thelist = []\n",
    "    numchars = len(line_of_chars)\n",
    "    num4grams = (numchars - 4) + 1 #given the number of chars in line_of_chars, \n",
    "                                   #this is how many 4grams we'll find within it\n",
    "    for i in range(num4grams):\n",
    "        fourgram = line_of_chars[i:i+4]\n",
    "        thelist.append(fourgram)\n",
    "    \n",
    "    return(thelist)\n",
    "\n",
    "def createLabels(someLine, list_of_4grams):\n",
    "    someLineList = list(someLine)\n",
    "    theLabels = []\n",
    "    ptr = 0\n",
    "    for fourGram in list_of_4grams:\n",
    "        if someLineList[ptr + 2] == ' ':\n",
    "            theLabels.append(1)\n",
    "            someLineList.pop(ptr+2) #get rid of that space we just accounted for\n",
    "        else:\n",
    "            theLabels.append(0)\n",
    "            \n",
    "        ptr = ptr + 1\n",
    "            \n",
    "    return theLabels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def findIndexOfBigram(bigram, alphabet, unk_token):\n",
    "    alphabetLength = len(alphabet)\n",
    "    firstChar = bigram[0]\n",
    "    secondChar = bigram[1]\n",
    "    if (firstChar not in alphabet):\n",
    "        firstChar = unk_token\n",
    "    if(secondChar not in alphabet):\n",
    "        secondChar = unk_token\n",
    "    \n",
    "    firstCharMultiplier = alphabet.index(firstChar)\n",
    "    secondCharMultiplier = alphabet.index(secondChar)\n",
    "    uniqueIndexOfBigram = firstCharMultiplier*alphabetLength**1 + secondCharMultiplier*alphabetLength**0\n",
    "    return uniqueIndexOfBigram\n",
    "\n",
    "def findBigramOfIndex(index, alphabet):\n",
    "    alphabetLength = len(alphabet)\n",
    "    row = index // alphabetLength\n",
    "    col = index % alphabetLength\n",
    "    firstchar = alphabet[row]\n",
    "    secondchar = alphabet[col]\n",
    "    return firstchar + secondchar\n",
    "\n",
    "def findIndexOfUnigram(unigram, alphabet, unk_token):\n",
    "    if unigram not in alphabet:\n",
    "        unigram = unk_token\n",
    "    theindex = alphabet.index(unigram)\n",
    "    return theindex\n",
    "\n",
    "def findUnigramOfIndex(index, alphabet):\n",
    "    theunigram = alphabet[index]\n",
    "    return theunigram\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create5dStrFeatureVector(fourGram):\n",
    "    f1 = fourGram[0] + fourGram[1]\n",
    "    f2 = fourGram[1]\n",
    "    f3 = fourGram[1] + fourGram[2]\n",
    "    f4 = fourGram[2]\n",
    "    f5 = fourGram[2] + fourGram[3]\n",
    "    featureVector = [f1,f2,f3,f4,f5]\n",
    "    return featureVector\n",
    "\n",
    "def create5dIntFeatureVector(strFeatureVector, alphabet, unk_token):\n",
    "    lengthAlphabet = len(alphabet)\n",
    "    lengthBigramPartOfVector = lengthAlphabet * lengthAlphabet\n",
    "    lengthUnigramPartOfVector = lengthAlphabet\n",
    "    lengthOneHotVector = lengthBigramPartOfVector + lengthUnigramPartOfVector\n",
    "\n",
    "    f1,f2,f3,f4,f5 = strFeatureVector\n",
    "    f1int = findIndexOfBigram(f1, alphabet, unk_token) \n",
    "    f2int = findIndexOfUnigram(f2, alphabet, unk_token) + lengthBigramPartOfVector\n",
    "    f3int = findIndexOfBigram(f3, alphabet, unk_token) \n",
    "    f4int = findIndexOfUnigram(f4, alphabet, unk_token) + lengthBigramPartOfVector\n",
    "    f5int = findIndexOfBigram(f5, alphabet, unk_token) \n",
    "    theVector = [f1int,f2int,f3int,f4int,f5int]\n",
    "    return theVector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadChineseDataFile(filepath, numLines=None):\n",
    "    encoding = 'big5hkscs'\n",
    "    lines = []\n",
    "    num_errors = 0\n",
    "    #Need the 'rb' argument below when opening the file. This info, thanks to:\n",
    "    #https://stackoverflow.com/questions/22216076/unicodedecodeerror-utf8-codec-cant-decode-byte-0xa5-in-position-0-invalid-s\n",
    "    linecounter = 1\n",
    "\n",
    "    for line in open(filepath, 'rb'):\n",
    "        try:\n",
    "            decodedLine = line.decode(encoding)\n",
    "            cleanLine = ' '.join(decodedLine.split()) #replace double spaces with a single space\n",
    "            cleanLine2 = re.sub(r\"[\\n]*\", \"\", cleanLine) #get rid of newline character\n",
    "            lines.append(cleanLine2)\n",
    "        except UnicodeDecodeError as e:\n",
    "            num_errors += 1\n",
    "            print(\"error encountered at line\", linecounter)\n",
    "        linecounter = linecounter + 1\n",
    "        \n",
    "        if numLines is not None:\n",
    "            if linecounter > numLines:\n",
    "                break\n",
    "\n",
    "    print('Encountered %d decoding errors.' % num_errors)\n",
    "    # The `lines` list contains strings you can use.\n",
    "    return lines\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def createAlphabet(lines, unk_token, start_of_line_token, end_of_line_token):\n",
    "\n",
    "    unique_chinese_chars_set = set()\n",
    "\n",
    "    for line in lines:\n",
    "        somestr = re.sub(r\"[\\n\\t\\s]*\", \"\", line) #get rid of whitespace characters\n",
    "        for somechar in somestr:\n",
    "            unique_chinese_chars_set.add(somechar)\n",
    "\n",
    "    #Let's make sure the characters i want to use as special tokens don't already occur in the\n",
    "    #training data\n",
    "\n",
    "    if start_of_line_token in unique_chinese_chars_set:\n",
    "        print(\"error. The char\", start_of_line_token, \"which I use as a special token, appears as\")\n",
    "        print(\"a regular char in the datafile. Do not proceed!\")\n",
    "\n",
    "    if end_of_line_token in unique_chinese_chars_set:\n",
    "        print(\"error. The char\", end_of_line_token, \"which I use as a special token, appears as\")\n",
    "        print(\"a regular char in the datafile. Do not proceed!\")\n",
    "        \n",
    "    chinese_alphabet = [unk_token] + [start_of_line_token] + [end_of_line_token] + list(unique_chinese_chars_set) #we cast the set into a list here\n",
    "    return chinese_alphabet\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def buildSparseMatrix(lines, start_of_line_token, end_of_line_token, unk_token):\n",
    "    row = np.array([])\n",
    "    col = np.array([])\n",
    "    humanLabelsNumpy = np.array([])\n",
    "    sparseMatrixRowPtr = 0\n",
    "    lineNumber = 0\n",
    "\n",
    "    for someLineOriginal in lines:\n",
    "        if (lineNumber % 1000 == 0):\n",
    "            print(\"processing line\", lineNumber, \"...\")\n",
    "\n",
    "        someLine = start_of_line_token + someLineOriginal #we need a start of line token to form proper feature vector for \n",
    "        #first char in a line\n",
    "\n",
    "        someLine = someLine + end_of_line_token #And we need a end of sentence token to form proper feature vector for\n",
    "        #the last char in a line\n",
    "        #print(\"Here is a line from the file:\", someLine)\n",
    "\n",
    "        someLineSansSpaces = eliminate_whitespace(someLine)\n",
    "        all_the_4grams = []\n",
    "\n",
    "        if len(someLine) >=3:\n",
    "            all_the_4grams = find_all_4grams(someLineSansSpaces)\n",
    "        else:\n",
    "            print(\"An error occurred. Some line in the training file is either blank or has just a\")\n",
    "            print(\"single Chinese character on it without a punctuation mark at the end. This breaks\")\n",
    "            print(\"an important assumption I made about the data.\")\n",
    "            sys.exit()\n",
    "\n",
    "        #print(\"Here are all its 4grams:\", all_the_4grams)\n",
    "        labels = createLabels(someLine, all_the_4grams)\n",
    "        labels = np.asarray(labels)\n",
    "        #print(\"Here are the labels for the 4grams:\", labels)\n",
    "\n",
    "        humanLabelsNumpy = np.concatenate((humanLabelsNumpy, labels))\n",
    "        #print(\"the labels for the 4grams:\", labels)\n",
    "        #print(\"------------------\")\n",
    "        for i in range(len(all_the_4grams)):\n",
    "            ngram = all_the_4grams[i]\n",
    "            #an ngram will look like ABCD for instance\n",
    "            strFeatureVector = create5dStrFeatureVector(ngram)\n",
    "            #print(\"Here is the 5d feature vector string form:\", strFeatureVector)\n",
    "            intFeatureVector = create5dIntFeatureVector(strFeatureVector, chinese_alphabet, unk_token)\n",
    "            #print(\"Here is the 5d feature vector int form:\", intFeatureVector)\n",
    "            #print(\"---------\")\n",
    "            intFeatureVectorNumpy = np.asarray(intFeatureVector)\n",
    "\n",
    "            temp = np.ones((5), dtype=int) * sparseMatrixRowPtr\n",
    "            row = np.concatenate((row, temp)) #which row in the sparse matrix we're building right now\n",
    "            col = np.concatenate((col, intFeatureVectorNumpy)) #the col indexes that will have a 1 in them\n",
    "\n",
    "            sparseMatrixRowPtr = sparseMatrixRowPtr + 1\n",
    "\n",
    "        lineNumber = lineNumber + 1\n",
    "\n",
    "    print(\"The number of 5d feature vectors built was:\", sparseMatrixRowPtr)\n",
    "    #print(humanLabelsNumpy.shape)\n",
    "    #print(row.shape) #row and col should have same length\n",
    "    #print(col.shape)\n",
    "\n",
    "    #We can now build the sparse matrix\n",
    "    lengthAlphabet = len(chinese_alphabet)\n",
    "    lengthBigramPartOfVector = lengthAlphabet * lengthAlphabet\n",
    "    lengthUnigramPartOfVector = lengthAlphabet\n",
    "    lengthOneHotVector = lengthBigramPartOfVector + lengthUnigramPartOfVector\n",
    "\n",
    "    data = np.ones_like(row) #We're building one-hot vector so non-zero entries in the matrix will have\n",
    "    #values of 1\n",
    "    \n",
    "    sparseOneHotMatrix = csr_matrix((data, (row, col)), shape=(sparseMatrixRowPtr, lengthOneHotVector))\n",
    "    dtrain = xgb.DMatrix(sparseOneHotMatrix,label=humanLabelsNumpy) #need to convert the sparse matrix \n",
    "    #to a dmatrix that the xgboost module understands\n",
    "\n",
    "    return (dtrain,humanLabelsNumpy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluateResults(classifierLabels, humanLabels):\n",
    "    assert len(classifierLabels) == len(humanLabels)\n",
    "    #these two lists have to have the same length\n",
    "    \n",
    "    numPredictions = len(classifierLabels)\n",
    "    truePositives = 0\n",
    "    trueNegatives = 0\n",
    "\n",
    "    falsePositives = 0\n",
    "    falseNegatives = 0\n",
    "    numCorrect = 0\n",
    "    numWrong = 0\n",
    "\n",
    "    for i in range(len(humanLabels)):\n",
    "        if (humanLabels[i]==1):\n",
    "            if (classifierLabels[i] == 1):\n",
    "                truePositives = truePositives + 1\n",
    "                numCorrect = numCorrect + 1\n",
    "            elif(classifierLabels[i] == 0):\n",
    "                falseNegatives = falseNegatives + 1\n",
    "                numWrong = numWrong + 1\n",
    "        elif(humanLabels[i]==0):\n",
    "            if (classifierLabels[i] == 0):\n",
    "                trueNegatives = trueNegatives + 1\n",
    "                numCorrect = numCorrect + 1\n",
    "            elif(classifierLabels[i]==1):\n",
    "                falsePositives = falsePositives + 1\n",
    "                numWrong = numWrong + 1\n",
    "\n",
    "\n",
    "    #print(\"true positives:\", truePositives)\n",
    "    #print(\"false negatives:\", falseNegatives)\n",
    "    #print(\"false positives:\", falsePositives)\n",
    "    #print()\n",
    "\n",
    "    accuracy = numCorrect/numPredictions\n",
    "    precision = truePositives/(truePositives + falsePositives)\n",
    "    recall = truePositives/(truePositives + falseNegatives)\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    return (accuracy,precision,recall, f1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's load the training set into a list called lines. Each line of the file becomes an item in this list.\n",
    "There are 745,806 (clean) lines in it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numberLinesForTrainAndDevSets = 100000\n",
    "numTrainLines = 90000\n",
    "numDevLines = 10000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encountered 0 decoding errors.\n",
      "100000\n"
     ]
    }
   ],
   "source": [
    "trainAndDevLines = loadChineseDataFile(\"./data/training.txt\", numberLinesForTrainAndDevSets)\n",
    "print(len(trainAndDevLines))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90000\n",
      "10000\n"
     ]
    }
   ],
   "source": [
    "trainLines = trainAndDevLines[0:numTrainLines]\n",
    "devLines = trainAndDevLines[-numDevLines:]\n",
    "print(len(trainLines))\n",
    "print(len(devLines))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's take a look at the first few lines of the file that we loaded into the list called lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "時間 ：\n",
      "三月 十日 （ 星期四 ） 上午 十時 。\n",
      "地點 ：\n",
      "學術 活動 中心 一樓 簡報室 。\n",
      "主講 ：\n",
      "民族所 所長 莊英章 先生 。\n",
      "講題 ：\n",
      "閩 、 台 漢人 社會 研究 的 若干 考察 。\n",
      "李 院長 於 二月 二十六日 至 三月 十五日 赴 美 訪問 ，\n"
     ]
    }
   ],
   "source": [
    "for line in trainLines[0:9]:\n",
    "    print(line)\n",
    "    \n",
    "#As you can see below, each line ends with a punctuation mark and the punctuation mark has \n",
    "#a whitespace before it. I'm assuming every line in the file is like this. An error will occur\n",
    "#otherwise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Line 8 up above is a special case that needs to be addressed. The first character in that line is a word but there's no character to the left of it so how would a 5d feature vector be built here to discover that a space should come after that first char? <br /> <br />\n",
    "Here's an example with English letters to clarify what I mean. Let's say the stream of characters on a line is: <br /> \n",
    "ABCDEFGH <br /> \n",
    "and the correct spacing is: <br /> \n",
    "A BCDEFGH <br />\n",
    "\n",
    "There isn't a character to the left of A so how would be build that 5 dimensional feature vector for A and find out that a space should come right after it? The answer is we need a start of line token. Let's use the carrot character ^ as the start of line token. <br /> <br />\n",
    "A similar argument can be made for the importance of an end-of-line token - how it's needed to determine if a space should appear before the last character in a line.\n",
    "We manually add these special tokens to the alphabet in the function createAlphabet()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's create the alphabet here. We add all the characters in a data file to a set in order to get rid of duplicates. We then convert the set to a list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unk_token = '*' #The '*' char will represent an unknown char (i.e. unseen in the training data)\n",
    "start_of_line_token = '^' #The '^' char will represent the start of a sentence\n",
    "end_of_line_token = '$'\n",
    "chinese_alphabet = createAlphabet(trainAndDevLines, unk_token, start_of_line_token, end_of_line_token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4660\n"
     ]
    }
   ],
   "source": [
    "print(len(chinese_alphabet))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['*', '^', '$', '鈴', '甦', '特', '藻', '並', '燹', '暨']\n"
     ]
    }
   ],
   "source": [
    "#take a look at the first few characters in the alphabet\n",
    "print(chinese_alphabet[0:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['chinese_alphabet.pickle']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(chinese_alphabet, 'chinese_alphabet.pickle', compress=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's create the feature vector for each 4gram and its label (i.e. let's create the training, dev and test sets)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing line 0 ...\n",
      "processing line 1000 ...\n",
      "processing line 2000 ...\n",
      "processing line 3000 ...\n",
      "processing line 4000 ...\n",
      "processing line 5000 ...\n",
      "processing line 6000 ...\n",
      "processing line 7000 ...\n",
      "processing line 8000 ...\n",
      "processing line 9000 ...\n",
      "processing line 10000 ...\n",
      "processing line 11000 ...\n",
      "processing line 12000 ...\n",
      "processing line 13000 ...\n",
      "processing line 14000 ...\n",
      "processing line 15000 ...\n",
      "processing line 16000 ...\n",
      "processing line 17000 ...\n",
      "processing line 18000 ...\n",
      "processing line 19000 ...\n",
      "processing line 20000 ...\n",
      "processing line 21000 ...\n",
      "processing line 22000 ...\n",
      "processing line 23000 ...\n",
      "processing line 24000 ...\n",
      "processing line 25000 ...\n",
      "processing line 26000 ...\n",
      "processing line 27000 ...\n",
      "processing line 28000 ...\n",
      "processing line 29000 ...\n",
      "processing line 30000 ...\n",
      "processing line 31000 ...\n",
      "processing line 32000 ...\n",
      "processing line 33000 ...\n",
      "processing line 34000 ...\n",
      "processing line 35000 ...\n",
      "processing line 36000 ...\n",
      "processing line 37000 ...\n",
      "processing line 38000 ...\n",
      "processing line 39000 ...\n",
      "processing line 40000 ...\n",
      "processing line 41000 ...\n",
      "processing line 42000 ...\n",
      "processing line 43000 ...\n",
      "processing line 44000 ...\n",
      "processing line 45000 ...\n",
      "processing line 46000 ...\n",
      "processing line 47000 ...\n",
      "processing line 48000 ...\n",
      "processing line 49000 ...\n",
      "processing line 50000 ...\n",
      "processing line 51000 ...\n",
      "processing line 52000 ...\n",
      "processing line 53000 ...\n",
      "processing line 54000 ...\n",
      "processing line 55000 ...\n",
      "processing line 56000 ...\n",
      "processing line 57000 ...\n",
      "processing line 58000 ...\n",
      "processing line 59000 ...\n",
      "processing line 60000 ...\n",
      "processing line 61000 ...\n",
      "processing line 62000 ...\n",
      "processing line 63000 ...\n",
      "processing line 64000 ...\n",
      "processing line 65000 ...\n",
      "processing line 66000 ...\n",
      "processing line 67000 ...\n",
      "processing line 68000 ...\n",
      "processing line 69000 ...\n",
      "processing line 70000 ...\n",
      "processing line 71000 ...\n",
      "processing line 72000 ...\n",
      "processing line 73000 ...\n",
      "processing line 74000 ...\n",
      "processing line 75000 ...\n",
      "processing line 76000 ...\n",
      "processing line 77000 ...\n",
      "processing line 78000 ...\n",
      "processing line 79000 ...\n",
      "processing line 80000 ...\n",
      "processing line 81000 ...\n",
      "processing line 82000 ...\n",
      "processing line 83000 ...\n",
      "processing line 84000 ...\n",
      "processing line 85000 ...\n",
      "processing line 86000 ...\n",
      "processing line 87000 ...\n",
      "processing line 88000 ...\n",
      "processing line 89000 ...\n",
      "The number of 5d feature vectors built was: 996924\n"
     ]
    }
   ],
   "source": [
    "(dtrain, ytrain) = buildSparseMatrix(trainLines, start_of_line_token, end_of_line_token, unk_token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing line 0 ...\n",
      "processing line 1000 ...\n",
      "processing line 2000 ...\n",
      "processing line 3000 ...\n",
      "processing line 4000 ...\n",
      "processing line 5000 ...\n",
      "processing line 6000 ...\n",
      "processing line 7000 ...\n",
      "processing line 8000 ...\n",
      "processing line 9000 ...\n",
      "The number of 5d feature vectors built was: 121960\n"
     ]
    }
   ],
   "source": [
    "#Now let's create the validation set\n",
    "(dvalid, yvalid) = buildSparseMatrix(devLines, start_of_line_token, end_of_line_token, unk_token)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Let's implement and train an xgboost decision tree classifier on the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#info from https://www.kdnuggets.com/2017/03/simple-xgboost-tutorial-iris-dataset.html\n",
    "\n",
    "num_round = 40  # the number of training iterations\n",
    "param = {\n",
    "    'max_depth': 20,  # the maximum depth of each tree\n",
    "    'eta': 0.3,  # the training step for each iteration\n",
    "    'silent': 1,  # logging mode - quiet\n",
    "    'objective': 'binary:logistic' # learning objective is binary classification\n",
    "     }  \n",
    "\n",
    "param['eval_metric'] = 'error'\n",
    "#note that you are not supposed to use num_class with 'binary:logistic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 40 epochs\n"
     ]
    }
   ],
   "source": [
    "print(\"Training for\", num_round, \"epochs\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\teval-error:0.31726\ttrain-error:0.324748\n",
      "[1]\teval-error:0.294228\ttrain-error:0.29754\n",
      "[2]\teval-error:0.273073\ttrain-error:0.274109\n",
      "[3]\teval-error:0.262193\ttrain-error:0.262916\n",
      "[4]\teval-error:0.249606\ttrain-error:0.244611\n",
      "[5]\teval-error:0.24135\ttrain-error:0.236876\n",
      "[6]\teval-error:0.232101\ttrain-error:0.22577\n",
      "[7]\teval-error:0.226722\ttrain-error:0.218999\n",
      "[8]\teval-error:0.218112\ttrain-error:0.20735\n",
      "[9]\teval-error:0.211504\ttrain-error:0.197408\n",
      "[10]\teval-error:0.196015\ttrain-error:0.18136\n",
      "[11]\teval-error:0.192899\ttrain-error:0.178059\n",
      "[12]\teval-error:0.191432\ttrain-error:0.176252\n",
      "[13]\teval-error:0.188972\ttrain-error:0.175557\n",
      "[14]\teval-error:0.185643\ttrain-error:0.169669\n",
      "[15]\teval-error:0.181076\ttrain-error:0.164024\n",
      "[16]\teval-error:0.1786\ttrain-error:0.161765\n",
      "[17]\teval-error:0.17714\ttrain-error:0.160921\n",
      "[18]\teval-error:0.175558\ttrain-error:0.159366\n",
      "[19]\teval-error:0.172614\ttrain-error:0.156471\n",
      "[20]\teval-error:0.170326\ttrain-error:0.154205\n",
      "[21]\teval-error:0.169392\ttrain-error:0.153121\n",
      "[22]\teval-error:0.169244\ttrain-error:0.152029\n",
      "[23]\teval-error:0.165431\ttrain-error:0.148753\n",
      "[24]\teval-error:0.163455\ttrain-error:0.145958\n",
      "[25]\teval-error:0.161865\ttrain-error:0.143633\n",
      "[26]\teval-error:0.16011\ttrain-error:0.139766\n",
      "[27]\teval-error:0.156789\ttrain-error:0.136099\n",
      "[28]\teval-error:0.156043\ttrain-error:0.135424\n",
      "[29]\teval-error:0.155149\ttrain-error:0.134515\n",
      "[30]\teval-error:0.151255\ttrain-error:0.131015\n",
      "[31]\teval-error:0.151156\ttrain-error:0.131256\n",
      "[32]\teval-error:0.14927\ttrain-error:0.129587\n",
      "[33]\teval-error:0.148172\ttrain-error:0.128052\n",
      "[34]\teval-error:0.150459\ttrain-error:0.130276\n",
      "[35]\teval-error:0.149631\ttrain-error:0.127637\n",
      "[36]\teval-error:0.148549\ttrain-error:0.125849\n",
      "[37]\teval-error:0.147073\ttrain-error:0.124301\n",
      "[38]\teval-error:0.146269\ttrain-error:0.124155\n",
      "[39]\teval-error:0.146171\ttrain-error:0.123679\n"
     ]
    }
   ],
   "source": [
    "evallist = [(dvalid, 'eval'), (dtrain, 'train')]\n",
    "bst = xgb.train(param, dtrain, num_round, evallist)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bst.pickle']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's save the model\n",
    "joblib.dump(bst, 'bst.pickle', compress=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's evaluate the trained model\n",
    "You can load the trained model and its alphabet in the next cell if you need to"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bst = joblib.load('chinese-xgb-model-final.pickle') #load the saved model\n",
    "chinese_alphabet = joblib.load('chinese-alphabet-final.pickle') #load the saved alphabet\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encountered 0 decoding errors.\n",
      "processing line 0 ...\n",
      "processing line 1000 ...\n",
      "The number of 5d feature vectors built was: 17345\n"
     ]
    }
   ],
   "source": [
    "#Let's create the test set\n",
    "testLines = loadChineseDataFile(\"./data/test.txt\")\n",
    "(dtest, ytest) = buildSparseMatrix(testLines, start_of_line_token, end_of_line_token, unk_token)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8608776  0.3243633  0.3243633  ... 0.3243633  0.3243633  0.99747854]\n"
     ]
    }
   ],
   "source": [
    "preds = bst.predict(dtest)\n",
    "print(preds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17345\n",
      "[ True False False ... False False  True]\n"
     ]
    }
   ],
   "source": [
    "predsBinary = (preds >=0.5)\n",
    "print(len(predsBinary))\n",
    "print(predsBinary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 0. 0. ... 1. 0. 1.]\n"
     ]
    }
   ],
   "source": [
    "print(ytest)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8504468146439896\n",
      "precision: 0.9468858324946886\n",
      "recall: 0.7998488712572023\n",
      "F1 score: 0.867178699436764\n"
     ]
    }
   ],
   "source": [
    "acc,prec,recall,f1 = evaluateResults(predsBinary, ytest)\n",
    "print(\"accuracy:\", acc)\n",
    "print(\"precision:\", prec)\n",
    "print(\"recall:\", recall)\n",
    "print(\"F1 score:\", f1)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python tf2019",
   "language": "python",
   "name": "tf2019"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}