classify1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import jieba\n",
    "data = pd.read_csv('sohu_train.txt', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(data[1], data[0], random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5628     业内论坛 版主：　毕普　ｔａｌｋａｔｉｖｅ　也是另类　华沉思　周海山　成业　醋咪　ｚｈａｍｕ...\n",
       "3615     中央印发建立健全惩治和预防腐败体系５年规划 中共中央近日印发了《建立健全惩治和预防腐败体系２...\n",
       "9963     深圳高新区嬗变：硅谷中国影像 本报记者　丘慧慧硅谷的幻想，投影到了深圳。２００７年，来自中国...\n",
       "14979     ２００８年各地高考录取分数线排名（以文科一本分数线的高低排序）省份一批二批三批详细文科理科...\n",
       "16248    塔塔买悍马　印证李书福＂通用会完蛋＇预言？ 悍马可能被通用卖掉！买家可能来自印度。这是继福特...\n",
       "15984    女孩赶考路上遇车祸　头受伤血流满面进考场（图） 缝了５针后，头上包扎着白纱布的小张乘车赶往考...\n",
       "20328    前门大街，通到中国的心脏 本文来源：《ＴｉｍｅＯｕｔ消费导刊》授权刊登，禁止转载前门大街，从...\n",
       "9458     北京奥运会城市志愿者服务站点发布仪式举行 ６月３０日上午，“微笑北京　奥运先锋”北京奥运会城...\n",
       "11054    德奥战前球迷＂对厢歌＂　＂坦克＂击碎科尔多巴美梦 特派记者　邬恺山　本报奥地利６月１７日电奥...\n",
       "6718     冬季到西洞庭去看鸟 －－“蒹葭苍苍，白露为霜。所谓伊人，在水一方。”这种向往离我们的现实生活...\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19686     新闻\n",
       "18695     新闻\n",
       "3209      财经\n",
       "9982      科技\n",
       "14683     教育\n",
       "17441     汽车\n",
       "22992     女人\n",
       "2898      财经\n",
       "7525      旅游\n",
       "769       娱乐\n",
       "8218      科技\n",
       "5393     房地产\n",
       "5231     房地产\n",
       "18775     新闻\n",
       "406       娱乐\n",
       "5311     房地产\n",
       "23279     女人\n",
       "15850     教育\n",
       "10830     体育\n",
       "15053     教育\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.sample(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18000"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6000"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22809    奢侈 品牌 竞技 顶级 足球联赛   前 几天 的 欧冠 决赛 在 莫斯科 这座 世界 亿万...\n",
      "3925     上海 交行 ： 特里谢 　 Ｐ Ｋ 　 贝南克 　 猴性 欧元 下 跳 上 窜   上周 点...\n",
      "1377     情感 力量 超越 时空 （ 组图 ）   ３ ０ 年前 《 于无声处 》 演出 的 剧照 。...\n",
      "8175     中国 酒店业 转型 数字化 服务   日前 ， 记者 从 “ ２ ０ ０ ８ 年 全球 酒店...\n",
      "9796     模拟 最 忙 赛事 进程 　 北京 奥运会 最终 技术 演练 结束   【 搜狐 Ｉ Ｔ 消...\n",
      "Name: 1, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def fenci(train_data):\n",
    "    words_df = train_data.apply(lambda x:' '.join(jieba.cut(x)))\n",
    "    return words_df\n",
    "\n",
    "x_train_fenci = fenci(x_train)\n",
    "print(x_train_fenci[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "infile = open(\"stopwords.txt\",encoding='utf-8')\n",
    "stopwords_lst = infile.readlines()\n",
    "stopwords = [x.strip() for x in stopwords_lst]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CountVectorizer是属于常见的特征数值计算类，是一个文本特征提取方法。 对于每一个训练文本，它只考虑每种词汇在该训练文本中出现的频率。 CountVectorizer会将文本中的词语转换为词频矩阵，它通过fit_transform函数计算各个词语出现的次数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=['', 'aboard', 'about', 'about', 'about', 'above', 'above', 'according', 'according to', 'across', 'across', 'afore', 'after', 'after', 'after', 'afterwards', 'again', 'against', 'against', 'against', 'agin', 'all', 'all', 'almost', 'almost', 'alone', 'along', 'along', 'alongside', 'alrea... '）', '，', '：', '：', '；', '；', '？', '？', '？', '＿', '￣', '&nbsp;', '&nbsp', 'pp', 'px', '摄于', 'info'],\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(stop_words=stopwords, max_features=5000)\n",
    "vectorizer.fit(x_train_fenci)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7778333333333334"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "classifier = MultinomialNB()\n",
    "#模型训练\n",
    "classifier.fit(vectorizer.transform(x_train_fenci), y_train)\n",
    "#使用训练好的模型进行预测\n",
    "classifier.score(vectorizer.transform(fenci(x_test)), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7915"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "#使用tf-idf把文本转为向量\n",
    "tv = TfidfVectorizer(stop_words=stopwords, max_features=5000, lowercase = False)\n",
    "tv.fit(x_train_fenci)\n",
    "#模型训练\n",
    "classifier.fit(tv.transform(fenci(x_train)), y_train)\n",
    "#利用训练好的模型测试\n",
    "classifier.score(tv.transform(fenci(x_test)), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7916666666666666"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#转词向量\n",
    "tv_2gram = TfidfVectorizer(stop_words=stopwords, max_features=5000, ngram_range=(1,2),lowercase = False)\n",
    "tv_2gram.fit(x_train_fenci)\n",
    "#训练模型\n",
    "clf_2gram = MultinomialNB()\n",
    "clf_2gram.fit(tv_2gram.transform(fenci(x_train)), y_train)\n",
    "#预测\n",
    "clf_2gram.score(tv_2gram.transform(fenci(x_test)), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}