crawler.py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''抓取文档目录，根据目录抓取对应的文章'''\n",
    "\n",
    "import urllib2\n",
    "import urllib\n",
    "# from lxml.cssselect import CSSSelector\n",
    "from lxml.html import fromstring,make_links_absolute\n",
    "import lxml\n",
    "import pdfkit\n",
    "# import sys\n",
    "# reload(sys)\n",
    "# sys.setdefaultencoding('utf-8')\n",
    "\n",
    "\n",
    "def get_html_text(url):\n",
    "    '''根据url获得html代码'''\n",
    "    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) \\\n",
    "                AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'    \n",
    "    headers = { 'User-Agent' : user_agent } \n",
    "    req = urllib2.Request(url, headers = headers)    \n",
    "    try:\n",
    "        response = urllib2.urlopen(req)\n",
    "        content = response.read()\n",
    "    except urllib2.URLError, e:\n",
    "        if hasattr(e, 'code'):  \n",
    "\n",
    "            print 'The server couldn\\'t fulfill the request.'  \n",
    "\n",
    "            print 'Error code: ', e.code  \n",
    "\n",
    "        elif hasattr(e, 'reason'):  \n",
    "\n",
    "            print 'We failed to reach a server.'  \n",
    "\n",
    "            print 'Reason: ', e.reason  \n",
    "\n",
    "\n",
    "        else:  \n",
    "            print 'No exception was raised.', e.code      \n",
    "       \n",
    "    return content\n",
    "\n",
    "\n",
    "def get_catalogue(url, css):\n",
    "    '''抓取符合css规则的html代码, 返回目录的链接列表和标题列表'''\n",
    "\n",
    "    content = get_html_text(url)\n",
    "    doc = lxml.html.fromstring(content)\n",
    "    doc.make_links_absolute('http://www.liaoxuefeng.com')\n",
    "    css_result = doc.cssselect(css)\n",
    "    href_list = [e.get('href') for e in css_result]\n",
    "    title_list = [ e.text_content() for e in css_result]\n",
    "    return(href_list, title_list)\n",
    "\n",
    "def download_img(url, file_name):  \n",
    "\n",
    "    f = urllib2.urlopen(url) \n",
    "    img_data = f.read() \n",
    "    with open(file_name, \"wb\") as code:     \n",
    "        code.write(img_data)\n",
    "    \n",
    "        \n",
    "\n",
    "def get_target_html(url, css, download_image = True, img_dir = './images/', img_head='a'):\n",
    "    content = get_html_text(url)\n",
    "    doc = fromstring(content)\n",
    "    doc.make_links_absolute('http://www.liaoxuefeng.com')\n",
    "    \n",
    "    #下载图片，并修改原始图片地址\n",
    "\n",
    "\n",
    "    css_result = doc.cssselect(css)\n",
    "    if (download_image):\n",
    "        \n",
    "        img_tags = css_result[0].cssselect('img')\n",
    "        for ii in range(len(img_tags)):\n",
    "            src = img_tags[ii].get('src')\n",
    "            img_path = img_dir + img_head + '_' + str(ii)+ '.png'\n",
    "            print img_path\n",
    "            download_img(src, img_path)\n",
    "            img_tags[ii].set('src', img_path)\n",
    "    out = lxml.html.tostring(css_result[0], encoding= 'utf-8')\n",
    "    return out\n",
    "\n",
    "\n",
    "def bind_html(title_list, div_list):\n",
    "    \n",
    "    ''' 将得到的div code 合并并添加title'''\n",
    "    \n",
    "    div_code = ''\n",
    "    for x in range(len(title_list)):\n",
    "        div_code = div_code + '<div class=\"chapter\"><h2 id=\"chapter'+ str(x) + '\">' + title_list[x] + '</h2>' + div_list[x] + '</div>' \n",
    "    \n",
    "    return div_code\n",
    "\n",
    "# 设置打印pdf的样式\n",
    "css_file = 'css.css'\n",
    "pre_html = '''\n",
    "    <html>\n",
    "      <head>\n",
    "        <meta name=\"pdfkit-page-size\" content=\"Legal\"/>\n",
    "        <meta name=\"pdfkit-orientation\" content=\"Landscape\"/>\n",
    "               <link rel=\"stylesheet\" type=\"text/css\" href=\"css.css\">\n",
    "      </head>\n",
    "      <body>\n",
    "      '''\n",
    "#设置pdf页面布局\n",
    "options = {\n",
    "    'page-size': 'Letter',\n",
    "    'margin-top': '0.75in',\n",
    "    'margin-right': '0.75in',\n",
    "    'margin-bottom': '0.75in',\n",
    "    'margin-left': '0.75in',\n",
    "    'encoding': \"UTF-8\",\n",
    "    'no-outline': None\n",
    "}\n",
    "\n",
    "def html2pdf(html_code, file_name, options=options, css= css_file):\n",
    "    '''\n",
    "    标题和html代码转换成pdf\n",
    "    '''\n",
    "#     html = pre_html + body_code + '</body></html>'\n",
    "    try:\n",
    "        pdfkit.from_string(html_code, file_name, options=options, css= css)\n",
    "    except  Exception, e:\n",
    "        print e\n",
    "\n",
    "def write_html(html_code, file_name):\n",
    "    f=open(file_name, 'w+')\n",
    "    f.truncate() \n",
    "    f.write(html_code)\n",
    "    f.close()\n",
    "    \n",
    "def generate_content(title_list):\n",
    "    html_code = '<div class=\"content\"><h2>Content</h2><ul>'\n",
    "    for ii in range(len(title_list)):\n",
    "        html_code = html_code + '<li><a href=\"#chapter'+ str(ii) + '\">'+ title_list[ii] + '</a></li>'\n",
    "    html_code = html_code + '</ul></div>'\n",
    "    return html_code\n",
    "\n",
    "\n",
    "def html_file2pdf(html_name, pdf_name,options = options, css_file=css_file):\n",
    "    pdfkit.from_file(html_name, pdf_name, options=options, css= css_file)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    css_catalogue = 'div.x-sidebar-left-content ul:nth-child(2) li a'\n",
    "    url_catalouge = 'http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000'\n",
    "    (href_list, title_list) = get_catalogue(url_catalouge, css_catalogue)\n",
    "    css_doc = 'div.x-wiki-content'\n",
    "\n",
    "    div_list = []\n",
    "    for ii in range(len(href_list)):\n",
    "        div_list.append(get_target_html(href_list[ii], css_doc, img_head = 'chapter'+ str(ii)))\n",
    "\n",
    "    html = bind_html(title_list, div_list)\n",
    "    html_code = pre_html + generate_content(title_list) +html + '</body></html>'\n",
    "    write_html(html_code, 'python3.0.html')\n",
    "#     html2pdf(html_code, 'python3.0.pdf')\n",
    "#     html_file2pdf('python3.0.html', 'python3.0.pdf', options, css_file)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./images/chapter0_0.png\n",
      "./images/chapter0_1.png\n",
      "./images/chapter1_0.png\n",
      "./images/chapter1_1.png\n",
      "./images/chapter1_2.png\n",
      "./images/chapter2_0.png\n",
      "./images/chapter2_1.png\n",
      "./images/chapter2_2.png\n",
      "./images/chapter4_0.png\n",
      "./images/chapter5_0.png\n",
      "./images/chapter5_1.png\n",
      "./images/chapter5_2.png\n",
      "./images/chapter5_3.png\n",
      "./images/chapter5_4.png\n",
      "./images/chapter6_0.png\n",
      "./images/chapter6_1.png\n",
      "./images/chapter6_2.png\n",
      "./images/chapter7_0.png\n",
      "./images/chapter9_0.png\n",
      "./images/chapter9_1.png\n",
      "./images/chapter9_2.png\n",
      "./images/chapter10_0.png\n",
      "./images/chapter10_1.png\n",
      "./images/chapter10_2.png\n",
      "./images/chapter10_3.png\n",
      "./images/chapter10_4.png\n",
      "./images/chapter10_5.png\n",
      "./images/chapter11_0.png\n",
      "./images/chapter11_1.png\n",
      "./images/chapter12_0.png\n",
      "./images/chapter14_0.png\n",
      "./images/chapter14_1.png\n",
      "./images/chapter28_0.png\n",
      "./images/chapter35_0.png\n",
      "./images/chapter35_1.png\n",
      "./images/chapter41_0.png\n",
      "./images/chapter47_0.png\n",
      "./images/chapter47_1.png\n",
      "./images/chapter47_2.png\n",
      "./images/chapter66_0.png\n",
      "./images/chapter71_0.png\n",
      "./images/chapter79_0.png\n",
      "./images/chapter79_1.png\n",
      "./images/chapter81_0.png\n",
      "./images/chapter81_1.png\n",
      "./images/chapter82_0.png\n",
      "./images/chapter83_0.png\n",
      "./images/chapter84_0.png\n",
      "./images/chapter85_0.png\n",
      "./images/chapter87_0.png\n",
      "./images/chapter87_1.png\n",
      "./images/chapter87_2.png\n",
      "./images/chapter87_3.png\n",
      "./images/chapter87_4.png\n",
      "./images/chapter88_0.png\n",
      "./images/chapter89_0.png\n",
      "./images/chapter89_1.png\n",
      "./images/chapter89_2.png\n",
      "./images/chapter89_3.png\n",
      "./images/chapter94_0.png\n",
      "./images/chapter94_1.png\n",
      "./images/chapter94_2.png\n",
      "./images/chapter94_3.png\n",
      "./images/chapter94_4.png\n",
      "./images/chapter95_0.png\n",
      "./images/chapter95_1.png\n",
      "./images/chapter95_2.png\n",
      "./images/chapter96_0.png\n",
      "./images/chapter96_1.png\n",
      "./images/chapter96_2.png\n",
      "./images/chapter96_3.png\n",
      "./images/chapter97_0.png\n",
      "./images/chapter97_1.png\n",
      "./images/chapter97_2.png\n",
      "./images/chapter97_3.png\n",
      "./images/chapter98_0.png\n",
      "./images/chapter98_1.png\n",
      "./images/chapter98_2.png\n",
      "./images/chapter103_0.png\n",
      "./images/chapter103_1.png\n",
      "./images/chapter103_2.png\n",
      "./images/chapter103_3.png\n",
      "./images/chapter105_0.png\n",
      "./images/chapter110_0.png\n",
      "./images/chapter111_0.png\n",
      "./images/chapter112_0.png\n",
      "./images/chapter113_0.png\n",
      "./images/chapter114_0.png\n",
      "./images/chapter114_1.png\n",
      "./images/chapter115_0.png\n",
      "./images/chapter118_0.png\n",
      "./images/chapter118_1.png\n",
      "./images/chapter118_2.png\n",
      "./images/chapter118_3.png\n",
      "./images/chapter119_0.png\n",
      "./images/chapter119_1.png\n",
      "./images/chapter121_0.png\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000\n"
     ]
    }
   ],
   "source": [
    "print href_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_toc(title_list, book_name, file_name):\n",
    "    pre_str = '''\n",
    "    <?xml version='1.0' encoding='utf-8'?>\n",
    "<!DOCTYPE ncx PUBLIC \"-//NISO//DTD ncx 2005-1//EN\" \n",
    "                 \"http://www.daisy.org/z3986/2005/ncx-2005-1.dtd\">\n",
    "<ncx xmlns=\"http://www.daisy.org/z3986/2005/ncx/\" version=\"2005-1\">\n",
    "  <head>\n",
    "    <meta name=\"dtb:uid\" content=\"urn:uuid:12345\"/>\n",
    "    <meta name=\"dtb:depth\" content=\"2\"/>\n",
    "    <meta name=\"dtb:totalPageCount\" content=\"0\"/>\n",
    "    <meta name=\"dtb:maxPageNumber\" content=\"0\"/>\n",
    "  </head> '''\n",
    "    doc_title = '<docTitle><text>'+ book_name + '</text></docTitle>'\n",
    "    cover = ' <navMap><navPoint id=\"cover\" playOrder=\"1\"><navLabel><text>Book cover</text></navLabel> \\\n",
    "      <content src=\"title.html\"/> </navPoint>'\n",
    "    xml_str = pre_str + doc_title + cover\n",
    "    for ii in range(len(title_list)):\n",
    "        xml_str = xml_str + ' <navPoint id=\"chapter' + str(ii) + \\\n",
    "        '\" playOrder=\"'+ str(ii+2) + '\"><navLabel> <text>'+ title_list[ii] + \\\n",
    "        '</text> </navLabel> <content src=\"content.html#chapter'+ str(ii)+ '\"/></navPoint>'\n",
    "    xml_str = xml_str + ' </navMap></ncx>'\n",
    "    f=open(file_name, 'w+')\n",
    "    f.truncate() \n",
    "    f.write(xml_str)\n",
    "    f.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "create_toc(title_list, 'python 3.0', 'toc.ncx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}