test.py

#!/usr/bin/python3
# coding=utf-8

import logging
import re
import aiohttp
import asyncio
from bs4 import BeautifulSoup
from pymongo import MongoClient


class DouBanCrawl():

    def __init__(self, url):
        self.url = url

    async def fetch(self, url, headers):
        res = await aiohttp.request('GET', url)
        body = res.read()
        return (await body)

    def infos_get(self, html, name=None):
        soup = BeautifulSoup(html, 'lxml')
        scores = soup.select('.rating_num')
        scores = [score.text for score in scores]
        quotes = soup.select('p.quote > span')
        quotes = [quote.text for quote in quotes]
        pattern = r"https://movie.douban.com/subject/\w+/"
        hrefs = re.findall(pattern, str(html))[::2]
        title_list = soup.select('div.pic > a')
        try:
            titles = [re.findall(r'alt="(.*?)"', str(title))[0]
                      for title in title_list]
            img_links = [re.findall(r'src="(.*?)"', str(src))[0]
                         for src in title_list]
        except IndexError:
            pass
        return img_links, titles, scores, quotes, hrefs

    async def save_info(self, page):
        url = self.url.format(page)
        # print(url)
        with await sem:
            html = await self.fetch(url, headers)
        img_links, titles, scores, quotes, hrefs = self.infos_get(html)
        for infos in zip(img_links, titles, scores, quotes, hrefs):
            info = {'img': infos[0],
                    'name': infos[1],
                    'score': infos[2],
                    'quote': infos[3],
                    'href': infos[4]
                    }
            count = coll.find({"name": infos[1]}).count()
            if count == 0:
                coll.insert(info)


if __name__ == '__main__':
    url = 'https://movie.douban.com/top250?start={}&filter='
    headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'}
    client = MongoClient('localhost', 27017)
    db = client.movies
    coll = db.coll
    douban = DouBanCrawl(url)
    pages = range(0, 250, 25)
    sem = asyncio.Semaphore(4)  # 限制协程并发量
    loop = asyncio.get_event_loop()
    f = asyncio.wait([douban.save_info(page) for page in pages])
    loop.run_until_complete(f)  # %time 为Ipython 自带功能模块
    print('Done')