From 6b198ed08c630d936d61bcb48230278829bb05e7 Mon Sep 17 00:00:00 2001 From: Dec-F Date: Thu, 10 Oct 2019 21:55:57 +0800 Subject: [PATCH] feat(scraper):Optimized and added avsox information source --- app/baidu.html | 448 ------------------ app/baidu.nfo | 1 - app/config-store/index.ts | 7 +- app/containers/Home/HeaderContent/index.tsx | 22 +- app/containers/Home/ScrapeInfoModal/index.tsx | 3 +- app/containers/Home/SettingModal/form.tsx | 4 +- app/scraper/heads/avsox.ts | 68 +++ app/scraper/heads/index.ts | 5 +- app/scraper/heads/javbus.ts | 2 +- app/scraper/heads/tmdb.ts | 5 +- app/scraper/mediaType.ts | 2 +- 11 files changed, 102 insertions(+), 465 deletions(-) delete mode 100644 app/baidu.html delete mode 100644 app/baidu.nfo create mode 100644 app/scraper/heads/avsox.ts diff --git a/app/baidu.html b/app/baidu.html deleted file mode 100644 index 8c98379..0000000 --- a/app/baidu.html +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - - - - - - - - - 222 - 电影 - 豆瓣搜索 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - -
-
正在搜索...
-
-
- - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/app/baidu.nfo b/app/baidu.nfo deleted file mode 100644 index 7afa2ac..0000000 --- a/app/baidu.nfo +++ /dev/null @@ -1 +0,0 @@ -222 - 电影 - 豆瓣搜索 \ No newline at end of file diff --git a/app/config-store/index.ts b/app/config-store/index.ts index f72a754..7cada90 100644 --- a/app/config-store/index.ts +++ b/app/config-store/index.ts @@ -1,5 +1,10 @@ import Store from 'electron-store'; -const store = new Store(); +const store = new Store({ + defaults: { + scene: ['movie', 'normal'], + tags: ['无'] + } +}); export default store; diff --git a/app/containers/Home/HeaderContent/index.tsx b/app/containers/Home/HeaderContent/index.tsx index 77c0ed0..98aebbb 100644 --- a/app/containers/Home/HeaderContent/index.tsx +++ b/app/containers/Home/HeaderContent/index.tsx @@ -1,6 +1,6 @@ -import React, { useState } from 'react'; +import React, { useState, useRef, useEffect } from 'react'; import { connect } from 'react-redux'; -import { Button, Row, Col, Input, Dropdown, Menu } from 'antd'; +import { Button, Row, Col, Input, Dropdown, Menu, message } from 'antd'; import * as R from 'ramda'; import CRD from '@vdts/collect-video'; @@ -33,6 +33,13 @@ const HeaderContent = ({ const [modalVisible, setModalVisible] = useState(false); const [settingVisible, setSettingVisible] = useState(false); const [taskQueue, setTaskQueue] = useState([]); + const [tasks, setTasks] = useState([]); + const scraperHead = useRef(''); + useEffect(() => { + if (tasks.length) { + scrape.start(tasks, scraperHead.current); + } + }, [tasks]); const handleInput = filename => { dispatch(setSelectedFilename(filename)); }; @@ -53,6 +60,10 @@ const HeaderContent = ({ ); }; const handleScrape = (headName?: string) => { + const defaultHead = getHeadsByMediaType(config.get('scene'))[0]; + if (!defaultHead) { + return message.error('当前场景未找到信息源'); + } dispatch(changeFailureKeys([])); const head = headName || getHeadsByMediaType(config.get('scene'))[0].name; let _taskQueue = []; @@ -86,7 +97,8 @@ const HeaderContent = ({ } setModalVisible(true); setTaskQueue(_taskQueue); - scrape.start(_tasks, head); + setTasks(_tasks); + scraperHead.current = head; }; const handleRebuild = () => { CRD(tree.wpath) @@ -96,9 +108,11 @@ const HeaderContent = ({ dispatch(changeSelected('')); dispatch(changeFailureKeys([])); dispatch(selectFiles(_tree[0])); + message.success('格式化成功'); return res; }) .catch(e => { + message.error('格式化失败'); console.log(e); }); }; @@ -108,7 +122,7 @@ const HeaderContent = ({ handleScrape(e.key); }} > - {getHeadsByMediaType(config.get('scene')).map(head => ( + {(getHeadsByMediaType(config.get('scene')) || []).map(head => ( {head.name} ))} diff --git a/app/containers/Home/ScrapeInfoModal/index.tsx b/app/containers/Home/ScrapeInfoModal/index.tsx index 6dd666c..f12b678 100644 --- a/app/containers/Home/ScrapeInfoModal/index.tsx +++ b/app/containers/Home/ScrapeInfoModal/index.tsx @@ -36,15 +36,14 @@ const ScrapeModal = ({ visible, taskQueue, onCancel, handleTaskEnd }) => { lastTaskQ.current = taskQueue; }, [taskQueue]); useEffect(() => { - console.log(44); emitter.on(EventType.SCRAPE_PENDING, ({ key }, str) => { - console.log(33); const _taskQ = lastTaskQ.current.map(task => ({ ...task, status: task.file.key === key ? 'pending' : task.status, str: task.file.key === key ? str : task.str ? task.str : '' })); setTaskQ(_taskQ); + console.log(_taskQ, 1); lastTaskQ.current = _taskQ; }); emitter.on(EventType.SCRAPE_SUCCESS, ({ key }, json) => { diff --git a/app/containers/Home/SettingModal/form.tsx b/app/containers/Home/SettingModal/form.tsx index bf53783..71fbc38 100644 --- a/app/containers/Home/SettingModal/form.tsx +++ b/app/containers/Home/SettingModal/form.tsx @@ -8,7 +8,7 @@ const { Option } = Select; const { Item } = Form; const SettingForm = ({ form }, ref) => { - const tags = config.get('tags', []); + const tags = config.get('tags'); const scene = config.get('scene', ['movie', 'normal']); // const sceneSourceMapping = config.get('sceneSourceMapping', {}); useImperativeHandle(ref, () => ({ @@ -35,7 +35,7 @@ const SettingForm = ({ form }, ref) => { {getFieldDecorator('tags', { initialValue: tags })( diff --git a/app/scraper/heads/avsox.ts b/app/scraper/heads/avsox.ts new file mode 100644 index 0000000..cc7bf28 --- /dev/null +++ b/app/scraper/heads/avsox.ts @@ -0,0 +1,68 @@ +import request from 'request-promise'; +import cheerio from 'cheerio'; +import MovieModel from '../core/model'; +import { MediaKeys } from '@types'; + +export default { + head: async (queryString: string): Promise => { + const movieModel = new MovieModel(); + const encodedQueryString = encodeURIComponent(queryString); + const searchPage = await request( + `https://avsox.asia/cn/search/${encodedQueryString}` + ); + const infoPageUrl = cheerio + .load(searchPage)('.movie-box') + .attr('href') + .replace(/https:\/\//, 'http://'); + const $ = cheerio.load(await request(infoPageUrl)); + movieModel.setModel({ + title: { + _text: $('h3') + .text() + .trim() + }, + premiered: { + _text: $('.info>p:nth-child(2)') + .text() + .split(': ')[1] + .trim() + }, + art: { + poster: { + _text: $('.bigImage') + .attr('href') + .trim() + .replace(/https:\/\//, 'http://') + }, + fanart: { + _text: $('.bigImage') + .attr('href') + .trim() + .replace(/https:\/\//, 'http://') + } + }, + actor: $('#avatar-waterfall .avatar-box img') + .map((index, $actor) => ({ + name: { _text: $actor.attribs.title.trim() }, + thumb: { + _text: $actor.attribs.src.trim().replace(/https:\/\//, 'http://') + } + })) + .toArray(), + uniqueid: [ + { + _attributes: { type: '1', default: true }, + _text: $('.info>p:nth-child(1)>span:nth-child(2)') + .text() + .trim() + } + ], + genre: $('.info .genre>a') + .map((index, $genre) => ({ _text: $genre.firstChild.data.trim() })) + .toArray() + }); + return movieModel; + }, + name: 'avsox', + type: [MediaKeys.Movie, MediaKeys.Gentleman] +}; diff --git a/app/scraper/heads/index.ts b/app/scraper/heads/index.ts index d2fe8d6..0fa91a2 100644 --- a/app/scraper/heads/index.ts +++ b/app/scraper/heads/index.ts @@ -1,4 +1,7 @@ +import { ToolHead } from '@types'; import tmdb from './tmdb'; import javbus from './javbus'; +import avsox from './avsox'; -export default [tmdb, javbus]; +const heads: ToolHead[] = [tmdb, javbus, avsox]; +export default heads; diff --git a/app/scraper/heads/javbus.ts b/app/scraper/heads/javbus.ts index fd191ce..83ee4b9 100644 --- a/app/scraper/heads/javbus.ts +++ b/app/scraper/heads/javbus.ts @@ -57,7 +57,7 @@ export default { .trim() } ], - genre: $('.info>p:nth-child(6) .genre>a') + genre: $('.info .genre>a') .map((index, $actor) => ({ _text: $actor.firstChild.data.trim() })) .toArray() }); diff --git a/app/scraper/heads/tmdb.ts b/app/scraper/heads/tmdb.ts index 2e972ac..14bde02 100644 --- a/app/scraper/heads/tmdb.ts +++ b/app/scraper/heads/tmdb.ts @@ -1,5 +1,4 @@ import request from 'request-promise'; -// import cheerio from 'cheerio'; import MovieModel from '../core/model'; import { MovieModelType, MediaKeys } from '@types'; @@ -21,8 +20,6 @@ export default { json: true }); - console.log(movieModel, queryString, 111); - if (!res.total_results) { throw new Error('无影片信息'); } @@ -48,7 +45,7 @@ export default { genres } = info; movieModel.setModel({ - title, + title: { _text: title }, plot: { _text: overview }, diff --git a/app/scraper/mediaType.ts b/app/scraper/mediaType.ts index fa753ad..3e6e811 100644 --- a/app/scraper/mediaType.ts +++ b/app/scraper/mediaType.ts @@ -32,5 +32,5 @@ const mediaSource = heads.reduce((acc, head) => { export const getHeadsByMediaType: (type: string[]) => ToolHead[] = type => { const sourceId = type.join('$$'); - return mediaSource[sourceId]; + return mediaSource[sourceId] || []; };