From b3c69d6d40d3ad70056f3e49af662ab2f7201fa1 Mon Sep 17 00:00:00 2001 From: Dec-F Date: Fri, 11 Oct 2019 22:54:31 +0800 Subject: [PATCH] feat(proxy): Increase agent functionality --- .travis.yml | 2 + app/config-store/index.ts | 6 ++- app/containers/Home/ScrapeInfoModal/index.tsx | 1 - app/containers/Home/SettingModal/form.tsx | 42 ++++++++++++++----- app/scraper/core/index.ts | 21 +++++++--- app/scraper/heads/avsox.ts | 13 +++--- app/scraper/heads/index.ts | 3 +- app/scraper/heads/javbus.ts | 13 +++--- app/scraper/heads/tmdb.ts | 5 +-- app/scraper/request.ts | 8 ++++ app/utils/index.ts | 4 +- package.json | 2 +- 12 files changed, 81 insertions(+), 39 deletions(-) create mode 100644 app/scraper/request.ts diff --git a/.travis.yml b/.travis.yml index fe2489e..3a3784f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,8 @@ cache: before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXX="g++-8"; fi + - rm -rf yarn.lock + - rm -rf package-lock.json install: - yarn --ignore-engines diff --git a/app/config-store/index.ts b/app/config-store/index.ts index 7cada90..26f0329 100644 --- a/app/config-store/index.ts +++ b/app/config-store/index.ts @@ -3,7 +3,11 @@ import Store from 'electron-store'; const store = new Store({ defaults: { scene: ['movie', 'normal'], - tags: ['无'] + tags: ['无'], + proxy: { + enable: false, + url: 'http://127.0.0.1:1087' + } } }); diff --git a/app/containers/Home/ScrapeInfoModal/index.tsx b/app/containers/Home/ScrapeInfoModal/index.tsx index f12b678..d2d6d43 100644 --- a/app/containers/Home/ScrapeInfoModal/index.tsx +++ b/app/containers/Home/ScrapeInfoModal/index.tsx @@ -43,7 +43,6 @@ const ScrapeModal = ({ visible, taskQueue, onCancel, handleTaskEnd }) => { str: task.file.key === key ? str : task.str ? task.str : '' })); setTaskQ(_taskQ); - console.log(_taskQ, 1); lastTaskQ.current = _taskQ; }); emitter.on(EventType.SCRAPE_SUCCESS, ({ key }, json) => { diff --git a/app/containers/Home/SettingModal/form.tsx b/app/containers/Home/SettingModal/form.tsx index 71fbc38..7f17eb6 100644 --- a/app/containers/Home/SettingModal/form.tsx +++ b/app/containers/Home/SettingModal/form.tsx @@ -1,5 +1,5 @@ -import React, { useImperativeHandle, forwardRef, useState } from 'react'; -import { Select, Form, Cascader, Row, Col } from 'antd'; +import React, { useImperativeHandle, forwardRef } from 'react'; +import { Select, Form, Cascader, Row, Col, Switch, Input } from 'antd'; import config from '@config'; import styles from './index.less'; import { mediaType } from '@scraper'; @@ -8,22 +8,29 @@ const { Option } = Select; const { Item } = Form; const SettingForm = ({ form }, ref) => { - const tags = config.get('tags'); - const scene = config.get('scene', ['movie', 'normal']); + const tagsConfig = config.get('tags'); + const sceneConfig = config.get('scene', ['movie', 'normal']); + const proxyConfig = config.get('proxy'); // const sceneSourceMapping = config.get('sceneSourceMapping', {}); useImperativeHandle(ref, () => ({ submitForm: () => new Promise((resolve, reject) => { form.validateFields((err, values) => { if (err) return reject(err); - config.set(values); - console.log(config.path); + const { tags, scene, proxyUrl, proxyEnable } = values; + config.set({ + tags, + scene, + proxy: { + enable: proxyEnable, + url: proxyUrl + } + }); resolve(); }); }) })); - // const [mediaSourceVisible, setMediaSourceVisible] = useState(false); - console.log(useState); + const { getFieldDecorator } = form; return (
{ className={styles.setting_form} > - {getFieldDecorator('tags', { initialValue: tags })( + {getFieldDecorator('tags', { initialValue: tagsConfig })( @@ -44,12 +51,25 @@ const SettingForm = ({ form }, ref) => { - {getFieldDecorator('scene', { initialValue: scene })( + {getFieldDecorator('scene', { initialValue: sceneConfig })( )} + + + + {getFieldDecorator('proxyEnable', { + initialValue: proxyConfig.enable, + valuePropName: 'checked' + })()} + {getFieldDecorator('proxyUrl', { + initialValue: proxyConfig.url + })()} + + + ); }; diff --git a/app/scraper/core/index.ts b/app/scraper/core/index.ts index 0b5e893..d99678f 100644 --- a/app/scraper/core/index.ts +++ b/app/scraper/core/index.ts @@ -1,9 +1,11 @@ import { writeFile, mkdirp } from 'fs-extra'; -// import javbus from '../javBus'; import { downloadImg, emitter } from '../../utils'; import { EventType, QueryOpt, ToolHead, FileNode } from '@types'; +import config from '@config'; const saveAsserts = async (model, file) => { + const proxy = config.get('proxy'); + const url = proxy.enable ? proxy.url : ''; const json = model.getModel(); await mkdirp(`${file.wpath}/.actors`); return Promise.all([ @@ -11,10 +13,20 @@ const saveAsserts = async (model, file) => { `${file.wpath + file.title}.nfo`, `${model.getXML()}` ), - downloadImg(json.art.poster._text, `${file.wpath + file.title}-poster.jpg`), - downloadImg(json.art.fanart._text, `${file.wpath + file.title}-fanart.jpg`), + downloadImg( + json.art.poster._text, + `${file.wpath + file.title}-poster.jpg`, + { proxy: url } + ), + downloadImg( + json.art.fanart._text, + `${file.wpath + file.title}-fanart.jpg`, + { proxy: url } + ), json.actor.map(v => - downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`) + downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`, { + proxy: url + }) ) ]) .then(() => model) @@ -43,7 +55,6 @@ class Scraper { async start(queryOpts: QueryOpt[], name: string): Promise { this.stopFlag = false; - console.log(name, queryOpts); const failureTasks = []; const successTasks = []; const { head } = this.heads.find(t => t.name === name); diff --git a/app/scraper/heads/avsox.ts b/app/scraper/heads/avsox.ts index cc7bf28..2d35bca 100644 --- a/app/scraper/heads/avsox.ts +++ b/app/scraper/heads/avsox.ts @@ -1,20 +1,19 @@ -import request from 'request-promise'; import cheerio from 'cheerio'; import MovieModel from '../core/model'; import { MediaKeys } from '@types'; -export default { +export default request => ({ head: async (queryString: string): Promise => { const movieModel = new MovieModel(); const encodedQueryString = encodeURIComponent(queryString); - const searchPage = await request( - `https://avsox.asia/cn/search/${encodedQueryString}` - ); + const searchPage = await request({ + url: `https://avsox.asia/cn/search/${encodedQueryString}` + }); const infoPageUrl = cheerio .load(searchPage)('.movie-box') .attr('href') .replace(/https:\/\//, 'http://'); - const $ = cheerio.load(await request(infoPageUrl)); + const $ = cheerio.load(await request({ url: infoPageUrl })); movieModel.setModel({ title: { _text: $('h3') @@ -65,4 +64,4 @@ export default { }, name: 'avsox', type: [MediaKeys.Movie, MediaKeys.Gentleman] -}; +}); diff --git a/app/scraper/heads/index.ts b/app/scraper/heads/index.ts index 0fa91a2..72927b4 100644 --- a/app/scraper/heads/index.ts +++ b/app/scraper/heads/index.ts @@ -1,7 +1,8 @@ +import r from '../request'; import { ToolHead } from '@types'; import tmdb from './tmdb'; import javbus from './javbus'; import avsox from './avsox'; -const heads: ToolHead[] = [tmdb, javbus, avsox]; +const heads: ToolHead[] = [tmdb(r), javbus(r), avsox(r)]; export default heads; diff --git a/app/scraper/heads/javbus.ts b/app/scraper/heads/javbus.ts index 83ee4b9..f46ddc3 100644 --- a/app/scraper/heads/javbus.ts +++ b/app/scraper/heads/javbus.ts @@ -1,20 +1,19 @@ -import request from 'request-promise'; import cheerio from 'cheerio'; import MovieModel from '../core/model'; import { MediaKeys } from '@types'; -export default { +export default request => ({ head: async (queryString: string): Promise => { const movieModel = new MovieModel(); const encodedQueryString = encodeURIComponent(queryString); - const searchPage = await request( - `http://www.javbus.com/uncensored/search/${encodedQueryString}` - ); + const searchPage = await request({ + url: `http://www.javbus.com/uncensored/search/${encodedQueryString}` + }); const infoPageUrl = cheerio .load(searchPage)('.movie-box') .attr('href') .replace(/https:\/\//, 'http://'); - const $ = cheerio.load(await request(infoPageUrl)); + const $ = cheerio.load(await request({ url: infoPageUrl })); movieModel.setModel({ title: { _text: $('h3') @@ -65,4 +64,4 @@ export default { }, name: 'javbus', type: [MediaKeys.Movie, MediaKeys.Gentleman] -}; +}); diff --git a/app/scraper/heads/tmdb.ts b/app/scraper/heads/tmdb.ts index 14bde02..6ffa41c 100644 --- a/app/scraper/heads/tmdb.ts +++ b/app/scraper/heads/tmdb.ts @@ -1,8 +1,7 @@ -import request from 'request-promise'; import MovieModel from '../core/model'; import { MovieModelType, MediaKeys } from '@types'; -export default { +export default request => ({ head: async (queryString: string): Promise => { const movieModel = new MovieModel(); const baseUrl = 'http://api.themoviedb.org/3'; @@ -77,4 +76,4 @@ export default { }, name: 'TMDB', type: [MediaKeys.Movie, MediaKeys.Normal] -}; +}); diff --git a/app/scraper/request.ts b/app/scraper/request.ts new file mode 100644 index 0000000..c957c96 --- /dev/null +++ b/app/scraper/request.ts @@ -0,0 +1,8 @@ +import request from 'request-promise'; +import config from '@config'; + +export default opts => { + const proxy = config.get('proxy'); + const url = proxy.enable ? proxy.url : ''; + return request({ ...opts, proxy: url }); +}; diff --git a/app/utils/index.ts b/app/utils/index.ts index 4139190..cc10cb0 100644 --- a/app/utils/index.ts +++ b/app/utils/index.ts @@ -96,9 +96,9 @@ export const sleep = time => } }); -export const downloadImg = (url, ipath) => +export const downloadImg = (url, ipath, opt?) => new Promise((resolve, reject) => { - request(url) + request({ url, ...opt }) .pipe(fs.createWriteStream(ipath)) .on('finish', () => { resolve(ipath); diff --git a/package.json b/package.json index 2a24a2a..72cb672 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "u-scraper", "productName": "uScraper", - "version": "0.1.0-beta", + "version": "0.2.0-beta", "description": "A scraper that switches between normal mode and gentleman mode, built on Eletron, React", "scripts": { "build": "concurrently \"yarn build-main\" \"yarn build-renderer\"",