feat(proxy): Increase agent functionality

.travis.yml

@@ -52,6 +52,8 @@ cache:
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXX="g++-8"; fi
+  - rm -rf yarn.lock
+  - rm -rf package-lock.json
 
 install:
   - yarn --ignore-engines

app/config-store/index.ts

@@ -3,7 +3,11 @@ import Store from 'electron-store';
 const store = new Store({
   defaults: {
     scene: ['movie', 'normal'],
-    tags: ['无']
+    tags: ['无'],
+    proxy: {
+      enable: false,
+      url: 'http://127.0.0.1:1087'
+    }
   }
 });
 

app/containers/Home/ScrapeInfoModal/index.tsx

@@ -43,7 +43,6 @@ const ScrapeModal = ({ visible, taskQueue, onCancel, handleTaskEnd }) => {
         str: task.file.key === key ? str : task.str ? task.str : ''
       }));
       setTaskQ(_taskQ);
-      console.log(_taskQ, 1);
       lastTaskQ.current = _taskQ;
     });
     emitter.on(EventType.SCRAPE_SUCCESS, ({ key }, json) => {

app/containers/Home/SettingModal/form.tsx

@@ -1,5 +1,5 @@
-import React, { useImperativeHandle, forwardRef, useState } from 'react';
-import { Select, Form, Cascader, Row, Col } from 'antd';
+import React, { useImperativeHandle, forwardRef } from 'react';
+import { Select, Form, Cascader, Row, Col, Switch, Input } from 'antd';
 import config from '@config';
 import styles from './index.less';
 import { mediaType } from '@scraper';
@@ -8,22 +8,29 @@ const { Option } = Select;
 const { Item } = Form;
 
 const SettingForm = ({ form }, ref) => {
-  const tags = config.get('tags');
-  const scene = config.get('scene', ['movie', 'normal']);
+  const tagsConfig = config.get('tags');
+  const sceneConfig = config.get('scene', ['movie', 'normal']);
+  const proxyConfig = config.get('proxy');
   // const sceneSourceMapping = config.get('sceneSourceMapping', {});
   useImperativeHandle(ref, () => ({
     submitForm: () =>
       new Promise((resolve, reject) => {
         form.validateFields((err, values) => {
           if (err) return reject(err);
-          config.set(values);
-          console.log(config.path);
+          const { tags, scene, proxyUrl, proxyEnable } = values;
+          config.set({
+            tags,
+            scene,
+            proxy: {
+              enable: proxyEnable,
+              url: proxyUrl
+            }
+          });
           resolve();
         });
       })
   }));
-  // const [mediaSourceVisible, setMediaSourceVisible] = useState(false);
-  console.log(useState);
+
   const { getFieldDecorator } = form;
   return (
     <Form
@@ -33,9 +40,9 @@ const SettingForm = ({ form }, ref) => {
       className={styles.setting_form}
     >
       <Item label="预设标签">
-        {getFieldDecorator('tags', { initialValue: tags })(
+        {getFieldDecorator('tags', { initialValue: tagsConfig })(
           <Select mode="tags" style={{ width: '100%' }} placeholder="Tags">
-            {tags.map(tag => (
+            {tagsConfig.map(tag => (
               <Option key={tag}>{tag}</Option>
             ))}
           </Select>
@@ -44,12 +51,25 @@ const SettingForm = ({ form }, ref) => {
       <Item label="场景">
         <Row>
           <Col span={10}>
-            {getFieldDecorator('scene', { initialValue: scene })(
+            {getFieldDecorator('scene', { initialValue: sceneConfig })(
               <Cascader options={mediaType} allowClear={false} />
             )}
           </Col>
         </Row>
       </Item>
+      <Item label="代理">
+        <Row>
+          <Col span={10}>
+            {getFieldDecorator('proxyEnable', {
+              initialValue: proxyConfig.enable,
+              valuePropName: 'checked'
+            })(<Switch />)}
+            {getFieldDecorator('proxyUrl', {
+              initialValue: proxyConfig.url
+            })(<Input />)}
+          </Col>
+        </Row>
+      </Item>
     </Form>
   );
 };

app/scraper/core/index.ts

@@ -1,20 +1,32 @@
 import { writeFile, mkdirp } from 'fs-extra';
-// import javbus from '../javBus';
 import { downloadImg, emitter } from '../../utils';
 import { EventType, QueryOpt, ToolHead, FileNode } from '@types';
+import config from '@config';
 
 const saveAsserts = async (model, file) => {
+  const proxy = config.get('proxy');
+  const url = proxy.enable ? proxy.url : '';
   const json = model.getModel();
   await mkdirp(`${file.wpath}/.actors`);
   return Promise.all([
     writeFile(
       `${file.wpath + file.title}.nfo`,
       `<?xml version="1.0" encoding="utf-8" standalone="yes"?>${model.getXML()}`
     ),
-    downloadImg(json.art.poster._text, `${file.wpath + file.title}-poster.jpg`),
-    downloadImg(json.art.fanart._text, `${file.wpath + file.title}-fanart.jpg`),
+    downloadImg(
+      json.art.poster._text,
+      `${file.wpath + file.title}-poster.jpg`,
+      { proxy: url }
+    ),
+    downloadImg(
+      json.art.fanart._text,
+      `${file.wpath + file.title}-fanart.jpg`,
+      { proxy: url }
+    ),
     json.actor.map(v =>
-      downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`)
+      downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`, {
+        proxy: url
+      })
     )
   ])
     .then(() => model)
@@ -43,7 +55,6 @@ class Scraper {
 
   async start(queryOpts: QueryOpt[], name: string): Promise<TaskResult> {
     this.stopFlag = false;
-    console.log(name, queryOpts);
     const failureTasks = [];
     const successTasks = [];
     const { head } = this.heads.find(t => t.name === name);

app/scraper/heads/avsox.ts

@@ -1,20 +1,19 @@
-import request from 'request-promise';
 import cheerio from 'cheerio';
 import MovieModel from '../core/model';
 import { MediaKeys } from '@types';
 
-export default {
+export default request => ({
   head: async (queryString: string): Promise<any> => {
     const movieModel = new MovieModel();
     const encodedQueryString = encodeURIComponent(queryString);
-    const searchPage = await request(
-      `https://avsox.asia/cn/search/${encodedQueryString}`
-    );
+    const searchPage = await request({
+      url: `https://avsox.asia/cn/search/${encodedQueryString}`
+    });
     const infoPageUrl = cheerio
       .load(searchPage)('.movie-box')
       .attr('href')
       .replace(/https:\/\//, 'http://');
-    const $ = cheerio.load(await request(infoPageUrl));
+    const $ = cheerio.load(await request({ url: infoPageUrl }));
     movieModel.setModel({
       title: {
         _text: $('h3')
@@ -65,4 +64,4 @@ export default {
   },
   name: 'avsox',
   type: [MediaKeys.Movie, MediaKeys.Gentleman]
-};
+});

app/scraper/heads/index.ts

@@ -1,7 +1,8 @@
+import r from '../request';
 import { ToolHead } from '@types';
 import tmdb from './tmdb';
 import javbus from './javbus';
 import avsox from './avsox';
 
-const heads: ToolHead[] = [tmdb, javbus, avsox];
+const heads: ToolHead[] = [tmdb(r), javbus(r), avsox(r)];
 export default heads;

app/scraper/heads/javbus.ts

@@ -1,20 +1,19 @@
-import request from 'request-promise';
 import cheerio from 'cheerio';
 import MovieModel from '../core/model';
 import { MediaKeys } from '@types';
 
-export default {
+export default request => ({
   head: async (queryString: string): Promise<any> => {
     const movieModel = new MovieModel();
     const encodedQueryString = encodeURIComponent(queryString);
-    const searchPage = await request(
-      `http://www.javbus.com/uncensored/search/${encodedQueryString}`
-    );
+    const searchPage = await request({
+      url: `http://www.javbus.com/uncensored/search/${encodedQueryString}`
+    });
     const infoPageUrl = cheerio
       .load(searchPage)('.movie-box')
       .attr('href')
       .replace(/https:\/\//, 'http://');
-    const $ = cheerio.load(await request(infoPageUrl));
+    const $ = cheerio.load(await request({ url: infoPageUrl }));
     movieModel.setModel({
       title: {
         _text: $('h3')
@@ -65,4 +64,4 @@ export default {
   },
   name: 'javbus',
   type: [MediaKeys.Movie, MediaKeys.Gentleman]
-};
+});

app/scraper/heads/tmdb.ts

@@ -1,8 +1,7 @@
-import request from 'request-promise';
 import MovieModel from '../core/model';
 import { MovieModelType, MediaKeys } from '@types';
 
-export default {
+export default request => ({
   head: async (queryString: string): Promise<MovieModelType> => {
     const movieModel = new MovieModel();
     const baseUrl = 'http://api.themoviedb.org/3';
@@ -77,4 +76,4 @@ export default {
   },
   name: 'TMDB',
   type: [MediaKeys.Movie, MediaKeys.Normal]
-};
+});

app/scraper/request.ts

@@ -0,0 +1,8 @@
+import request from 'request-promise';
+import config from '@config';
+
+export default opts => {
+  const proxy = config.get('proxy');
+  const url = proxy.enable ? proxy.url : '';
+  return request({ ...opts, proxy: url });
+};

app/utils/index.ts

@@ -96,9 +96,9 @@ export const sleep = time =>
     }
   });
 
-export const downloadImg = (url, ipath) =>
+export const downloadImg = (url, ipath, opt?) =>
   new Promise((resolve, reject) => {
-    request(url)
+    request({ url, ...opt })
       .pipe(fs.createWriteStream(ipath))
       .on('finish', () => {
         resolve(ipath);

package.json

@@ -1,7 +1,7 @@
 {
   "name": "u-scraper",
   "productName": "uScraper",
-  "version": "0.1.0-beta",
+  "version": "0.2.0-beta",
   "description": "A scraper that switches between normal mode and gentleman mode, built on Eletron, React",
   "scripts": {
     "build": "concurrently \"yarn build-main\" \"yarn build-renderer\"",