diff --git a/.travis.yml b/.travis.yml
index fe2489e..3a3784f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,6 +52,8 @@ cache:
before_install:
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXX="g++-8"; fi
+ - rm -rf yarn.lock
+ - rm -rf package-lock.json
install:
- yarn --ignore-engines
diff --git a/app/config-store/index.ts b/app/config-store/index.ts
index 7cada90..26f0329 100644
--- a/app/config-store/index.ts
+++ b/app/config-store/index.ts
@@ -3,7 +3,11 @@ import Store from 'electron-store';
const store = new Store({
defaults: {
scene: ['movie', 'normal'],
- tags: ['无']
+ tags: ['无'],
+ proxy: {
+ enable: false,
+ url: 'http://127.0.0.1:1087'
+ }
}
});
diff --git a/app/containers/Home/ScrapeInfoModal/index.tsx b/app/containers/Home/ScrapeInfoModal/index.tsx
index f12b678..d2d6d43 100644
--- a/app/containers/Home/ScrapeInfoModal/index.tsx
+++ b/app/containers/Home/ScrapeInfoModal/index.tsx
@@ -43,7 +43,6 @@ const ScrapeModal = ({ visible, taskQueue, onCancel, handleTaskEnd }) => {
str: task.file.key === key ? str : task.str ? task.str : ''
}));
setTaskQ(_taskQ);
- console.log(_taskQ, 1);
lastTaskQ.current = _taskQ;
});
emitter.on(EventType.SCRAPE_SUCCESS, ({ key }, json) => {
diff --git a/app/containers/Home/SettingModal/form.tsx b/app/containers/Home/SettingModal/form.tsx
index 71fbc38..7f17eb6 100644
--- a/app/containers/Home/SettingModal/form.tsx
+++ b/app/containers/Home/SettingModal/form.tsx
@@ -1,5 +1,5 @@
-import React, { useImperativeHandle, forwardRef, useState } from 'react';
-import { Select, Form, Cascader, Row, Col } from 'antd';
+import React, { useImperativeHandle, forwardRef } from 'react';
+import { Select, Form, Cascader, Row, Col, Switch, Input } from 'antd';
import config from '@config';
import styles from './index.less';
import { mediaType } from '@scraper';
@@ -8,22 +8,29 @@ const { Option } = Select;
const { Item } = Form;
const SettingForm = ({ form }, ref) => {
- const tags = config.get('tags');
- const scene = config.get('scene', ['movie', 'normal']);
+ const tagsConfig = config.get('tags');
+ const sceneConfig = config.get('scene', ['movie', 'normal']);
+ const proxyConfig = config.get('proxy');
// const sceneSourceMapping = config.get('sceneSourceMapping', {});
useImperativeHandle(ref, () => ({
submitForm: () =>
new Promise((resolve, reject) => {
form.validateFields((err, values) => {
if (err) return reject(err);
- config.set(values);
- console.log(config.path);
+ const { tags, scene, proxyUrl, proxyEnable } = values;
+ config.set({
+ tags,
+ scene,
+ proxy: {
+ enable: proxyEnable,
+ url: proxyUrl
+ }
+ });
resolve();
});
})
}));
- // const [mediaSourceVisible, setMediaSourceVisible] = useState(false);
- console.log(useState);
+
const { getFieldDecorator } = form;
return (
);
};
diff --git a/app/scraper/core/index.ts b/app/scraper/core/index.ts
index 0b5e893..d99678f 100644
--- a/app/scraper/core/index.ts
+++ b/app/scraper/core/index.ts
@@ -1,9 +1,11 @@
import { writeFile, mkdirp } from 'fs-extra';
-// import javbus from '../javBus';
import { downloadImg, emitter } from '../../utils';
import { EventType, QueryOpt, ToolHead, FileNode } from '@types';
+import config from '@config';
const saveAsserts = async (model, file) => {
+ const proxy = config.get('proxy');
+ const url = proxy.enable ? proxy.url : '';
const json = model.getModel();
await mkdirp(`${file.wpath}/.actors`);
return Promise.all([
@@ -11,10 +13,20 @@ const saveAsserts = async (model, file) => {
`${file.wpath + file.title}.nfo`,
`${model.getXML()}`
),
- downloadImg(json.art.poster._text, `${file.wpath + file.title}-poster.jpg`),
- downloadImg(json.art.fanart._text, `${file.wpath + file.title}-fanart.jpg`),
+ downloadImg(
+ json.art.poster._text,
+ `${file.wpath + file.title}-poster.jpg`,
+ { proxy: url }
+ ),
+ downloadImg(
+ json.art.fanart._text,
+ `${file.wpath + file.title}-fanart.jpg`,
+ { proxy: url }
+ ),
json.actor.map(v =>
- downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`)
+ downloadImg(v.thumb._text, `${file.wpath}.actors/${v.name._text}.jpg`, {
+ proxy: url
+ })
)
])
.then(() => model)
@@ -43,7 +55,6 @@ class Scraper {
async start(queryOpts: QueryOpt[], name: string): Promise {
this.stopFlag = false;
- console.log(name, queryOpts);
const failureTasks = [];
const successTasks = [];
const { head } = this.heads.find(t => t.name === name);
diff --git a/app/scraper/heads/avsox.ts b/app/scraper/heads/avsox.ts
index cc7bf28..2d35bca 100644
--- a/app/scraper/heads/avsox.ts
+++ b/app/scraper/heads/avsox.ts
@@ -1,20 +1,19 @@
-import request from 'request-promise';
import cheerio from 'cheerio';
import MovieModel from '../core/model';
import { MediaKeys } from '@types';
-export default {
+export default request => ({
head: async (queryString: string): Promise => {
const movieModel = new MovieModel();
const encodedQueryString = encodeURIComponent(queryString);
- const searchPage = await request(
- `https://avsox.asia/cn/search/${encodedQueryString}`
- );
+ const searchPage = await request({
+ url: `https://avsox.asia/cn/search/${encodedQueryString}`
+ });
const infoPageUrl = cheerio
.load(searchPage)('.movie-box')
.attr('href')
.replace(/https:\/\//, 'http://');
- const $ = cheerio.load(await request(infoPageUrl));
+ const $ = cheerio.load(await request({ url: infoPageUrl }));
movieModel.setModel({
title: {
_text: $('h3')
@@ -65,4 +64,4 @@ export default {
},
name: 'avsox',
type: [MediaKeys.Movie, MediaKeys.Gentleman]
-};
+});
diff --git a/app/scraper/heads/index.ts b/app/scraper/heads/index.ts
index 0fa91a2..72927b4 100644
--- a/app/scraper/heads/index.ts
+++ b/app/scraper/heads/index.ts
@@ -1,7 +1,8 @@
+import r from '../request';
import { ToolHead } from '@types';
import tmdb from './tmdb';
import javbus from './javbus';
import avsox from './avsox';
-const heads: ToolHead[] = [tmdb, javbus, avsox];
+const heads: ToolHead[] = [tmdb(r), javbus(r), avsox(r)];
export default heads;
diff --git a/app/scraper/heads/javbus.ts b/app/scraper/heads/javbus.ts
index 83ee4b9..f46ddc3 100644
--- a/app/scraper/heads/javbus.ts
+++ b/app/scraper/heads/javbus.ts
@@ -1,20 +1,19 @@
-import request from 'request-promise';
import cheerio from 'cheerio';
import MovieModel from '../core/model';
import { MediaKeys } from '@types';
-export default {
+export default request => ({
head: async (queryString: string): Promise => {
const movieModel = new MovieModel();
const encodedQueryString = encodeURIComponent(queryString);
- const searchPage = await request(
- `http://www.javbus.com/uncensored/search/${encodedQueryString}`
- );
+ const searchPage = await request({
+ url: `http://www.javbus.com/uncensored/search/${encodedQueryString}`
+ });
const infoPageUrl = cheerio
.load(searchPage)('.movie-box')
.attr('href')
.replace(/https:\/\//, 'http://');
- const $ = cheerio.load(await request(infoPageUrl));
+ const $ = cheerio.load(await request({ url: infoPageUrl }));
movieModel.setModel({
title: {
_text: $('h3')
@@ -65,4 +64,4 @@ export default {
},
name: 'javbus',
type: [MediaKeys.Movie, MediaKeys.Gentleman]
-};
+});
diff --git a/app/scraper/heads/tmdb.ts b/app/scraper/heads/tmdb.ts
index 14bde02..6ffa41c 100644
--- a/app/scraper/heads/tmdb.ts
+++ b/app/scraper/heads/tmdb.ts
@@ -1,8 +1,7 @@
-import request from 'request-promise';
import MovieModel from '../core/model';
import { MovieModelType, MediaKeys } from '@types';
-export default {
+export default request => ({
head: async (queryString: string): Promise => {
const movieModel = new MovieModel();
const baseUrl = 'http://api.themoviedb.org/3';
@@ -77,4 +76,4 @@ export default {
},
name: 'TMDB',
type: [MediaKeys.Movie, MediaKeys.Normal]
-};
+});
diff --git a/app/scraper/request.ts b/app/scraper/request.ts
new file mode 100644
index 0000000..c957c96
--- /dev/null
+++ b/app/scraper/request.ts
@@ -0,0 +1,8 @@
+import request from 'request-promise';
+import config from '@config';
+
+export default opts => {
+ const proxy = config.get('proxy');
+ const url = proxy.enable ? proxy.url : '';
+ return request({ ...opts, proxy: url });
+};
diff --git a/app/utils/index.ts b/app/utils/index.ts
index 4139190..cc10cb0 100644
--- a/app/utils/index.ts
+++ b/app/utils/index.ts
@@ -96,9 +96,9 @@ export const sleep = time =>
}
});
-export const downloadImg = (url, ipath) =>
+export const downloadImg = (url, ipath, opt?) =>
new Promise((resolve, reject) => {
- request(url)
+ request({ url, ...opt })
.pipe(fs.createWriteStream(ipath))
.on('finish', () => {
resolve(ipath);
diff --git a/package.json b/package.json
index 2a24a2a..72cb672 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "u-scraper",
"productName": "uScraper",
- "version": "0.1.0-beta",
+ "version": "0.2.0-beta",
"description": "A scraper that switches between normal mode and gentleman mode, built on Eletron, React",
"scripts": {
"build": "concurrently \"yarn build-main\" \"yarn build-renderer\"",