feat(scraper):Optimized and added avsox information source

app/config-store/index.ts

@@ -1,5 +1,10 @@
 import Store from 'electron-store';
 
-const store = new Store();
+const store = new Store({
+  defaults: {
+    scene: ['movie', 'normal'],
+    tags: ['无']
+  }
+});
 
 export default store;

app/containers/Home/HeaderContent/index.tsx

@@ -1,6 +1,6 @@
-import React, { useState } from 'react';
+import React, { useState, useRef, useEffect } from 'react';
 import { connect } from 'react-redux';
-import { Button, Row, Col, Input, Dropdown, Menu } from 'antd';
+import { Button, Row, Col, Input, Dropdown, Menu, message } from 'antd';
 
 import * as R from 'ramda';
 import CRD from '@vdts/collect-video';
@@ -33,6 +33,13 @@ const HeaderContent = ({
   const [modalVisible, setModalVisible] = useState(false);
   const [settingVisible, setSettingVisible] = useState(false);
   const [taskQueue, setTaskQueue] = useState([]);
+  const [tasks, setTasks] = useState([]);
+  const scraperHead = useRef('');
+  useEffect(() => {
+    if (tasks.length) {
+      scrape.start(tasks, scraperHead.current);
+    }
+  }, [tasks]);
   const handleInput = filename => {
     dispatch(setSelectedFilename(filename));
   };
@@ -53,6 +60,10 @@ const HeaderContent = ({
     );
   };
   const handleScrape = (headName?: string) => {
+    const defaultHead = getHeadsByMediaType(config.get('scene'))[0];
+    if (!defaultHead) {
+      return message.error('当前场景未找到信息源');
+    }
     dispatch(changeFailureKeys([]));
     const head = headName || getHeadsByMediaType(config.get('scene'))[0].name;
     let _taskQueue = [];
@@ -86,7 +97,8 @@ const HeaderContent = ({
     }
     setModalVisible(true);
     setTaskQueue(_taskQueue);
-    scrape.start(_tasks, head);
+    setTasks(_tasks);
+    scraperHead.current = head;
   };
   const handleRebuild = () => {
     CRD(tree.wpath)
@@ -96,9 +108,11 @@ const HeaderContent = ({
         dispatch(changeSelected(''));
         dispatch(changeFailureKeys([]));
         dispatch(selectFiles(_tree[0]));
+        message.success('格式化成功');
         return res;
       })
       .catch(e => {
+        message.error('格式化失败');
         console.log(e);
       });
   };
@@ -108,7 +122,7 @@ const HeaderContent = ({
         handleScrape(e.key);
       }}
     >
-      {getHeadsByMediaType(config.get('scene')).map(head => (
+      {(getHeadsByMediaType(config.get('scene')) || []).map(head => (
         <Menu.Item key={head.name}>{head.name}</Menu.Item>
       ))}
     </Menu>

app/containers/Home/ScrapeInfoModal/index.tsx

@@ -36,15 +36,14 @@ const ScrapeModal = ({ visible, taskQueue, onCancel, handleTaskEnd }) => {
     lastTaskQ.current = taskQueue;
   }, [taskQueue]);
   useEffect(() => {
-    console.log(44);
     emitter.on(EventType.SCRAPE_PENDING, ({ key }, str) => {
-      console.log(33);
       const _taskQ = lastTaskQ.current.map(task => ({
         ...task,
         status: task.file.key === key ? 'pending' : task.status,
         str: task.file.key === key ? str : task.str ? task.str : ''
       }));
       setTaskQ(_taskQ);
+      console.log(_taskQ, 1);
       lastTaskQ.current = _taskQ;
     });
     emitter.on(EventType.SCRAPE_SUCCESS, ({ key }, json) => {

app/containers/Home/SettingModal/form.tsx

@@ -8,7 +8,7 @@ const { Option } = Select;
 const { Item } = Form;
 
 const SettingForm = ({ form }, ref) => {
-  const tags = config.get('tags', []);
+  const tags = config.get('tags');
   const scene = config.get('scene', ['movie', 'normal']);
   // const sceneSourceMapping = config.get('sceneSourceMapping', {});
   useImperativeHandle(ref, () => ({
@@ -35,7 +35,7 @@ const SettingForm = ({ form }, ref) => {
       <Item label="预设标签">
         {getFieldDecorator('tags', { initialValue: tags })(
           <Select mode="tags" style={{ width: '100%' }} placeholder="Tags">
-            {tags.map((tag: string) => (
+            {tags.map(tag => (
               <Option key={tag}>{tag}</Option>
             ))}
           </Select>

app/scraper/heads/avsox.ts

@@ -0,0 +1,68 @@
+import request from 'request-promise';
+import cheerio from 'cheerio';
+import MovieModel from '../core/model';
+import { MediaKeys } from '@types';
+
+export default {
+  head: async (queryString: string): Promise<any> => {
+    const movieModel = new MovieModel();
+    const encodedQueryString = encodeURIComponent(queryString);
+    const searchPage = await request(
+      `https://avsox.asia/cn/search/${encodedQueryString}`
+    );
+    const infoPageUrl = cheerio
+      .load(searchPage)('.movie-box')
+      .attr('href')
+      .replace(/https:\/\//, 'http://');
+    const $ = cheerio.load(await request(infoPageUrl));
+    movieModel.setModel({
+      title: {
+        _text: $('h3')
+          .text()
+          .trim()
+      },
+      premiered: {
+        _text: $('.info>p:nth-child(2)')
+          .text()
+          .split(': ')[1]
+          .trim()
+      },
+      art: {
+        poster: {
+          _text: $('.bigImage')
+            .attr('href')
+            .trim()
+            .replace(/https:\/\//, 'http://')
+        },
+        fanart: {
+          _text: $('.bigImage')
+            .attr('href')
+            .trim()
+            .replace(/https:\/\//, 'http://')
+        }
+      },
+      actor: $('#avatar-waterfall .avatar-box img')
+        .map((index, $actor) => ({
+          name: { _text: $actor.attribs.title.trim() },
+          thumb: {
+            _text: $actor.attribs.src.trim().replace(/https:\/\//, 'http://')
+          }
+        }))
+        .toArray(),
+      uniqueid: [
+        {
+          _attributes: { type: '1', default: true },
+          _text: $('.info>p:nth-child(1)>span:nth-child(2)')
+            .text()
+            .trim()
+        }
+      ],
+      genre: $('.info .genre>a')
+        .map((index, $genre) => ({ _text: $genre.firstChild.data.trim() }))
+        .toArray()
+    });
+    return movieModel;
+  },
+  name: 'avsox',
+  type: [MediaKeys.Movie, MediaKeys.Gentleman]
+};

app/scraper/heads/index.ts

@@ -1,4 +1,7 @@
+import { ToolHead } from '@types';
 import tmdb from './tmdb';
 import javbus from './javbus';
+import avsox from './avsox';
 
-export default [tmdb, javbus];
+const heads: ToolHead[] = [tmdb, javbus, avsox];
+export default heads;

app/scraper/heads/javbus.ts

@@ -57,7 +57,7 @@ export default {
             .trim()
         }
       ],
-      genre: $('.info>p:nth-child(6) .genre>a')
+      genre: $('.info .genre>a')
         .map((index, $actor) => ({ _text: $actor.firstChild.data.trim() }))
         .toArray()
     });

app/scraper/heads/tmdb.ts

@@ -1,5 +1,4 @@
 import request from 'request-promise';
-// import cheerio from 'cheerio';
 import MovieModel from '../core/model';
 import { MovieModelType, MediaKeys } from '@types';
 
@@ -21,8 +20,6 @@ export default {
       json: true
     });
 
-    console.log(movieModel, queryString, 111);
-
     if (!res.total_results) {
       throw new Error('无影片信息');
     }
@@ -48,7 +45,7 @@ export default {
       genres
     } = info;
     movieModel.setModel({
-      title,
+      title: { _text: title },
       plot: {
         _text: overview
       },

app/scraper/mediaType.ts

@@ -32,5 +32,5 @@ const mediaSource = heads.reduce((acc, head) => {
 
 export const getHeadsByMediaType: (type: string[]) => ToolHead[] = type => {
   const sourceId = type.join('$$');
-  return mediaSource[sourceId];
+  return mediaSource[sourceId] || [];
 };