Update Jackett search engine (multi thread). Resolves #210 (#239)

* Make search faster with multi-thread implementation
* Show error messages per indexer
* Number of threads is configurable in jackett.json config file

Based on #222 Thank you @galeksandrp and @WojtekKowaluk

nova3/engines/jackett.py

@@ -1,14 +1,17 @@
-#VERSION: 3.5
+#VERSION: 4.0
 # AUTHORS: Diego de las Heras ([email protected])
 # CONTRIBUTORS: ukharley
 #               hannsen (github.com/hannsen)
+#               Alexander Georgievskiy <[email protected]>
 
 import json
 import os
 import xml.etree.ElementTree
 from urllib.parse import urlencode, unquote
 from urllib import request as urllib_request
 from http.cookiejar import CookieJar
+from multiprocessing.dummy import Pool
+from threading import Lock
 
 from novaprinter import prettyPrinter
 from helpers import download_file
@@ -20,9 +23,11 @@
 CONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), CONFIG_FILE)
 CONFIG_DATA = {
     'api_key': 'YOUR_API_KEY_HERE',  # jackett api
-    'tracker_first': False,          # (False/True) add tracker name to beginning of search result
     'url': 'http://127.0.0.1:9117',  # jackett url
+    'tracker_first': False,          # (False/True) add tracker name to beginning of search result
+    'thread_count': 20,              # number of threads to use for http requests
 }
+PRINTER_THREAD_LOCK = Lock()
 
 
 def load_configuration():
@@ -32,17 +37,27 @@ def load_configuration():
         with open(CONFIG_PATH) as f:
             CONFIG_DATA = json.load(f)
     except ValueError:
-        # if file exists but it's malformed we load add a flag
+        # if file exists, but it's malformed we load add a flag
         CONFIG_DATA['malformed'] = True
     except Exception:
         # if file doesn't exist, we create it
-        with open(CONFIG_PATH, 'w') as f:
-            f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True))
+        save_configuration()
 
     # do some checks
     if any(item not in CONFIG_DATA for item in ['api_key', 'tracker_first', 'url']):
         CONFIG_DATA['malformed'] = True
 
+    # add missing keys
+    if 'thread_count' not in CONFIG_DATA:
+        CONFIG_DATA['thread_count'] = 20
+        save_configuration()
+
+
+def save_configuration():
+    global CONFIG_PATH, CONFIG_DATA
+    with open(CONFIG_PATH, 'w') as f:
+        f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True))
+
 
 load_configuration()
 ###############################################################################
@@ -52,6 +67,7 @@ class jackett(object):
     name = 'Jackett'
     url = CONFIG_DATA['url'] if CONFIG_DATA['url'][-1] != '/' else CONFIG_DATA['url'][:-1]
     api_key = CONFIG_DATA['api_key']
+    thread_count = CONFIG_DATA['thread_count']
     supported_categories = {
         'all': None,
         'anime': ['5070'],
@@ -87,6 +103,37 @@ def search(self, what, cat='all'):
             self.handle_error("api key error", what)
             return
 
+        # search in Jackett API
+        if self.thread_count > 1:
+            args = []
+            indexers = self.get_jackett_indexers(what)
+            for indexer in indexers:
+                args.append((what, category, indexer))
+            with Pool(min(len(indexers), self.thread_count)) as pool:
+                pool.starmap(self.search_jackett_indexer, args)
+        else:
+            self.search_jackett_indexer(what, category, 'all')
+
+    def get_jackett_indexers(self, what):
+        params = [
+            ('apikey', self.api_key),
+            ('t', 'indexers'),
+            ('configured', 'true')
+        ]
+        params = urlencode(params)
+        jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params
+        response = self.get_response(jacket_url)
+        if response is None:
+            self.handle_error("connection error getting indexer list", what)
+            return
+        # process results
+        response_xml = xml.etree.ElementTree.fromstring(response)
+        indexers = []
+        for indexer in response_xml.findall('indexer'):
+            indexers.append(indexer.attrib['id'])
+        return indexers
+
+    def search_jackett_indexer(self, what, category, indexer_id):
         # prepare jackett url
         params = [
             ('apikey', self.api_key),
@@ -95,12 +142,11 @@ def search(self, what, cat='all'):
         if category is not None:
             params.append(('cat', ','.join(category)))
         params = urlencode(params)
-        jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params
+        jacket_url = self.url + "/api/v2.0/indexers/" + indexer_id + "/results/torznab/api?%s" % params  # noqa
         response = self.get_response(jacket_url)
         if response is None:
-            self.handle_error("connection error", what)
+            self.handle_error("connection error for indexer: " + indexer_id, what)
             return
-
         # process search results
         response_xml = xml.etree.ElementTree.fromstring(response)
         for result in response_xml.find('channel').findall('item'):
@@ -151,18 +197,11 @@ def search(self, what, cat='all'):
             # note: engine_url can't be changed, torrent download stops working
             res['engine_url'] = self.url
 
-            prettyPrinter(self.escape_pipe(res))
+            self.pretty_printer_thread_safe(res)
 
     def generate_xpath(self, tag):
         return './{http://torznab.com/schemas/2015/feed}attr[@name="%s"]' % tag
 
-    # Safety measure until it's fixed in prettyPrinter
-    def escape_pipe(self, dictionary):
-        for key in dictionary.keys():
-            if isinstance(dictionary[key], str):
-                dictionary[key] = dictionary[key].replace('|', '%7C')
-        return dictionary
-
     def get_response(self, query):
         response = None
         try:
@@ -181,7 +220,7 @@ def get_response(self, query):
     def handle_error(self, error_msg, what):
         # we need to print the search text to be displayed in qBittorrent when
         # 'Torrent names only' is enabled
-        prettyPrinter({
+        self.pretty_printer_thread_safe({
             'seeds': -1,
             'size': -1,
             'leech': -1,
@@ -191,6 +230,18 @@ def handle_error(self, error_msg, what):
             'name': "Jackett: %s! Right-click this row and select 'Open description page' to open help. Configuration file: '%s' Search: '%s'" % (error_msg, CONFIG_PATH, what)  # noqa
         })
 
+    def pretty_printer_thread_safe(self, dictionary):
+        global PRINTER_THREAD_LOCK
+        with PRINTER_THREAD_LOCK:
+            prettyPrinter(self.escape_pipe(dictionary))
+
+    def escape_pipe(self, dictionary):
+        # Safety measure until it's fixed in prettyPrinter
+        for key in dictionary.keys():
+            if isinstance(dictionary[key], str):
+                dictionary[key] = dictionary[key].replace('|', '%7C')
+        return dictionary
+
 
 if __name__ == "__main__":
     jackett_se = jackett()

nova3/engines/versions.txt

@@ -1,5 +1,5 @@
 eztv: 1.14
-jackett: 3.5
+jackett: 4.0
 limetorrents: 4.7
 piratebay: 3.2
 rarbg: 2.14