From 6677f9a8c079f58c5352207144df55b13cfa9ede Mon Sep 17 00:00:00 2001 From: Diego Heras Date: Sun, 21 May 2023 15:33:02 +0200 Subject: [PATCH] Update Jackett search engine (multi thread). Resolves #210 (#239) * Make search faster with multi-thread implementation * Show error messages per indexer * Number of threads is configurable in jackett.json config file Based on #222 Thank you @galeksandrp and @WojtekKowaluk --- nova3/engines/jackett.py | 85 ++++++++++++++++++++++++++++++-------- nova3/engines/versions.txt | 2 +- 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/nova3/engines/jackett.py b/nova3/engines/jackett.py index bbf58e1..291ac7d 100644 --- a/nova3/engines/jackett.py +++ b/nova3/engines/jackett.py @@ -1,7 +1,8 @@ -#VERSION: 3.5 +#VERSION: 4.0 # AUTHORS: Diego de las Heras (ngosang@hotmail.es) # CONTRIBUTORS: ukharley # hannsen (github.com/hannsen) +# Alexander Georgievskiy import json import os @@ -9,6 +10,8 @@ from urllib.parse import urlencode, unquote from urllib import request as urllib_request from http.cookiejar import CookieJar +from multiprocessing.dummy import Pool +from threading import Lock from novaprinter import prettyPrinter from helpers import download_file @@ -20,9 +23,11 @@ CONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), CONFIG_FILE) CONFIG_DATA = { 'api_key': 'YOUR_API_KEY_HERE', # jackett api - 'tracker_first': False, # (False/True) add tracker name to beginning of search result 'url': 'http://127.0.0.1:9117', # jackett url + 'tracker_first': False, # (False/True) add tracker name to beginning of search result + 'thread_count': 20, # number of threads to use for http requests } +PRINTER_THREAD_LOCK = Lock() def load_configuration(): @@ -32,17 +37,27 @@ def load_configuration(): with open(CONFIG_PATH) as f: CONFIG_DATA = json.load(f) except ValueError: - # if file exists but it's malformed we load add a flag + # if file exists, but it's malformed we load add a flag CONFIG_DATA['malformed'] = True except Exception: # if file doesn't exist, we create it - with open(CONFIG_PATH, 'w') as f: - f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True)) + save_configuration() # do some checks if any(item not in CONFIG_DATA for item in ['api_key', 'tracker_first', 'url']): CONFIG_DATA['malformed'] = True + # add missing keys + if 'thread_count' not in CONFIG_DATA: + CONFIG_DATA['thread_count'] = 20 + save_configuration() + + +def save_configuration(): + global CONFIG_PATH, CONFIG_DATA + with open(CONFIG_PATH, 'w') as f: + f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True)) + load_configuration() ############################################################################### @@ -52,6 +67,7 @@ class jackett(object): name = 'Jackett' url = CONFIG_DATA['url'] if CONFIG_DATA['url'][-1] != '/' else CONFIG_DATA['url'][:-1] api_key = CONFIG_DATA['api_key'] + thread_count = CONFIG_DATA['thread_count'] supported_categories = { 'all': None, 'anime': ['5070'], @@ -87,6 +103,37 @@ def search(self, what, cat='all'): self.handle_error("api key error", what) return + # search in Jackett API + if self.thread_count > 1: + args = [] + indexers = self.get_jackett_indexers(what) + for indexer in indexers: + args.append((what, category, indexer)) + with Pool(min(len(indexers), self.thread_count)) as pool: + pool.starmap(self.search_jackett_indexer, args) + else: + self.search_jackett_indexer(what, category, 'all') + + def get_jackett_indexers(self, what): + params = [ + ('apikey', self.api_key), + ('t', 'indexers'), + ('configured', 'true') + ] + params = urlencode(params) + jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params + response = self.get_response(jacket_url) + if response is None: + self.handle_error("connection error getting indexer list", what) + return + # process results + response_xml = xml.etree.ElementTree.fromstring(response) + indexers = [] + for indexer in response_xml.findall('indexer'): + indexers.append(indexer.attrib['id']) + return indexers + + def search_jackett_indexer(self, what, category, indexer_id): # prepare jackett url params = [ ('apikey', self.api_key), @@ -95,12 +142,11 @@ def search(self, what, cat='all'): if category is not None: params.append(('cat', ','.join(category))) params = urlencode(params) - jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params + jacket_url = self.url + "/api/v2.0/indexers/" + indexer_id + "/results/torznab/api?%s" % params # noqa response = self.get_response(jacket_url) if response is None: - self.handle_error("connection error", what) + self.handle_error("connection error for indexer: " + indexer_id, what) return - # process search results response_xml = xml.etree.ElementTree.fromstring(response) for result in response_xml.find('channel').findall('item'): @@ -151,18 +197,11 @@ def search(self, what, cat='all'): # note: engine_url can't be changed, torrent download stops working res['engine_url'] = self.url - prettyPrinter(self.escape_pipe(res)) + self.pretty_printer_thread_safe(res) def generate_xpath(self, tag): return './{http://torznab.com/schemas/2015/feed}attr[@name="%s"]' % tag - # Safety measure until it's fixed in prettyPrinter - def escape_pipe(self, dictionary): - for key in dictionary.keys(): - if isinstance(dictionary[key], str): - dictionary[key] = dictionary[key].replace('|', '%7C') - return dictionary - def get_response(self, query): response = None try: @@ -181,7 +220,7 @@ def get_response(self, query): def handle_error(self, error_msg, what): # we need to print the search text to be displayed in qBittorrent when # 'Torrent names only' is enabled - prettyPrinter({ + self.pretty_printer_thread_safe({ 'seeds': -1, 'size': -1, 'leech': -1, @@ -191,6 +230,18 @@ def handle_error(self, error_msg, what): 'name': "Jackett: %s! Right-click this row and select 'Open description page' to open help. Configuration file: '%s' Search: '%s'" % (error_msg, CONFIG_PATH, what) # noqa }) + def pretty_printer_thread_safe(self, dictionary): + global PRINTER_THREAD_LOCK + with PRINTER_THREAD_LOCK: + prettyPrinter(self.escape_pipe(dictionary)) + + def escape_pipe(self, dictionary): + # Safety measure until it's fixed in prettyPrinter + for key in dictionary.keys(): + if isinstance(dictionary[key], str): + dictionary[key] = dictionary[key].replace('|', '%7C') + return dictionary + if __name__ == "__main__": jackett_se = jackett() diff --git a/nova3/engines/versions.txt b/nova3/engines/versions.txt index 92d5b79..b7e4a90 100644 --- a/nova3/engines/versions.txt +++ b/nova3/engines/versions.txt @@ -1,5 +1,5 @@ eztv: 1.14 -jackett: 3.5 +jackett: 4.0 limetorrents: 4.7 piratebay: 3.2 rarbg: 2.14