Skip to content

Commit

Permalink
[jackett] fix qbittorrent#210 speed up search
Browse files Browse the repository at this point in the history
`Jackett` from version [`v0.10.589` Jan 2, 2019](Jackett/Jackett@25ddcb1) does not recommend using `all` indexer.

Problem seems to be that many clients including `nova2.py` will wait for `HTTP` request to `Jackett` to finish before starting to process `HTTP` response.
This effectivly slows `all` indexer to the speed of most slow tracker.

While it's possible to process received parts of `HTTP` response before `HTTP` request is finished, `Jackett` instead recommends separate requests to trackers.

Finally rework `search` function to:
1. Retrieve list of configured indexers.
2. Make request to trackers in parallel threads. `Multithreading` is chosen instead of `multiprocessing` because we already in separate process thanks to `nova2.py` `multiprocessing` `main`.
Doing `multiprocessing` in `multiprocessing` is questionable matter and actually not working for me (`python` `3.11.0`).
  • Loading branch information
galeksandrp committed Jan 14, 2023
1 parent ac82c6b commit b278051
Showing 1 changed file with 49 additions and 10 deletions.
59 changes: 49 additions & 10 deletions nova3/engines/jackett.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,28 @@
#VERSION: 3.5
#VERSION: 3.6
# AUTHORS: Diego de las Heras ([email protected])
# CONTRIBUTORS: ukharley
# hannsen (github.com/hannsen)
# Alexander Georgievskiy <[email protected]>

import json
import os
import xml.etree.ElementTree
from urllib.parse import urlencode, unquote
from urllib import request as urllib_request
from http.cookiejar import CookieJar
from multiprocessing.dummy import Pool
from multiprocessing import cpu_count

from novaprinter import prettyPrinter
from helpers import download_file


THREADED = True
try:
MAX_THREADS = cpu_count()
except NotImplementedError:
MAX_THREADS = 1

###############################################################################
# load configuration from file
CONFIG_FILE = 'jackett.json'
Expand Down Expand Up @@ -74,36 +83,66 @@ def download_torrent(self, download_url):
print(download_file(download_url))

def search(self, what, cat='all'):
what = unquote(what)
category = self.supported_categories[cat.lower()]

# check for malformed configuration
if 'malformed' in CONFIG_DATA:
self.handle_error("malformed configuration file", what)
return
return

# check api_key
if self.api_key == "YOUR_API_KEY_HERE":
self.handle_error("api key error", what)
return

# prepare jackett url
response = self.jackettAPIGetResponse('all', [
('t', 'indexers'),
('configured', 'true')
]);
if response is None:
self.handle_error("connection error", what)
return

arguments = []
for indexer in xml.etree.ElementTree.fromstring(response).findall('indexer'):
arguments.append({'what': what,
'jackettIndexerName': indexer.attrib['id'],
'cat': cat})

if THREADED:
with Pool(min(len(arguments), MAX_THREADS)) as pool:
pool.map(self.jackettSearch, arguments)
else:
list(map(self.jackettSearch, arguments))

def jackettAPIGetResponse(self, jackettIndexerName, params):
params.append(('apikey', self.api_key))
return self.get_response(self.url + "/api/v2.0/indexers/" + jackettIndexerName + "/results/torznab/api?%s" % urlencode(params))

def jackettSearch(self, arguments):
what = arguments['what']
jackettIndexerName = arguments['jackettIndexerName']
cat = arguments['cat']

what = unquote(what)
category = self.supported_categories[cat.lower()]

# prepare jackett url
params = [
('apikey', self.api_key),
('q', what)
]
if category is not None:
params.append(('cat', ','.join(category)))
params = urlencode(params)
jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params
response = self.get_response(jacket_url)
response = self.jackettAPIGetResponse(jackettIndexerName, params)
if response is None:
self.handle_error("connection error", what)
return

# process search results
response_xml = xml.etree.ElementTree.fromstring(response)
for result in response_xml.find('channel').findall('item'):
channel = response_xml.find('channel');
if channel is None:
return
for result in channel.findall('item'):
res = {}

title = result.find('title')
Expand Down

0 comments on commit b278051

Please sign in to comment.