forked from qbittorrent/search-plugins
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[jackett] fix qbittorrent#210 speed up search
`Jackett` from version [`v0.10.589` Jan 2, 2019](Jackett/Jackett@25ddcb1) does not recommend using `all` indexer. Problem seems to be that many clients including `nova2.py` will wait for `HTTP` request to `Jackett` to finish before starting to process `HTTP` response. This effectivly slows `all` indexer to the speed of most slow tracker. While it's possible to process received parts of `HTTP` response before `HTTP` request is finished, `Jackett` instead recommends separate requests to trackers. Finally rework `search` function to: 1. Retrieve list of configured indexers. 2. Make request to trackers in parallel threads. `Multithreading` is chosen instead of `multiprocessing` because we already in separate process thanks to `nova2.py` `multiprocessing` `main`. Doing `multiprocessing` in `multiprocessing` is questionable matter and actually not working for me (`python` `3.11.0`).
- Loading branch information
1 parent
ac82c6b
commit b278051
Showing
1 changed file
with
49 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,28 @@ | ||
#VERSION: 3.5 | ||
#VERSION: 3.6 | ||
# AUTHORS: Diego de las Heras ([email protected]) | ||
# CONTRIBUTORS: ukharley | ||
# hannsen (github.com/hannsen) | ||
# Alexander Georgievskiy <[email protected]> | ||
|
||
import json | ||
import os | ||
import xml.etree.ElementTree | ||
from urllib.parse import urlencode, unquote | ||
from urllib import request as urllib_request | ||
from http.cookiejar import CookieJar | ||
from multiprocessing.dummy import Pool | ||
from multiprocessing import cpu_count | ||
|
||
from novaprinter import prettyPrinter | ||
from helpers import download_file | ||
|
||
|
||
THREADED = True | ||
try: | ||
MAX_THREADS = cpu_count() | ||
except NotImplementedError: | ||
MAX_THREADS = 1 | ||
|
||
############################################################################### | ||
# load configuration from file | ||
CONFIG_FILE = 'jackett.json' | ||
|
@@ -74,36 +83,66 @@ def download_torrent(self, download_url): | |
print(download_file(download_url)) | ||
|
||
def search(self, what, cat='all'): | ||
what = unquote(what) | ||
category = self.supported_categories[cat.lower()] | ||
|
||
# check for malformed configuration | ||
if 'malformed' in CONFIG_DATA: | ||
self.handle_error("malformed configuration file", what) | ||
return | ||
return | ||
|
||
# check api_key | ||
if self.api_key == "YOUR_API_KEY_HERE": | ||
self.handle_error("api key error", what) | ||
return | ||
|
||
# prepare jackett url | ||
response = self.jackettAPIGetResponse('all', [ | ||
('t', 'indexers'), | ||
('configured', 'true') | ||
]); | ||
if response is None: | ||
self.handle_error("connection error", what) | ||
return | ||
|
||
arguments = [] | ||
for indexer in xml.etree.ElementTree.fromstring(response).findall('indexer'): | ||
arguments.append({'what': what, | ||
'jackettIndexerName': indexer.attrib['id'], | ||
'cat': cat}) | ||
|
||
if THREADED: | ||
with Pool(min(len(arguments), MAX_THREADS)) as pool: | ||
pool.map(self.jackettSearch, arguments) | ||
else: | ||
list(map(self.jackettSearch, arguments)) | ||
|
||
def jackettAPIGetResponse(self, jackettIndexerName, params): | ||
params.append(('apikey', self.api_key)) | ||
return self.get_response(self.url + "/api/v2.0/indexers/" + jackettIndexerName + "/results/torznab/api?%s" % urlencode(params)) | ||
|
||
def jackettSearch(self, arguments): | ||
what = arguments['what'] | ||
jackettIndexerName = arguments['jackettIndexerName'] | ||
cat = arguments['cat'] | ||
|
||
what = unquote(what) | ||
category = self.supported_categories[cat.lower()] | ||
|
||
# prepare jackett url | ||
params = [ | ||
('apikey', self.api_key), | ||
('q', what) | ||
] | ||
if category is not None: | ||
params.append(('cat', ','.join(category))) | ||
params = urlencode(params) | ||
jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params | ||
response = self.get_response(jacket_url) | ||
response = self.jackettAPIGetResponse(jackettIndexerName, params) | ||
if response is None: | ||
self.handle_error("connection error", what) | ||
return | ||
|
||
# process search results | ||
response_xml = xml.etree.ElementTree.fromstring(response) | ||
for result in response_xml.find('channel').findall('item'): | ||
channel = response_xml.find('channel'); | ||
if channel is None: | ||
return | ||
for result in channel.findall('item'): | ||
res = {} | ||
|
||
title = result.find('title') | ||
|