diff --git a/CB/Core.py b/CB/Core.py index e8559f3..629d9fd 100644 --- a/CB/Core.py +++ b/CB/Core.py @@ -35,6 +35,8 @@ def __init__(self): self.dirIndex = None self.wowiCache = {} self.wagoCache = {} + self.githubCache = {} + self.githubPackagerCache = {} self.wagoIdCache = None self.tukuiCache = None self.checksumCache = {} @@ -191,6 +193,15 @@ def check_if_dev_global(self): return addon['Development'] return 0 + def check_if_from_gh(self): # sourcery skip: sum-comprehension + if self.config['GHAPIKey'] != '': + return False + count = 0 + for addon in self.config['Addons']: + if addon['URL'].startswith('https://github.com/'): + count += 1 + return count > 3 + def cleanup(self, directories): if len(directories) > 0: for directory in directories: @@ -203,7 +214,8 @@ def parse_url(self, url): elif url.startswith('https://www.wowinterface.com/downloads/'): return WoWInterfaceAddon(url, self.wowiCache, self.http) elif url.startswith('https://github.com/'): - return GitHubAddon(url, self.clientType, self.config['GHAPIKey'], self.http) + return GitHubAddon(url, self.githubCache, self.githubPackagerCache, self.clientType, + self.config['GHAPIKey'], self.http) elif url.lower() == 'elvui': self.bulk_tukui_check() return TukuiAddon('elvui', self.tukuiCache, @@ -504,41 +516,82 @@ def parse_wagoapp_payload(self, url): payload = payload.json() return f'https://addons.wago.io/addons/{payload["slug"]}' - # TODO Improve the check when Wago Addons API will be will be expanded - def bulk_check(self, addons): # sourcery skip: extract-method + def bulk_check(self, addons): ids_wowi = [] ids_wago = [] + ids_gh = [] for addon in addons: if addon['URL'].startswith('https://www.wowinterface.com/downloads/'): ids_wowi.append(re.findall(r'\d+', addon['URL'])[0].strip()) elif addon['URL'].startswith('https://addons.wago.io/addons/') and \ addon['URL'] not in self.config['IgnoreClientVersion'].keys(): ids_wago.append({'slug': addon['URL'].replace('https://addons.wago.io/addons/', ''), 'id': ''}) + elif addon['URL'].startswith('https://github.com/'): + ids_gh.append(addon['URL'].replace('https://github.com/', '')) if ids_wowi: - payload = self.http.get(f'https://api.mmoui.com/v3/game/WOW/filedetails/{",".join(ids_wowi)}.json').json() - if 'ERROR' not in payload: - for addon in payload: - self.wowiCache[str(addon['UID'])] = addon + self.bulk_wowi_check(ids_wowi) if ids_wago and self.config['WAAAPIKey'] != '': - if not self.wagoIdCache: - self.wagoIdCache = self.http.get(f'https://addons.wago.io/api/data/slugs?game_version=' - f'{self.clientType}') - self.parse_wagoaddons_error(self.wagoIdCache.status_code) - self.wagoIdCache = self.wagoIdCache.json() - for addon in ids_wago: - if addon['slug'] in self.wagoIdCache['addons']: - addon['id'] = self.wagoIdCache['addons'][addon['slug']]['id'] - payload = self.http.post(f'https://addons.wago.io/api/external/addons/_recents?game_version=' - f'{self.clientType}', - json={'addons': [addon["id"] for addon in ids_wago if addon["id"] != ""]}, - auth=APIAuth('Bearer', self.config['WAAAPIKey'])) - self.parse_wagoaddons_error(payload.status_code) - payload = payload.json() - for addonid in payload['addons']: - for addon in ids_wago: - if addon['id'] == addonid: - self.wagoCache[addon['slug']] = payload['addons'][addonid] - break + self.bulk_wago_check(ids_wago) + if ids_gh and self.config['GHAPIKey'] != '': + self.bulk_gh_check(ids_gh) + + def bulk_wowi_check(self, ids): + payload = self.http.get(f'https://api.mmoui.com/v3/game/WOW/filedetails/{",".join(ids)}.json').json() + if 'ERROR' not in payload: + for addon in payload: + self.wowiCache[str(addon['UID'])] = addon + + def bulk_wago_check(self, ids): + if not self.wagoIdCache: + self.wagoIdCache = self.http.get(f'https://addons.wago.io/api/data/slugs?game_version={self.clientType}') + self.parse_wagoaddons_error(self.wagoIdCache.status_code) + self.wagoIdCache = self.wagoIdCache.json() + for addon in ids: + if addon['slug'] in self.wagoIdCache['addons']: + addon['id'] = self.wagoIdCache['addons'][addon['slug']]['id'] + payload = self.http.post(f'https://addons.wago.io/api/external/addons/_recents?game_version={self.clientType}', + json={'addons': [addon["id"] for addon in ids if addon["id"] != ""]}, + auth=APIAuth('Bearer', self.config['WAAAPIKey'])) + self.parse_wagoaddons_error(payload.status_code) + payload = payload.json() + for addonid in payload['addons']: + for addon in ids: + if addon['id'] == addonid: + self.wagoCache[addon['slug']] = payload['addons'][addonid] + break + + def bulk_gh_check_worker(self, node_id, url): + return node_id, self.http.get(url, auth=APIAuth('token', self.config['GHAPIKey'])).json() + + def bulk_gh_check(self, ids): + query = ('{\n "query": "{ search( type: REPOSITORY query: \\"' + f'repo:{" repo:".join(ids)}' + '\\" first: 10' + '0 ) { nodes { ... on Repository { nameWithOwner releases(first: 15) { nodes { tag_name: tagName name ' + 'html_url: url draft: isDraft prerelease: isPrerelease assets: releaseAssets(first: 100) { nodes { nod' + 'e_id: id name content_type: contentType url } } } } } } }}"\n}') + payload = self.http.post('https://api.github.com/graphql', json=json.loads(query), + auth=APIAuth('bearer', self.config['GHAPIKey'])) + if payload.status_code != 200: + return + payload = payload.json() + for addon in payload['data']['search']['nodes']: + self.githubCache[addon['nameWithOwner']] = addon['releases']['nodes'] + for addon in self.githubCache: + for i in range(len(self.githubCache[addon])): + self.githubCache[addon][i]['assets'] = self.githubCache[addon][i]['assets']['nodes'] + for release in self.githubCache[addon]: + if not release['draft'] and not release['prerelease']: + for asset in release['assets']: + if asset['name'] == 'release.json': + self.githubPackagerCache[asset['node_id']] = asset['url'] + break + break + with concurrent.futures.ThreadPoolExecutor() as executor: + workers = [] + for node_id, url in self.githubPackagerCache.items(): + workers.append(executor.submit(self.bulk_gh_check_worker, node_id, url)) + for future in concurrent.futures.as_completed(workers): + output = future.result() + self.githubPackagerCache[output[0]] = output[1] @retry(custom_error='Failed to parse Tukui API data') def bulk_tukui_check(self): diff --git a/CB/GitHub.py b/CB/GitHub.py index d8fd521..bbc00dc 100644 --- a/CB/GitHub.py +++ b/CB/GitHub.py @@ -9,34 +9,38 @@ # noinspection PyTypeChecker class GitHubAddon: @retry() - def __init__(self, url, clienttype, apikey, http): + def __init__(self, url, checkcache, packagercache, clienttype, apikey, http): project = url.replace('https://github.com/', '') self.http = http self.apiKey = apikey self.payloads = [] - try: - self.payload = self.http.get(f'https://api.github.com/repos/{project}/releases', - auth=APIAuth('token', self.apiKey)) - except httpx.RequestError as e: - raise RuntimeError(f'{project}\nGitHub API failed to respond.') from e - if self.payload.status_code == 401: - raise RuntimeError(f'{project}\nIncorrect or expired GitHub API personal access token.') - elif self.payload.status_code == 403: - raise RuntimeError(f'{project}\nGitHub API rate limit exceeded. Try later or provide personal access ' - f'token.') - elif self.payload.status_code == 404: - raise RuntimeError(url) + self.packagerCache = packagercache + if project in checkcache: + self.payload = checkcache[project] else: - self.payload = self.payload.json() - for release in self.payload: - if release['assets'] and len(release['assets']) > 0 \ - and not release['draft'] and not release['prerelease']: - self.payloads.append(release) - if len(self.payloads) > 14: - break - if not self.payloads: - raise RuntimeError(f'{url}\nThis integration supports only the projects that provide packaged' - f' releases.') + try: + self.payload = self.http.get(f'https://api.github.com/repos/{project}/releases', + auth=APIAuth('token', self.apiKey)) + except httpx.RequestError as e: + raise RuntimeError(f'{project}\nGitHub API failed to respond.') from e + if self.payload.status_code == 401: + raise RuntimeError(f'{project}\nIncorrect or expired GitHub API personal access token.') + elif self.payload.status_code == 403: + raise RuntimeError(f'{project}\nGitHub API rate limit exceeded. Try later or provide personal access ' + f'token.') + elif self.payload.status_code == 404: + raise RuntimeError(url) + else: + self.payload = self.payload.json() + for release in self.payload: + if release['assets'] and len(release['assets']) > 0 \ + and not release['draft'] and not release['prerelease']: + self.payloads.append(release) + if len(self.payloads) > 14: + break + if not self.payloads: + raise RuntimeError(f'{url}\nThis integration supports only the projects that provide packaged' + f' releases.') self.name = project.split('/')[1] self.clientType = clienttype self.currentVersion = None @@ -64,8 +68,11 @@ def parse(self): def parse_metadata(self): for release in self.payloads[self.releaseDepth]['assets']: if release['name'] and release['name'] == 'release.json': - self.metadata = self.http.get(release['url'], headers={'Accept': 'application/octet-stream'}, - auth=APIAuth('token', self.apiKey)).json() + if release['node_id'] in self.packagerCache: + self.metadata = self.packagerCache[release['node_id']] + else: + self.metadata = self.http.get(release['url'], headers={'Accept': 'application/octet-stream'}, + auth=APIAuth('token', self.apiKey)).json() break else: self.metadata = None diff --git a/CurseBreaker.py b/CurseBreaker.py index dfc3b6b..f39b14c 100644 --- a/CurseBreaker.py +++ b/CurseBreaker.py @@ -204,7 +204,7 @@ def start(self): else: self.console.print('Command not found.') - def auto_update(self): # sourcery skip: extract-method + def auto_update(self): # sourcery skip: extract-duplicate-method, extract-method if not getattr(sys, 'frozen', False) or 'CURSEBREAKER_VARDEXMODE' in os.environ: return try: @@ -540,11 +540,12 @@ def c_update(self, args, addline=False, update=True, force=False, reverseprovide exceptions = [] if len(addons) > 0: with Progress('{task.completed:.0f}/{task.total}', '|', BarColumn(bar_width=None), '|', - auto_refresh=False, console=None if self.headless else self.console) as progress: - task = progress.add_task('', total=len(addons)) + console=None if self.headless else self.console) as progress: + task = progress.add_task('', total=len(addons), start=bool(args)) if not args: - with suppress(RuntimeError): + with suppress(RuntimeError, httpx.RequestError): self.core.bulk_check(addons) + progress.start_task(task) self.core.bulk_check_checksum(addons, progress) while not progress.finished: for addon in addons: @@ -600,6 +601,9 @@ def c_update(self, args, addline=False, update=True, force=False, reverseprovide if overlap := self.core.check_if_overlap(): self.console.print(f'\n[bold red]Detected addon directory overlap. This will cause issues. Affected add' f'ons:[/bold red]\n{overlap}') + if self.core.check_if_from_gh(): + self.console.print('\n[bold red]Multiple addons acquired from GitHub have been detected. Providing a p' + 'ersonal GitHub token is highly recommended.[/bold red]') else: self.console.print('Apparently there are no addons installed by CurseBreaker (or you provided incorrect add' 'on name).\nCommand [green]import[/green] might be used to detect already installed addo' diff --git a/README.md b/README.md index 12e57ec..ecacf60 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ Windows 10+, Ubuntu 20.04+, Debian 11+ and macOS 11+ are supported. ## USAGE Place **CurseBreaker** binary inside the directory containing `Wow.exe`, `WowClassic.exe` or `World of Warcraft.app`.\ -Read the instructions on the top of the screen. +Read the instructions at the top of the screen. -Already installed addons will not be recognized by **CurseBreaker** and they need to be reinstalled.\ +Already installed addons will not be recognized by **CurseBreaker**, and they need to be reinstalled.\ This process can be partially automated by using the `import` command. _Retail_, _Cataclysm Classic_ and _Classic_ clients are supported. The client version is detected automatically.\ @@ -43,16 +43,17 @@ By default **CurseBreaker** will create backups of the entire `WTF` directory. To use Wago Addons as addon source user needs to provide a personal API key. It is a paid feature.\ The key can be obtained [here](https://addons.wago.io/patreon) and needs to be added to the application configuration by using the `set wago_addons_api` command. +## GITHUB SUPPORT +Providing [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) **greatly** increase speed of bulk version check and solve possible issues with rate limiting.\ +Both classic and fine-grained tokens are supported. No additional permissions are required.\ +Token can be added to application by using the `set gh_api` command. + ## WEAKAURAS SUPPORT **CurseBreaker** by default will try to update all detected WeakAuras and Plater profiles/scripts. The process works the same as WeakAuras Companion.\ All updates will still need to be applied in-game in the WeakAuras/Plater option menu.\ Command `toggle wago` can be used to set a single author name that will be ignored during the update.\ Additionally Wago API key can be set with the `set wa_api` command so non-public entries will also be upgradeable. -## GITHUB SUPPORT -When GitHub is frequently used as a source for addons there is the possibility of reaching a query limit.\ -If that occurs user must get a [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). Then add it to the application configuration by using the `set gh_api` command. - ## KNOWN ISSUES - Using WoWInterface projects that provide multiple addon releases ([example](https://www.wowinterface.com/downloads/info5086-BigWigsBossmods)) will always install a retail version of the addon. It can't be fixed as WoWInterface API doesn't support this type of project. - Some WoWInterface addon categories (e.g. Compilations, Optional) are not handled by their API. Addons in these categories can't be installed.