Added GitHub bulk version check

CB/Core.py

@@ -35,6 +35,8 @@ def __init__(self):
         self.dirIndex = None
         self.wowiCache = {}
         self.wagoCache = {}
+        self.githubCache = {}
+        self.githubPackagerCache = {}
         self.wagoIdCache = None
         self.tukuiCache = None
         self.checksumCache = {}
@@ -191,6 +193,15 @@ def check_if_dev_global(self):
                 return addon['Development']
         return 0
 
+    def check_if_from_gh(self):  # sourcery skip: sum-comprehension
+        if self.config['GHAPIKey'] != '':
+            return False
+        count = 0
+        for addon in self.config['Addons']:
+            if addon['URL'].startswith('https://github.com/'):
+                count += 1
+        return count > 3
+
     def cleanup(self, directories):
         if len(directories) > 0:
             for directory in directories:
@@ -203,7 +214,8 @@ def parse_url(self, url):
         elif url.startswith('https://www.wowinterface.com/downloads/'):
             return WoWInterfaceAddon(url, self.wowiCache, self.http)
         elif url.startswith('https://github.com/'):
-            return GitHubAddon(url, self.clientType, self.config['GHAPIKey'], self.http)
+            return GitHubAddon(url, self.githubCache, self.githubPackagerCache, self.clientType,
+                               self.config['GHAPIKey'], self.http)
         elif url.lower() == 'elvui':
             self.bulk_tukui_check()
             return TukuiAddon('elvui', self.tukuiCache,
@@ -504,41 +516,82 @@ def parse_wagoapp_payload(self, url):
         payload = payload.json()
         return f'https://addons.wago.io/addons/{payload["slug"]}'
 
-    # TODO Improve the check when Wago Addons API will be will be expanded
-    def bulk_check(self, addons):  # sourcery skip: extract-method
+    def bulk_check(self, addons):
         ids_wowi = []
         ids_wago = []
+        ids_gh = []
         for addon in addons:
             if addon['URL'].startswith('https://www.wowinterface.com/downloads/'):
                 ids_wowi.append(re.findall(r'\d+', addon['URL'])[0].strip())
             elif addon['URL'].startswith('https://addons.wago.io/addons/') and \
                     addon['URL'] not in self.config['IgnoreClientVersion'].keys():
                 ids_wago.append({'slug': addon['URL'].replace('https://addons.wago.io/addons/', ''), 'id': ''})
+            elif addon['URL'].startswith('https://github.com/'):
+                ids_gh.append(addon['URL'].replace('https://github.com/', ''))
         if ids_wowi:
-            payload = self.http.get(f'https://api.mmoui.com/v3/game/WOW/filedetails/{",".join(ids_wowi)}.json').json()
-            if 'ERROR' not in payload:
-                for addon in payload:
-                    self.wowiCache[str(addon['UID'])] = addon
+            self.bulk_wowi_check(ids_wowi)
         if ids_wago and self.config['WAAAPIKey'] != '':
-            if not self.wagoIdCache:
-                self.wagoIdCache = self.http.get(f'https://addons.wago.io/api/data/slugs?game_version='
-                                                 f'{self.clientType}')
-                self.parse_wagoaddons_error(self.wagoIdCache.status_code)
-                self.wagoIdCache = self.wagoIdCache.json()
-            for addon in ids_wago:
-                if addon['slug'] in self.wagoIdCache['addons']:
-                    addon['id'] = self.wagoIdCache['addons'][addon['slug']]['id']
-            payload = self.http.post(f'https://addons.wago.io/api/external/addons/_recents?game_version='
-                                     f'{self.clientType}',
-                                     json={'addons': [addon["id"] for addon in ids_wago if addon["id"] != ""]},
-                                     auth=APIAuth('Bearer', self.config['WAAAPIKey']))
-            self.parse_wagoaddons_error(payload.status_code)
-            payload = payload.json()
-            for addonid in payload['addons']:
-                for addon in ids_wago:
-                    if addon['id'] == addonid:
-                        self.wagoCache[addon['slug']] = payload['addons'][addonid]
-                        break
+            self.bulk_wago_check(ids_wago)
+        if ids_gh and self.config['GHAPIKey'] != '':
+            self.bulk_gh_check(ids_gh)
+
+    def bulk_wowi_check(self, ids):
+        payload = self.http.get(f'https://api.mmoui.com/v3/game/WOW/filedetails/{",".join(ids)}.json').json()
+        if 'ERROR' not in payload:
+            for addon in payload:
+                self.wowiCache[str(addon['UID'])] = addon
+
+    def bulk_wago_check(self, ids):
+        if not self.wagoIdCache:
+            self.wagoIdCache = self.http.get(f'https://addons.wago.io/api/data/slugs?game_version={self.clientType}')
+            self.parse_wagoaddons_error(self.wagoIdCache.status_code)
+            self.wagoIdCache = self.wagoIdCache.json()
+        for addon in ids:
+            if addon['slug'] in self.wagoIdCache['addons']:
+                addon['id'] = self.wagoIdCache['addons'][addon['slug']]['id']
+        payload = self.http.post(f'https://addons.wago.io/api/external/addons/_recents?game_version={self.clientType}',
+                                 json={'addons': [addon["id"] for addon in ids if addon["id"] != ""]},
+                                 auth=APIAuth('Bearer', self.config['WAAAPIKey']))
+        self.parse_wagoaddons_error(payload.status_code)
+        payload = payload.json()
+        for addonid in payload['addons']:
+            for addon in ids:
+                if addon['id'] == addonid:
+                    self.wagoCache[addon['slug']] = payload['addons'][addonid]
+                    break
+
+    def bulk_gh_check_worker(self, node_id, url):
+        return node_id, self.http.get(url, auth=APIAuth('token', self.config['GHAPIKey'])).json()
+
+    def bulk_gh_check(self, ids):
+        query = ('{\n  "query": "{ search( type: REPOSITORY query: \\"' + f'repo:{" repo:".join(ids)}' + '\\" first: 10'
+                 '0 ) { nodes { ... on Repository { nameWithOwner releases(first: 15) { nodes { tag_name: tagName name '
+                 'html_url: url draft: isDraft prerelease: isPrerelease assets: releaseAssets(first: 100) { nodes { nod'
+                 'e_id: id name content_type: contentType url } } } } } } }}"\n}')
+        payload = self.http.post('https://api.github.com/graphql', json=json.loads(query),
+                                 auth=APIAuth('bearer', self.config['GHAPIKey']))
+        if payload.status_code != 200:
+            return
+        payload = payload.json()
+        for addon in payload['data']['search']['nodes']:
+            self.githubCache[addon['nameWithOwner']] = addon['releases']['nodes']
+        for addon in self.githubCache:
+            for i in range(len(self.githubCache[addon])):
+                self.githubCache[addon][i]['assets'] = self.githubCache[addon][i]['assets']['nodes']
+            for release in self.githubCache[addon]:
+                if not release['draft'] and not release['prerelease']:
+                    for asset in release['assets']:
+                        if asset['name'] == 'release.json':
+                            self.githubPackagerCache[asset['node_id']] = asset['url']
+                            break
+                    break
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            workers = []
+            for node_id, url in self.githubPackagerCache.items():
+                workers.append(executor.submit(self.bulk_gh_check_worker, node_id, url))
+            for future in concurrent.futures.as_completed(workers):
+                output = future.result()
+                self.githubPackagerCache[output[0]] = output[1]
 
     @retry(custom_error='Failed to parse Tukui API data')
     def bulk_tukui_check(self):

CB/GitHub.py

@@ -9,34 +9,38 @@
 # noinspection PyTypeChecker
 class GitHubAddon:
     @retry()
-    def __init__(self, url, clienttype, apikey, http):
+    def __init__(self, url, checkcache, packagercache, clienttype, apikey, http):
         project = url.replace('https://github.com/', '')
         self.http = http
         self.apiKey = apikey
         self.payloads = []
-        try:
-            self.payload = self.http.get(f'https://api.github.com/repos/{project}/releases',
-                                         auth=APIAuth('token', self.apiKey))
-        except httpx.RequestError as e:
-            raise RuntimeError(f'{project}\nGitHub API failed to respond.') from e
-        if self.payload.status_code == 401:
-            raise RuntimeError(f'{project}\nIncorrect or expired GitHub API personal access token.')
-        elif self.payload.status_code == 403:
-            raise RuntimeError(f'{project}\nGitHub API rate limit exceeded. Try later or provide personal access '
-                               f'token.')
-        elif self.payload.status_code == 404:
-            raise RuntimeError(url)
+        self.packagerCache = packagercache
+        if project in checkcache:
+            self.payload = checkcache[project]
         else:
-            self.payload = self.payload.json()
-            for release in self.payload:
-                if release['assets'] and len(release['assets']) > 0 \
-                        and not release['draft'] and not release['prerelease']:
-                    self.payloads.append(release)
-                    if len(self.payloads) > 14:
-                        break
-            if not self.payloads:
-                raise RuntimeError(f'{url}\nThis integration supports only the projects that provide packaged'
-                                   f' releases.')
+            try:
+                self.payload = self.http.get(f'https://api.github.com/repos/{project}/releases',
+                                             auth=APIAuth('token', self.apiKey))
+            except httpx.RequestError as e:
+                raise RuntimeError(f'{project}\nGitHub API failed to respond.') from e
+            if self.payload.status_code == 401:
+                raise RuntimeError(f'{project}\nIncorrect or expired GitHub API personal access token.')
+            elif self.payload.status_code == 403:
+                raise RuntimeError(f'{project}\nGitHub API rate limit exceeded. Try later or provide personal access '
+                                   f'token.')
+            elif self.payload.status_code == 404:
+                raise RuntimeError(url)
+            else:
+                self.payload = self.payload.json()
+        for release in self.payload:
+            if release['assets'] and len(release['assets']) > 0 \
+                    and not release['draft'] and not release['prerelease']:
+                self.payloads.append(release)
+                if len(self.payloads) > 14:
+                    break
+        if not self.payloads:
+            raise RuntimeError(f'{url}\nThis integration supports only the projects that provide packaged'
+                               f' releases.')
         self.name = project.split('/')[1]
         self.clientType = clienttype
         self.currentVersion = None
@@ -64,8 +68,11 @@ def parse(self):
     def parse_metadata(self):
         for release in self.payloads[self.releaseDepth]['assets']:
             if release['name'] and release['name'] == 'release.json':
-                self.metadata = self.http.get(release['url'], headers={'Accept': 'application/octet-stream'},
-                                              auth=APIAuth('token', self.apiKey)).json()
+                if release['node_id'] in self.packagerCache:
+                    self.metadata = self.packagerCache[release['node_id']]
+                else:
+                    self.metadata = self.http.get(release['url'], headers={'Accept': 'application/octet-stream'},
+                                                  auth=APIAuth('token', self.apiKey)).json()
                 break
         else:
             self.metadata = None

CurseBreaker.py

@@ -204,7 +204,7 @@ def start(self):
                 else:
                     self.console.print('Command not found.')
 
-    def auto_update(self):  # sourcery skip: extract-method
+    def auto_update(self):  # sourcery skip: extract-duplicate-method, extract-method
         if not getattr(sys, 'frozen', False) or 'CURSEBREAKER_VARDEXMODE' in os.environ:
             return
         try:
@@ -540,11 +540,12 @@ def c_update(self, args, addline=False, update=True, force=False, reverseprovide
         exceptions = []
         if len(addons) > 0:
             with Progress('{task.completed:.0f}/{task.total}', '|', BarColumn(bar_width=None), '|',
-                          auto_refresh=False, console=None if self.headless else self.console) as progress:
-                task = progress.add_task('', total=len(addons))
+                          console=None if self.headless else self.console) as progress:
+                task = progress.add_task('', total=len(addons), start=bool(args))
                 if not args:
-                    with suppress(RuntimeError):
+                    with suppress(RuntimeError, httpx.RequestError):
                         self.core.bulk_check(addons)
+                    progress.start_task(task)
                     self.core.bulk_check_checksum(addons, progress)
                 while not progress.finished:
                     for addon in addons:
@@ -600,6 +601,9 @@ def c_update(self, args, addline=False, update=True, force=False, reverseprovide
             if overlap := self.core.check_if_overlap():
                 self.console.print(f'\n[bold red]Detected addon directory overlap. This will cause issues. Affected add'
                                    f'ons:[/bold red]\n{overlap}')
+            if self.core.check_if_from_gh():
+                self.console.print('\n[bold red]Multiple addons acquired from GitHub have been detected. Providing a p'
+                                   'ersonal GitHub token is highly recommended.[/bold red]')
         else:
             self.console.print('Apparently there are no addons installed by CurseBreaker (or you provided incorrect add'
                                'on name).\nCommand [green]import[/green] might be used to detect already installed addo'

README.md

@@ -13,9 +13,9 @@ Windows 10+, Ubuntu 20.04+, Debian 11+ and macOS 11+ are supported.
 
 ## USAGE
 Place **CurseBreaker** binary inside the directory containing `Wow.exe`, `WowClassic.exe` or `World of Warcraft.app`.\
-Read the instructions on the top of the screen.
+Read the instructions at the top of the screen.
 
-Already installed addons will not be recognized by **CurseBreaker** and they need to be reinstalled.\
+Already installed addons will not be recognized by **CurseBreaker**, and they need to be reinstalled.\
 This process can be partially automated by using the `import` command.
 
 _Retail_, _Cataclysm Classic_ and _Classic_ clients are supported. The client version is detected automatically.\
@@ -43,16 +43,17 @@ By default **CurseBreaker** will create backups of the entire `WTF` directory.
 To use Wago Addons as addon source user needs to provide a personal API key. It is a paid feature.\
 The key can be obtained [here](https://addons.wago.io/patreon) and needs to be added to the application configuration by using the `set wago_addons_api` command.
 
+## GITHUB SUPPORT
+Providing [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) **greatly** increase speed of bulk version check and solve possible issues with rate limiting.\
+Both classic and fine-grained tokens are supported. No additional permissions are required.\
+Token can be added to application by using the `set gh_api` command.
+
 ## WEAKAURAS SUPPORT
 **CurseBreaker** by default will try to update all detected WeakAuras and Plater profiles/scripts. The process works the same as WeakAuras Companion.\
 All updates will still need to be applied in-game in the WeakAuras/Plater option menu.\
 Command `toggle wago` can be used to set a single author name that will be ignored during the update.\
 Additionally Wago API key can be set with the `set wa_api` command so non-public entries will also be upgradeable.
 
-## GITHUB SUPPORT
-When GitHub is frequently used as a source for addons there is the possibility of reaching a query limit.\
-If that occurs user must get a [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). Then add it to the application configuration by using the `set gh_api` command.
-
 ## KNOWN ISSUES
 - Using WoWInterface projects that provide multiple addon releases ([example](https://www.wowinterface.com/downloads/info5086-BigWigsBossmods)) will always install a retail version of the addon. It can't be fixed as WoWInterface API doesn't support this type of project.
 - Some WoWInterface addon categories (e.g. Compilations, Optional) are not handled by their API. Addons in these categories can't be installed.