too much stuff to say... most notably new lfg system

Backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.3
+FROM python:3.11
 
 # we want stdout
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -8,10 +8,10 @@ ENV PYTHONUNBUFFERED=1
 ENV PYTHONPATH="${PYTHONPATH}:/app"
 
 # install the requirements
-RUN pip3.10 install uvloop
-RUN pip3.10 install psycopg2
+RUN pip3.11 install uvloop
+RUN pip3.11 install psycopg2
 COPY ./Backend/requirements.txt /app/Backend/requirements.txt
-RUN pip3.10 install -r /app/Backend/requirements.txt
+RUN pip3.11 install -r /app/Backend/requirements.txt
 
 # copy over the required files
 COPY ./settings.toml /app/settings.toml

Backend/backgroundEvents/rssFeedChecker.py

@@ -1,7 +1,5 @@
-import aiohttp
-import feedparser
-
 from Backend.backgroundEvents.base import BaseEvent
+from Backend.bungio.client import get_bungio_client
 from Backend.core.errors import CustomException
 from Backend.crud import persistent_messages, rss_feed
 from Backend.database.base import acquire_db_session
@@ -16,21 +14,14 @@ def __init__(self):
         super().__init__(scheduler_type="interval", interval_minutes=interval_minutes)
 
     async def run(self):
-        # use aiohttp to make the request instead of feedparsers request usage
-        async with aiohttp.ClientSession() as session:
-            async with session.get("https://www.bungie.net/en/rss/News") as resp:
-                if resp.status == 200:
-                    text = await resp.text()
-                else:
-                    return
+        bungio_client = get_bungio_client()
+        news = await bungio_client.api.rss_news_articles(page_token="0", includebody=False)
 
         async with acquire_db_session() as db:
-            feed = feedparser.parse(text)
-
             # loop through the articles and check if they have been published
             to_publish = []
-            for item in feed["entries"]:
-                if not await rss_feed.get(db=db, item_id=item["id"]):
+            for item in news.news_articles:
+                if not await rss_feed.get(db=db, item_id=item.unique_identifier):
                     to_publish.append(item)
                 else:
                     # dont need to re-check all of them every time
@@ -52,9 +43,9 @@ async def run(self):
                     elevator_api = ElevatorApi()
                     for item in to_publish:
                         data = {
-                            "message": bungie_url(item["link"]),
-                            "embed_title": None,
-                            "embed_description": None,
+                            "embed_title": item.title,
+                            "embed_description": f"[{item.description}]({bungie_url(item.link)})",
+                            "embed_image_url": item.image_path,
                             "guilds": subscribed_data,
                         }
 
@@ -73,7 +64,7 @@ async def run(self):
                                     )
 
                         # save item in DB
-                        await rss_feed.insert(db=db, item_id=item["id"])
+                        await rss_feed.insert(db=db, item_id=item.unique_identifier)
                 except CustomException:
                     pass
 

Backend/bungio/client.py

@@ -5,7 +5,7 @@
 from aiohttp_client_cache import RedisBackend
 from bungio import Client
 from bungio.error import InvalidAuthentication
-from bungio.http import HttpClient, Route
+from bungio.http import HttpClient, RateLimiter, Route
 from bungio.models import AuthData
 
 from Backend.database import acquire_db_session, is_test_mode, setup_engine
@@ -70,6 +70,12 @@ async def request(self, route: Route) -> dict:
 _BUNGIO_CLIENT: MyClient = None
 
 
+# do some monkey patching to add a ratelimiter for this route
+async def get_activity_history(self, *args, **kwargs):
+    await self.activity_updater_ratelimiter.wait_for_token()
+    await self._get_activity_history(*args, **kwargs)
+
+
 def get_bungio_client() -> MyClient:
     global _BUNGIO_CLIENT
 
@@ -100,5 +106,8 @@ def get_bungio_client() -> MyClient:
             manifest_storage=setup_engine(),
             http_client_class=MyHttpClient,
         )
+        _BUNGIO_CLIENT.api._get_activity_history = _BUNGIO_CLIENT.api.get_activity_history
+        _BUNGIO_CLIENT.api._get_activity_history = get_activity_history
+        _BUNGIO_CLIENT.api.activity_updater_ratelimiter = RateLimiter(max_tokens=230)
 
     return _BUNGIO_CLIENT

Backend/bungio/manifest.py

@@ -190,7 +190,10 @@ async def get_activity_mode(
         # sometimes bungie is not including some fields which is very annoying
         # luckily we can get the info from somewhere else
         if activity.direct_activity_mode_type is not MISSING and activity.activity_mode_types is not MISSING:
-            return DestinyActivityModeType(activity.direct_activity_mode_type), activity.activity_mode_types
+            try:
+                return DestinyActivityModeType(activity.direct_activity_mode_type), activity.activity_mode_types
+            except ValueError:
+                pass
 
         # fill out the data with the activityTypeHash field which is the key to *some* mode definitions
         result: DestinyActivityModeDefinition = await get_bungio_client().manifest.fetch(
@@ -222,13 +225,15 @@ async def get_all_activities(self) -> dict[int, DestinyActivityModel]:
                 # format them correctly
                 result = {}
                 for activities in data.values():
+                    mode, modes = await self.get_activity_mode(activities[0])
                     model = DestinyActivityModel(
                         name=activities[0].display_properties.name or "Unknown",
                         description=activities[0].display_properties.description,
                         matchmade=activities[0].matchmaking.is_matchmade if activities[0].matchmaking else False,
                         max_players=activities[0].matchmaking.max_players if activities[0].matchmaking else False,
                         activity_ids=[activity.hash for activity in activities],
-                        mode=(await self.get_activity_mode(activities[0]))[0].value,
+                        mode=mode.value,
+                        modes=[m.value for m in modes],
                         image_url=f"https://www.bungie.net/{activities[0].pgcr_image}"
                         if activities[0].pgcr_image
                         else None,
@@ -289,8 +294,9 @@ async def get_seals(self) -> dict[DestinyPresentationNodeDefinition, list[Destin
 
                 seals = []
                 for node in presentation_nodes:
-                    if DestinyPresentationNodesEnum.SEALS.value in node.parent_node_hashes:
-                        seals.append(node)
+                    if node.parent_node_hashes:
+                        if DestinyPresentationNodesEnum.SEALS.value in node.parent_node_hashes:
+                            seals.append(node)
 
                 # now loop through all the seals and get the record infos
                 for seal in seals:
@@ -372,12 +378,15 @@ async def get_grandmaster_nfs(self) -> list[DestinyActivityModel]:
                     matchmade=False,
                     max_players=3,
                     activity_ids=[],
+                    mode=DestinyActivityModeType.SCORED_NIGHTFALL.value,
+                    modes=[DestinyActivityModeType.SCORED_NIGHTFALL.value],
                 )
 
                 # format them correctly
                 result = []
                 for activities in gms.values():
                     reference_ids = [activity.hash for activity in activities]
+                    mode, modes = await self.get_activity_mode(activities[0])
                     all_grandmaster.activity_ids.extend(reference_ids)
                     result.append(
                         DestinyActivityModel(
@@ -386,7 +395,8 @@ async def get_grandmaster_nfs(self) -> list[DestinyActivityModel]:
                             matchmade=activities[0].matchmaking.is_matchmade,
                             max_players=activities[0].matchmaking.max_players,
                             activity_ids=reference_ids,
-                            mode=activities[0].direct_activity_mode_type,
+                            mode=mode.value,
+                            modes=[m.value for m in modes],
                             image_url=f"https://www.bungie.net/{activities[0].pgcr_image}"
                             if activities[0].pgcr_image
                             else None,
@@ -466,13 +476,15 @@ async def get_challenging_solo_activities(self) -> dict[str, list[DestinyActivit
                             data = None
                             for activity in db_result:
                                 if not data:
+                                    mode, modes = await self.get_activity_mode(activity)
                                     data = DestinyActivityModel(
                                         name=activity_name,
                                         description=activity.display_properties.description,
                                         matchmade=activity.matchmaking.is_matchmade,
                                         max_players=activity.matchmaking.max_players,
                                         activity_ids=[activity.hash],
-                                        mode=(await self.get_activity_mode(activity))[0].value,
+                                        mode=mode.value,
+                                        modes=[m.value for m in modes],
                                         image_url=f"https://www.bungie.net/{activity.pgcr_image}"
                                         if activity.pgcr_image
                                         else None,

Backend/core/destiny/activities.py

@@ -6,6 +6,7 @@
 
 from anyio import create_task_group, to_thread
 from bungio.error import BungieDead, BungIOException, InvalidAuthentication, TimeoutException
+from bungio.http import RateLimiter
 from bungio.models import DestinyActivityModeType, DestinyPostGameCarnageReportData
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -200,7 +201,11 @@ async def handle(
 
             try:
                 async with pgcr_getter_semaphore:
-                    pgcr = await get_bungio_client().api.get_post_game_carnage_report(i)
+                    # make get pgcrs at a lower rate to free space for other requests
+                    # prevents complete blockage
+                    await bungio_client.api.activity_updater_ratelimiter.wait_for_token()
+
+                    pgcr = await bungio_client.api.get_post_game_carnage_report(i)
                     results.append((i, t, pgcr))
 
             except Exception as e:
@@ -234,6 +239,8 @@ async def input_data(gather_instances: dict[int, datetime.datetime]):
         logger = logging.getLogger("updateActivityDb")
         logger_exceptions = logging.getLogger("updateActivityDbExceptions")
 
+        bungio_client = get_bungio_client()
+
         # get the destiny clan members for descend
         async with acquire_db_session() as session:
             clan = DestinyClan(db=session, guild_id=-1)

Backend/crud/destiny/lfgSystem.py

@@ -25,7 +25,7 @@ async def get_channel_id(db: AsyncSession, guild_id: int) -> int:
             cache_key = f"{guild_id}|lfg_channel"
 
             # populate cache
-            if cache_key not in cache.persistent_messages:
+            if not cache.persistent_messages.get(cache_key, None):
                 try:
                     await persistent_messages.get(db=db, guild_id=guild_id, message_name="lfg_channel")
                 except CustomException as e:

Backend/requirements.txt

@@ -1,28 +1,28 @@
-aiohttp[speedups]==3.8.1
-aiohttp_client_cache==0.5.2
-aioredis==1.3.1     # aiohttp_client_cache currently requires this version -> https://github.com/aio-libs/aioredis-py/issues/1273
+aiohttp[speedups]==3.8.3
+aiohttp_client_cache[redis]==0.7.3
+# aioredis==1.3.1     # aiohttp_client_cache currently requires this version -> https://github.com/aio-libs/aioredis-py/issues/1273
 # aiohttp_client_cache[redis]==0.7.0
 alembic==1.8.1
-anyio==3.6.1
+anyio==3.6.2
 apscheduler==3.9.1
-asyncpg==0.26.0
-bungio==0.6.6
-fastapi==0.81.0
-feedparser==6.0.10
+asyncpg==0.27.0
+attrs~=22.1.0
+bungio==0.7.0
+fastapi==0.88.0
 ics==0.7.2
 matplotlib==3.5.3
-orjson==3.8.0
-pandas==1.4.3
+orjson==3.8.3
+pandas==1.5.2
 passlib[bcrypt]==1.7.4
-prometheus_client==0.14.1
-psutil==5.9.1
-pydantic==1.10.0
+prometheus_client==0.15.0
+psutil==5.9.4
+pydantic==1.10.2
 python-dateutil==2.8.2
 python-jose[cryptography]>=3.3.0
 python-multipart==0.0.5
 requests==2.28.1
-rich==12.5.1
-sqlalchemy==1.4.41
+rich==12.6.0
+sqlalchemy==1.4.44
 toml==0.10.2
-tzdata==2022.2
-uvicorn==0.18.3
+tzdata==2022.7
+uvicorn==0.20.0

ElevatorBot/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.3
+FROM python:3.11
 
 # we want stdout
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -12,9 +12,9 @@ RUN apt-get -y update
 RUN apt-get -y install ffmpeg
 
 # install the requirements
-RUN pip3.10 install uvloop
+RUN pip3.11 install uvloop
 COPY ./ElevatorBot/requirements.txt /app/ElevatorBot/requirements.txt
-RUN pip3.10 install -r /app/ElevatorBot/requirements.txt
+RUN pip3.11 install -r /app/ElevatorBot/requirements.txt
 
 # copy over the required files
 COPY ./version.py /app/version.py
@@ -24,5 +24,5 @@ COPY ./ElevatorBot /app/ElevatorBot
 
 # start elevator
 WORKDIR /app
-CMD ["python3.10", "ElevatorBot/elevator.py"]
+CMD ["python3.11", "ElevatorBot/elevator.py"]
 EXPOSE $ELEVATOR_PORT

ElevatorBot/commandHelpers/paginator.py

@@ -43,7 +43,7 @@ async def paginate(
 
     # wait for a button press for 60s
     def check(component_check: Component):
-        return component_check.context.author == ctx.author
+        return component_check.ctx.author == ctx.author
 
     try:
         component = await ctx.bot.wait_for_component(
@@ -64,8 +64,8 @@ def check(component_check: Component):
         await callback(
             ctx=ctx,
             member=member,
-            key=component.context.custom_id.split("|")[2],
-            button_ctx=component.context,
+            key=component.ctx.custom_id.split("|")[2],
+            button_ctx=component.ctx,
             message=message,
             data=callback_data,
         )

ElevatorBot/commands/a_destiny/lfg/create.py

@@ -82,6 +82,7 @@ async def create(
                 ),
             ],
         )
+
         await ctx.send_modal(modal=modal)
 
         # wait 5 minutes for them to fill it out

ElevatorBot/commands/a_destiny/lfg/delete.py

@@ -30,7 +30,7 @@ async def delete(self, ctx: ElevatorInteractionContext, lfg_id: int):
             if not await has_admin_permission(ctx=ctx, member=ctx.author):
                 return
 
-        await lfg_message.delete()
+        await lfg_message.delete(hard=True)
         await ctx.send(
             ephemeral=True,
             embeds=embed_message("Success", f"The LFG event with the id `{lfg_id}` has been deleted"),

ElevatorBot/commands/a_destiny/lfg/share.py

@@ -57,7 +57,7 @@ async def share(self, ctx: ElevatorInteractionContext, lfg_id: int, channel: Gui
         message = await channel.send(
             embeds=embed_message(
                 f"Sharing LFG Event",
-                f"Activity: **{lfg_message.activity}**\nJoined: **{len(lfg_message.joined_ids)}/{lfg_message.max_joined_members}**\nStarting: {timestamp.format(style=TimestampStyles.ShortDateTime)} - {timestamp.format(style=TimestampStyles.RelativeTime)}",
+                f"Activity: **{lfg_message.activity}**\nStarting: {timestamp.format(style=TimestampStyles.ShortDateTime)} - {timestamp.format(style=TimestampStyles.RelativeTime)}\nDescription: ```{lfg_message.description}```",
             ),
             components=Button(
                 style=ButtonStyles.URL,

ElevatorBot/commands/a_destiny/seasonalChallenges.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from naff import ActionRow, Member, Message, Select, SelectOption, slash_command
+from naff import ActionRow, Member, Message, SelectOption, StringSelectMenu, slash_command
 from naff.api.events import Component
 
 from ElevatorBot.commandHelpers.optionTemplates import default_user_option
@@ -32,7 +32,7 @@ async def challenges(
         # create select components
         select = [
             ActionRow(
-                Select(
+                StringSelectMenu(
                     options=[
                         SelectOption(
                             emoji="📅",
@@ -96,7 +96,7 @@ async def _send_challenge_info(
 
         # wait 60s for selection
         def check(component_check: Component):
-            return component_check.context.author == author
+            return component_check.ctx.author == author
 
         try:
             component: Component = await self.client.wait_for_component(components=select, timeout=60, check=check)
@@ -105,7 +105,7 @@ def check(component_check: Component):
             return
         else:
             # noinspection PyTypeChecker
-            select_ctx: ElevatorComponentContext = component.context
+            select_ctx: ElevatorComponentContext = component.ctx
             new_week = select_ctx.values[0]
 
             # recursively call this function

ElevatorBot/commands/a_destiny/website.py

@@ -110,7 +110,7 @@ async def website(self, ctx: ElevatorInteractionContext, website: str, user: Mem
                 ),
             ),
         ]
-        await ctx.send(content="⁣", components=components)
+        await ctx.send(components=components)
 
 
 def setup(client):