merge dev into master

anyway/infographic_image_generator.py

@@ -18,7 +18,7 @@
 selenium_hub_url = f"https://{selenium_hub_url}/wd/hub"
 selenium_remote_results_url = f"https://{selenium_url}/tempdata"
 CHROME_PARTIALLY_DOWNLOADED_FILE_EXTENSION = "crdownload"
-
+SLEEP_DURATION_FOR_MAP_TO_HAVE_FOCUS = 5
 
 def create_chrome_browser_session(newsflash_id):
     options = webdriver.ChromeOptions()
@@ -29,6 +29,7 @@ def create_chrome_browser_session(newsflash_id):
         "profile.content_settings.exceptions.automatic_downloads.*.setting": 1,
     }
     options.add_experimental_option("prefs", prefs)
+    options.add_argument("--incognito")
 
     browser = webdriver.Remote(
         command_executor=selenium_hub_url,
@@ -94,7 +95,9 @@ def generate_infographics_in_selenium_container(browser, newsflash_id):
         logging.debug(f"found {buttons_found} buttons")
         if buttons_found > 0:
             for element in elements:
-                ActionChains(browser).move_to_element(element).click().perform()
+                ActionChains(browser).move_to_element(element).perform()
+                time.sleep(SLEEP_DURATION_FOR_MAP_TO_HAVE_FOCUS) #without sleep map infographic may be blurry
+                element.click()
                 time.sleep(1) #prevents click arriving before the last finished
             is_download_done, generated_images_names = wait_for_folder_to_contain_all_files(newsflash_id,
                                                                                             buttons_found, timeout=60)

anyway/infographics_utils.py

@@ -212,8 +212,8 @@ def get_infographics_data_for_location(request_params: RequestParams) -> Dict:
     elif WIDGETS not in output:
         logging.error(f"get_infographics_data: 'widgets' key missing from output:{output}")
     else:
-        non_empty = list(filter(lambda x: x[DATA][ITEMS], output[WIDGETS]))
-        output[WIDGETS] = update_cache_results(request_params, non_empty)
+        updated_output = update_cache_results(request_params, output[WIDGETS])
+        output[WIDGETS] = list(filter(lambda x: x[DATA][ITEMS], updated_output))
     return output
 
 
@@ -265,3 +265,8 @@ def get_infographics_mock_data():
             mock_data[WIDGETS].append(widget)
     mock_data[WIDGETS] = sorted(mock_data[WIDGETS], key=lambda widget: widget[META]["rank"])
     return mock_data
+
+
+def get_infographics_data_by_newsflash(newsflash_id):
+    request_params = get_request_params_from_request_values({"news_flash_id": newsflash_id})
+    return get_infographics_data_for_location(request_params)

anyway/telegram_accident_notifications.py

@@ -4,6 +4,7 @@
 from anyway.models import TelegramForwardedMessages
 from anyway.utilities import trigger_airflow_dag
 from anyway.app_and_db import db
+from anyway.infographics_utils import get_infographics_data_by_newsflash
 import telebot
 import boto3
 import time
@@ -50,11 +51,9 @@ def create_accident_text(newsflash_id):
     return f"{first_line}\n\n{newsflash['description']}"
 
 
-def fetch_transcription_by_widget_name(newsflash_id):
-    widgets_url = f"{ANYWAY_BASE_API_URL}/infographics-data?lang=he&news_flash_id={newsflash_id}&years_ago=5"
-    widgets_json = requests.get(widgets_url).json()
+def get_transcription_by_widget_name(widgets):
     transcription_by_widget_name = {widget["name"]: widget["data"]["text"]["transcription"]
-                                    for widget in widgets_json["widgets"]
+                                    for widget in widgets
                                     if "transcription" in widget["data"]["text"]}
     return transcription_by_widget_name
 
@@ -65,6 +64,8 @@ def send_after_infographics_message(bot, message_id_in_group, newsflash_id, link
     return bot.send_message(linked_group, message, reply_to_message_id=message_id_in_group)
 
 
+#this function sends the "root" message for the newsflash in telegram.
+#the flow continues when the telegram server sends a request to our /api/telegram/webhook
 def publish_notification(newsflash_id, chat_id=TELEGRAM_CHANNEL_CHAT_ID):
     accident_text = create_accident_text(newsflash_id)
     bot = telebot.TeleBot(secrets.get("BOT_TOKEN"))
@@ -77,23 +78,35 @@ def publish_notification(newsflash_id, chat_id=TELEGRAM_CHANNEL_CHAT_ID):
     db.session.commit()
 
 
+def get_items_for_send(newsflash_id):
+    items = []
+    widgets = get_infographics_data_by_newsflash(newsflash_id)["widgets"]
+    transcription_by_widget_name = get_transcription_by_widget_name(widgets)
+    urls_by_infographic_name = create_public_urls_for_infographics_images(str(newsflash_id))
+    for widget in widgets:
+        name = widget.get("name")
+        if name in urls_by_infographic_name:
+            url = urls_by_infographic_name.get(name)
+            text = transcription_by_widget_name.get(name)
+            items.append((url, text))
+    return items
+
+
 def send_infographics_to_telegram(root_message_id, newsflash_id, channel_of_initial_message):
     #every message in the channel is automatically forwarded to the linked discussion group.
     #to create a comment on the channel message, we need to send a reply to the
     #forwareded message in the discussion group.
     bot = telebot.TeleBot(secrets.get("BOT_TOKEN"))
 
-    transcription_by_widget_name = fetch_transcription_by_widget_name(newsflash_id)
-    urls_by_infographic_name = create_public_urls_for_infographics_images(str(newsflash_id))
-
     linked_group = telegram_linked_group_by_channel[channel_of_initial_message]
-    for infographic_name, url in urls_by_infographic_name.items():
-        text = transcription_by_widget_name[infographic_name] \
-            if infographic_name in transcription_by_widget_name else None
+    items_for_send = get_items_for_send(newsflash_id)
+    for url, text in items_for_send:
         bot.send_photo(linked_group, url, reply_to_message_id=root_message_id, caption=text)
+
     send_after_infographics_message(bot, root_message_id, newsflash_id, linked_group)
     logging.info("notification send done")
 
+
 def extract_infographic_name_from_s3_object(s3_object_name):
     left = s3_object_name.rindex("/")
     right = s3_object_name.rindex(".")

anyway/views/news_flash/api.py

@@ -375,8 +375,10 @@ def update_news_flash_qualifying(id):
             new_location=new_location,
             new_qualification=new_location_qualifiction,
         )
+        VERIFIED_QUALIFICATIONS = [NewsflashLocationQualification.MANUAL.value,
+                                   NewsflashLocationQualification.VERIFIED.value]
         if os.environ.get("FLASK_ENV") == "production" and \
-                new_location_qualifiction == NewsflashLocationQualification.MANUAL.value and \
+                new_location_qualifiction in VERIFIED_QUALIFICATIONS and \
                 old_location_qualifiction != NewsflashLocationQualification.MANUAL.value:
             trigger_generate_infographics_and_send_to_telegram(id, False)
         return Response(status=HTTPStatus.OK)

anyway/widgets/all_locations_widgets/accident_count_by_severity_widget.py

@@ -27,7 +27,7 @@ def generate_items(self) -> None:
         )
 
     def is_included(self) -> bool:
-        return self.items["total_accidents_count"] > 0
+        return self.items.get("total_accidents_count", 0) > 0
 
     @staticmethod
     def get_accident_count_by_severity(location_info, start_time, end_time, resolution):

anyway/widgets/all_locations_widgets/injured_count_by_severity_widget.py

@@ -33,7 +33,7 @@ def generate_items(self) -> None:
         )
 
     def is_included(self) -> bool:
-        return self.items["total_injured_count"] > 0
+        return self.items.get("total_injured_count", 0) > 0
 
     @staticmethod
     def get_injured_count_by_severity(

anyway/widgets/road_segment_widgets/street_view_widget.py

@@ -1,7 +1,8 @@
+import logging
 from anyway.request_params import RequestParams
 from anyway.widgets.widget import register
 from anyway.widgets.road_segment_widgets.road_segment_widget import RoadSegmentWidget
-from typing import Dict
+from typing import Dict, Optional
 from flask_babel import _
 
 
@@ -23,8 +24,14 @@ def generate_items(self) -> None:
     def is_included(self):
         return self.request_params.gps and self.request_params.gps.get("lon") and self.request_params.gps.get("lat")
 
-
     @staticmethod
     def localize_items(request_params: RequestParams, items: Dict) -> Dict:
         items["data"]["text"] = {"title": _("Street view widget")}
         return items
+
+    @classmethod
+    def update_result(cls, request_params: RequestParams, cached_items: Dict) -> Optional[Dict]:
+        try:
+            return cls.generate_widget_data(request_params)
+        except Exception as e:
+            logging.debug(f"Encountered error when generating items for widget class {cls} : {e}")