[RFR] update infra to use dynamic coordinates when locating elements (#…

…135)

* -add find_on_screen method that finds element on screen and returns Coordinates
-modify configuration initializing in Intellij.py
-modify image_locator method
-modify is is_open_mta_perspective and open_mta_perspective
-Update images

Signed-off-by: nhamza <[email protected]>

* pre commit changes

Signed-off-by: nhamza <[email protected]>

---------

Signed-off-by: nhamza <[email protected]>

src/models/IDE/Intellij.py

@@ -4,9 +4,12 @@
 import subprocess
 import time
 
+import pyautogui
+
 from src.models.application import Application
 from src.models.configuration.configurations_object import ConfigurationsObject
 from src.utils.general import read_file
+from src.utils.ocr import find_on_screen
 
 
 class Intellij(Application):
@@ -21,7 +24,7 @@ def __init__(self):
             x_coordinate=110,
             y_coordinate=470,
         )
-        self.configurations = []
+        self.configurations_object = ConfigurationsObject()
 
     def get_ide_version(self, ide_directory):
         info_file_path = os.path.join(ide_directory, "product-info.json")
@@ -49,7 +52,7 @@ def delete_all_configurations(self):
         self.press_keys("up")
         self.press_keys("enter")
         time.sleep(1)
-        self.configurations = []
+        self.configurations_object.configurations.clear()
 
     def create_configuration_in_ui(self):
         self.click(self.config_run_region)
@@ -66,62 +69,48 @@ def create_configuration_in_file(
         config,
         uuid,
     ):
-        configurations_object = ConfigurationsObject()
-        configuration = configurations_object.create(
+        self.configurations_object.create(
             analysis_data,
             app_name,
             application_config,
             config,
             uuid,
         )
-        self.configurations.append(configuration)
-        self.refresh_configuration()
 
     def image_locator(self, locator):
-        return f"image:{self.IMG_DIR}/intellij/{locator}"
+        return os.path.join(self.IMG_DIR, "intellij", locator)
 
     def is_open_mta_perspective(self):
         """
-        Checks if MTA perspective is already opened in IntelliJ
+        Checks if the MTA perspective is currently visible on the screen.
 
         Returns:
-            (bool): True or False
+            bool: True if the MTA perspective is visible, False otherwise.
         """
-        try:
-            self.wait_find_element(locator_type="image", locator="mta_perspective_active.png")
+        mta_perspective_active_path = self.image_locator("mta_perspective_active.png")
+        if find_on_screen(mta_perspective_active_path) is not None:
             return True
-        except Exception as exc:
-            try:
-                self.wait_find_element(
-                    locator_type="image",
-                    locator="mta_perspective_active_alt.png",
-                )
-                return True
-            except Exception:
-                logging.debug(
-                    "An error occured while finding \
-                    MTA perspective tab ! {}".format(
-                        str(exc),
-                    ),
-                )
-            if "No matches found" in str(exc):
-                return False
-            else:
-                raise Exception(exc)
+        else:
+            logging.info("MTA perspective not visible on the screen.")
+            return False
 
     def open_mta_perspective(self):
         """
-        Opens MTA perspective in IntelliJ
+        Opens MTA perspective
         """
         if self.is_open_mta_perspective():
-            logging.info("MTA perspective is already opened !")
+            logging.info("MTA perspective is already open!")
         else:
-            # Click on the MTA tab in left sidebar
-            self.click_element(locator_type="image", locator="mta_tab.png")
-
-        self.click(self.config_run_region)
+            coordinates = find_on_screen(self.image_locator("mta_tab.png"))
+            print(coordinates)
+            if coordinates is None:
+                logging.info("MTA tab not found!")
+                return False
+            else:
+                pyautogui.click(coordinates)
+            self.click(self.config_run_region)
 
-    def run_simple_analysis(self, app_name, wait_for_analysis_finish=False):
+    def run_simple_analysis(self, app_name, wait_for_analysis_finish=True):
         self.refresh_configuration()
 
         # Search for configuration name that has to be run

src/utils/ocr.py

@@ -1,9 +1,11 @@
+import logging
 import re
 
 import cv2
 import numpy as np
 import pyautogui
 import pytesseract
+from PIL import ImageGrab
 
 
 def find_all_string_occurrences(string):
@@ -25,6 +27,33 @@ def find_all_string_occurrences(string):
     return re.findall(re.escape(string), extracted_text)
 
 
+def find_on_screen(image_path, threshold=0.8):
+    """
+    Uses OpenCV to find the element on the screen based on the provided image path.
+    Args:
+        image_path (str): The path to the image file used as the template.
+        threshold (float): The threshold for the match (default is 0.8).
+    Returns:
+        Coordinates x,y of the found element
+    """
+    screen = np.array(ImageGrab.grab())  # Take a screenshot
+    screen_gray = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
+    template = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
+    if template is None:
+        logging.error(f"Template image not loaded: {image_path}")
+        return None
+    res = cv2.matchTemplate(screen_gray, template, cv2.TM_CCOEFF_NORMED)
+    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
+    if max_val >= threshold:
+        # Calculate the center of the template match
+        top_left = max_loc
+        w, h = template.shape[::-1]
+        center_x = top_left[0] + w // 2
+        center_y = top_left[1] + h // 2
+        return (center_x, center_y)
+    return None
+
+
 def find_all_sentence_occurrences(sentence):
     screenshot = pyautogui.screenshot()